From adf559c084d7fb32be0b1c32f7b18ed0160021b8 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Fri, 6 Dec 2024 15:48:55 +0000 Subject: [PATCH 01/31] coordinator: replace Teku ExecutionPayloadV1.kt by our own Domain Block --- build.gradle | 6 + coordinator/app/build.gradle | 4 +- .../zkevm/coordinator/app/CoordinatorApp.kt | 10 - .../zkevm/coordinator/app/L1DependentApp.kt | 10 +- .../blockcreation/BlockCreationMonitor.kt | 70 +-- .../GethCliqueSafeBlockProvider.kt | 22 +- .../ethereum/coordination/MaxLongTracker.kt | 2 +- .../blockcreation/BlockCreationMonitorTest.kt | 537 ------------------ .../file-based-client/build.gradle | 7 +- .../FileBasedExecutionProverClientV2.kt | 11 +- .../ExecutionProofRequestDataDecoratorTest.kt | 24 +- .../clients/shomei-client/build.gradle | 3 - coordinator/core/build.gradle | 3 +- .../BatchExecutionProverRequestResponse.kt | 9 +- .../net/consensys/zkevm/domain/Conflation.kt | 11 +- .../consensys/zkevm/encoding/BlockEncoder.kt | 7 + .../encoding/ExecutionPayloadV1Encoder.kt | 7 - .../blockcreation/BlockCreationCoordinator.kt | 4 +- .../blockcreation/SafeBlockProvider.kt | 45 +- .../BlockToBatchSubmissionCoordinator.kt | 66 +-- .../conflation/ConflationServiceImpl.kt | 29 +- .../ProofGeneratingConflationHandlerImpl.kt | 6 +- .../conflation/TracesConflationCalculator.kt | 4 +- .../upgrade/SwitchAwareConflationHandler.kt | 5 +- .../ZkProofCreationCoordinatorImpl.kt | 5 +- ...regationTriggerCalculatorByDeadlineTest.kt | 2 +- .../GlobalAggregationCalculatorTest.kt | 2 +- .../BlockToBatchSubmissionCoordinatorTest.kt | 23 +- .../ConflationCalculatorByTimeDeadlineTest.kt | 2 +- .../conflation/ConflationServiceImplTest.kt | 17 +- ...GlobalBlobAwareConflationCalculatorTest.kt | 2 +- .../GlobalBlockConflationCalculatorIntTest.kt | 2 +- .../SwitchAwareConflationHandlerTest.kt | 12 +- .../ethereum/blob-submitter/build.gradle | 1 - .../dynamiccap/GasPriceCapProviderImplTest.kt | 2 - .../ethereum/models-helper/build.gradle | 11 +- .../kotlin/linea/encoding/BlockRLPEncoder.kt | 8 + ...PayloadV1RLPEncoderByBesuImplementation.kt | 44 -- ...oadV1RLPEncoderByBesuImplementationTest.kt | 33 -- coordinator/persistence/blob/build.gradle | 1 - .../persistence/feehistory/build.gradle | 1 - gradle/libs.versions.toml | 1 + .../generic/extensions/futures/build.gradle | 2 +- .../kotlin/net/consensys/StringExtensions.kt | 6 + .../kotlin/net/consensys/TypingsExtensions.kt | 2 +- .../net/consensys/StringExtensionsTest.kt | 26 + jvm-libs/linea/besu-libs/build.gradle | 12 + .../linea/besu-rlp-and-mappers/build.gradle | 11 + .../linea/domain/MapperBesuToLineaDomain.kt | 57 ++ .../linea/domain/MapperLineaDomainToBesu.kt | 122 ++++ .../linea/domain/TransactionTypeMapper.kt | 23 + .../kotlin/linea/rlp/BesuRLPBlockEncoder.kt | 40 ++ .../src/main/kotlin/linea/rlp/RLP.kt | 53 ++ .../main/kotlin/linea/rlp/RLPBlockEncoder.kt | 18 + .../linea/core/domain-models/build.gradle | 2 + .../src/main/kotlin/linea/domain/Block.kt | 146 +++++ .../main/kotlin/linea/domain/Transaction.kt | 158 ++++++ .../kotlin/linea/domain/BlockFactory.kt | 47 ++ .../linea/teku-execution-client/build.gradle | 19 - .../linea/testing/teku-helper/build.gradle | 14 - .../schema/ExecutionPayloadV1.kt | 131 ----- jvm-libs/linea/web3j-extensions/build.gradle | 5 +- .../web3j/EthGetBlockToLineaBlockMappers.kt | 74 +++ .../main/kotlin/linea/web3j/Web3JFactory.kt | 33 ++ .../linea/web3j/DomainObjectMappers.kt | 138 ----- .../consensys/linea/web3j/ExtendedWeb3J.kt | 22 - .../EthGetBlockToLineaBlockMapperTest.kt | 112 ++++ .../linea/web3j/DomainObjectMappersTest.kt | 2 + settings.gradle | 3 +- transaction-decoder-tool/build.gradle | 2 +- .../kotlin/net/consensys/linea/BlockReader.kt | 23 - .../linea/TransactionEncodingToolMain.kt | 14 +- 72 files changed, 1154 insertions(+), 1234 deletions(-) delete mode 100644 coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitorTest.kt create mode 100644 coordinator/core/src/main/kotlin/net/consensys/zkevm/encoding/BlockEncoder.kt delete mode 100644 coordinator/core/src/main/kotlin/net/consensys/zkevm/encoding/ExecutionPayloadV1Encoder.kt create mode 100644 coordinator/ethereum/models-helper/src/main/kotlin/linea/encoding/BlockRLPEncoder.kt delete mode 100644 coordinator/ethereum/models-helper/src/main/kotlin/net/consensys/zkevm/encoding/ExecutionPayloadV1RLPEncoderByBesuImplementation.kt delete mode 100644 coordinator/ethereum/models-helper/src/test/kotlin/net/consensys/zkevm/encoding/ExecutionPayloadV1RLPEncoderByBesuImplementationTest.kt create mode 100644 jvm-libs/linea/besu-rlp-and-mappers/build.gradle create mode 100644 jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperBesuToLineaDomain.kt create mode 100644 jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperLineaDomainToBesu.kt create mode 100644 jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/TransactionTypeMapper.kt create mode 100644 jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRLPBlockEncoder.kt create mode 100644 jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/RLP.kt create mode 100644 jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/RLPBlockEncoder.kt create mode 100644 jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/Block.kt create mode 100644 jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/Transaction.kt create mode 100644 jvm-libs/linea/core/domain-models/src/testFixtures/kotlin/linea/domain/BlockFactory.kt delete mode 100644 jvm-libs/linea/teku-execution-client/build.gradle delete mode 100644 jvm-libs/linea/testing/teku-helper/build.gradle delete mode 100644 jvm-libs/linea/testing/teku-helper/src/main/kotlin/tech/pegasys/teku/ethereum/executionclient/schema/ExecutionPayloadV1.kt create mode 100644 jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/EthGetBlockToLineaBlockMappers.kt create mode 100644 jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/Web3JFactory.kt delete mode 100644 jvm-libs/linea/web3j-extensions/src/main/kotlin/net/consensys/linea/web3j/DomainObjectMappers.kt create mode 100644 jvm-libs/linea/web3j-extensions/src/test/kotlin/linea/web3j/EthGetBlockToLineaBlockMapperTest.kt delete mode 100644 transaction-decoder-tool/src/main/kotlin/net/consensys/linea/BlockReader.kt diff --git a/build.gradle b/build.gradle index 46307076e..28dfc60d5 100644 --- a/build.gradle +++ b/build.gradle @@ -23,6 +23,12 @@ allprojects { apply plugin: 'java' // do not add kotlin plugin here, it will add unnecessary Kotlin runtime dependencies apply plugin: 'jacoco' + configurations.all { + resolutionStrategy { + force "org.web3j:core:${libs.versions.web3j.get()}" + } + } + tasks.withType(KotlinCompile).configureEach { compileAll.dependsOn it compilerOptions { diff --git a/coordinator/app/build.gradle b/coordinator/app/build.gradle index bfb7f6f55..4a0e208ae 100644 --- a/coordinator/app/build.gradle +++ b/coordinator/app/build.gradle @@ -19,6 +19,7 @@ dependencies { implementation project(':jvm-libs:generic:vertx-helper') implementation project(':jvm-libs:generic:extensions:futures') implementation project(':jvm-libs:generic:persistence:db') + // implementation project(':jvm-libs:generic:serialization:jackson') implementation project(':jvm-libs:linea:web3j-extensions') implementation project(':jvm-libs:linea:core:metrics') implementation project(':jvm-libs:linea:metrics:micrometer') @@ -41,8 +42,6 @@ dependencies { implementation project(':coordinator:persistence:batch') implementation project(':coordinator:persistence:feehistory') implementation project(':coordinator:persistence:db-common') - implementation project(":jvm-libs:linea:teku-execution-client") - implementation "tech.pegasys.teku.internal:bytes:${libs.versions.teku.get()}" implementation project(':coordinator:ethereum:gas-pricing:static-cap') implementation project(':coordinator:ethereum:gas-pricing:dynamic-cap') @@ -67,7 +66,6 @@ dependencies { implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${libs.versions.jackson.get()}") testImplementation "org.apache.logging.log4j:log4j-slf4j2-impl:${libs.versions.log4j.get()}" testImplementation project(':coordinator:ethereum:test-utils') - testImplementation project(':jvm-libs:linea:testing:teku-helper') testImplementation "io.vertx:vertx-junit5" } diff --git a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/CoordinatorApp.kt b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/CoordinatorApp.kt index 0adf224f8..1b1f75b69 100644 --- a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/CoordinatorApp.kt +++ b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/CoordinatorApp.kt @@ -1,9 +1,7 @@ package net.consensys.zkevm.coordinator.app -import com.fasterxml.jackson.databind.module.SimpleModule import io.micrometer.core.instrument.MeterRegistry import io.vertx.core.Vertx -import io.vertx.core.json.jackson.DatabindCodec import io.vertx.micrometer.backends.BackendRegistries import io.vertx.sqlclient.SqlClient import net.consensys.linea.async.toSafeFuture @@ -32,11 +30,9 @@ import net.consensys.zkevm.persistence.db.PersistenceRetryer import org.apache.logging.log4j.Level import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger -import org.apache.tuweni.bytes.Bytes import org.web3j.protocol.Web3j import org.web3j.protocol.http.HttpService import org.web3j.utils.Async -import tech.pegasys.teku.ethereum.executionclient.serialization.BytesSerializer import tech.pegasys.teku.infrastructure.async.SafeFuture import kotlin.time.toKotlinDuration @@ -48,12 +44,6 @@ class CoordinatorApp(private val configs: CoordinatorConfig) { log.debug("Vertx full configs: {}", vertxConfig) log.info("App configs: {}", configs) - // TODO: adapt JsonMessageProcessor to use custom ObjectMapper - // this is just dark magic. - val module = SimpleModule() - module.addSerializer(Bytes::class.java, BytesSerializer()) - DatabindCodec.mapper().registerModule(module) - // .enable(SerializationFeature.INDENT_OUTPUT) Vertx.vertx(vertxConfig) } private val meterRegistry: MeterRegistry = BackendRegistries.getDefaultNow() diff --git a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt index 369cc46f3..c88594551 100644 --- a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt +++ b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt @@ -6,6 +6,7 @@ import build.linea.contract.l1.LineaRollupSmartContractClientReadOnly import build.linea.contract.l1.Web3JLineaRollupSmartContractClientReadOnly import io.vertx.core.Vertx import kotlinx.datetime.Clock +import linea.encoding.BlockRLPEncoder import net.consensys.linea.BlockNumberAndHash import net.consensys.linea.blob.ShnarfCalculatorVersion import net.consensys.linea.contract.LineaRollupAsyncFriendly @@ -56,7 +57,6 @@ import net.consensys.zkevm.coordinator.clients.smartcontract.LineaRollupSmartCon import net.consensys.zkevm.domain.BlobSubmittedEvent import net.consensys.zkevm.domain.BlocksConflation import net.consensys.zkevm.domain.FinalizationSubmittedEvent -import net.consensys.zkevm.encoding.ExecutionPayloadV1RLPEncoderByBesuImplementation import net.consensys.zkevm.ethereum.coordination.EventDispatcher import net.consensys.zkevm.ethereum.coordination.HighestConflationTracker import net.consensys.zkevm.ethereum.coordination.HighestProvenBatchTracker @@ -323,7 +323,7 @@ class L1DependentApp( lastBlockNumber = lastProcessedBlockNumber, clock = Clock.System, latestBlockProvider = GethCliqueSafeBlockProvider( - l2ExtendedWeb3j, + l2ExtendedWeb3j.web3jClient, GethCliqueSafeBlockProvider.Config(configs.l2.blocksToFinalization.toLong()) ) ) @@ -608,7 +608,7 @@ class L1DependentApp( deadlineCheckInterval = configs.proofAggregation.deadlineCheckInterval.toKotlinDuration(), aggregationDeadline = configs.proofAggregation.aggregationDeadline.toKotlinDuration(), latestBlockProvider = GethCliqueSafeBlockProvider( - l2ExtendedWeb3j, + l2ExtendedWeb3j.web3jClient, GethCliqueSafeBlockProvider.Config(configs.l2.blocksToFinalization.toLong()) ), maxProofsPerAggregation = configs.proofAggregation.aggregationProofsLimit.toUInt(), @@ -868,7 +868,7 @@ class L1DependentApp( conflationService = conflationService, tracesCountersClient = tracesCountersClient, vertx = vertx, - payloadEncoder = ExecutionPayloadV1RLPEncoderByBesuImplementation + encoder = BlockRLPEncoder ) } @@ -904,7 +904,7 @@ class L1DependentApp( log.info("Resuming conflation from block={} inclusive", lastProcessedBlockNumber + 1UL) val blockCreationMonitor = BlockCreationMonitor( vertx = vertx, - extendedWeb3j = l2ExtendedWeb3j, + web3j = l2ExtendedWeb3j.web3jClient, startingBlockNumberExclusive = lastProcessedBlockNumber.toLong(), blockCreationListener = block2BatchCoordinator, lastProvenBlockNumberProviderAsync = lastProvenBlockNumberProvider, diff --git a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitor.kt b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitor.kt index 8b3b5db5b..2219a49e8 100644 --- a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitor.kt +++ b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitor.kt @@ -1,17 +1,20 @@ package net.consensys.zkevm.coordinator.blockcreation import io.vertx.core.Vertx +import linea.domain.Block +import linea.web3j.toDomain +import net.consensys.decodeHex +import net.consensys.encodeHex import net.consensys.linea.async.AsyncRetryer -import net.consensys.linea.web3j.ExtendedWeb3J +import net.consensys.linea.async.toSafeFuture import net.consensys.zkevm.PeriodicPollingService import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreated import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreationListener import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger -import org.apache.tuweni.bytes.Bytes32 +import org.web3j.protocol.Web3j import org.web3j.protocol.core.DefaultBlockParameter import org.web3j.protocol.core.methods.response.EthBlock -import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1 import tech.pegasys.teku.infrastructure.async.SafeFuture import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicLong @@ -21,7 +24,7 @@ import kotlin.time.Duration.Companion.days class BlockCreationMonitor( private val vertx: Vertx, - private val extendedWeb3j: ExtendedWeb3J, + private val web3j: Web3j, private val startingBlockNumberExclusive: Long, private val blockCreationListener: BlockCreationListener, private val lastProvenBlockNumberProviderAsync: LastProvenBlockNumberProviderAsync, @@ -41,7 +44,7 @@ class BlockCreationMonitor( ) private val _nexBlockNumberToFetch: AtomicLong = AtomicLong(startingBlockNumberExclusive + 1) - private val expectedParentBlockHash: AtomicReference = AtomicReference(null) + private val expectedParentBlockHash: AtomicReference = AtomicReference(null) private val reorgDetected: AtomicBoolean = AtomicBoolean(false) private var statingBlockAvailabilityFuture: SafeFuture<*>? = null @@ -82,19 +85,15 @@ class BlockCreationMonitor( false } else { log.info("Block {} found. Resuming block monitor", startingBlockNumberExclusive) - expectedParentBlockHash.set(Bytes32.fromHexString(block.block.hash)) + expectedParentBlockHash.set(block.block.hash.decodeHex()) true } } ) { - SafeFuture.of( - extendedWeb3j.web3jClient - .ethGetBlockByNumber( - DefaultBlockParameter.valueOf(startingBlockNumberExclusive.toBigInteger()), - false - ) - .sendAsync() - ) + web3j + .ethGetBlockByNumber(DefaultBlockParameter.valueOf(startingBlockNumberExclusive.toBigInteger()), false) + .sendAsync() + .toSafeFuture() } } @@ -128,29 +127,29 @@ class BlockCreationMonitor( SafeFuture.COMPLETE } else { getNetNextSafeBlock() - .thenCompose { payload -> - if (payload != null) { - if (payload.parentHash == expectedParentBlockHash.get()) { - notifyListener(payload) + .thenCompose { block -> + if (block != null) { + if (block.parentHash.contentEquals(expectedParentBlockHash.get())) { + notifyListener(block) .whenSuccess { log.debug( "updating nexBlockNumberToFetch from {} --> {}", _nexBlockNumberToFetch.get(), _nexBlockNumberToFetch.incrementAndGet() ) - expectedParentBlockHash.set(payload.blockHash) + expectedParentBlockHash.set(block.hash) } } else { reorgDetected.set(true) log.error( "Shooting down conflation poller, " + "chain reorg detected: block { blockNumber={} hash={} parentHash={} } should have parentHash={}", - payload.blockNumber.longValue(), - payload.blockHash.toHexString().subSequence(0, 8), - payload.parentHash.toHexString().subSequence(0, 8), - expectedParentBlockHash.get().toHexString().subSequence(0, 8) + block.number, + block.hash.encodeHex(), + block.parentHash.encodeHex(), + expectedParentBlockHash.get().encodeHex().subSequence(0, 8) ) - SafeFuture.failedFuture(IllegalStateException("Reorg detected on block ${payload.blockNumber}")) + SafeFuture.failedFuture(IllegalStateException("Reorg detected on block ${block.number}")) } } else { SafeFuture.completedFuture(Unit) @@ -165,36 +164,43 @@ class BlockCreationMonitor( } } - private fun notifyListener(payload: ExecutionPayloadV1): SafeFuture { - return blockCreationListener.acceptBlock(BlockCreated(payload)) + private fun notifyListener(payload: Block): SafeFuture { + return blockCreationListener + .acceptBlock(BlockCreated(payload)) .thenApply { log.debug( "blockCreationListener blockNumber={} resolved with success", - payload.blockNumber + payload.number ) } .whenException { throwable -> log.warn( "Failed to notify blockCreationListener: blockNumber={} errorMessage={}", - payload.blockNumber.bigIntegerValue(), + payload.number, throwable.message, throwable ) } } - private fun getNetNextSafeBlock(): SafeFuture { - return extendedWeb3j + private fun getNetNextSafeBlock(): SafeFuture { + return web3j .ethBlockNumber() + .sendAsync() + .toSafeFuture() + .thenApply { it.blockNumber } .thenCompose { latestBlockNumber -> // Check if is safe to fetch nextWaitingBlockNumber if (latestBlockNumber.toLong() >= _nexBlockNumberToFetch.get() + config.blocksToFinalization ) { val blockNumber = _nexBlockNumberToFetch.get() - extendedWeb3j.ethGetExecutionPayloadByNumber(blockNumber) + web3j.ethGetBlockByNumber(DefaultBlockParameter.valueOf(blockNumber.toBigInteger()), true) + .sendAsync() + .toSafeFuture() + .thenApply { it.block.toDomain() } .whenException { - log.error( + log.warn( "eth_getBlockByNumber({}) failed: errorMessage={}", blockNumber, it.message, diff --git a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/GethCliqueSafeBlockProvider.kt b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/GethCliqueSafeBlockProvider.kt index d0086888d..d66e80e0c 100644 --- a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/GethCliqueSafeBlockProvider.kt +++ b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/GethCliqueSafeBlockProvider.kt @@ -1,27 +1,31 @@ package net.consensys.zkevm.coordinator.blockcreation -import net.consensys.linea.web3j.ExtendedWeb3J +import build.linea.web3j.domain.toWeb3j +import linea.domain.Block +import linea.web3j.toDomain +import net.consensys.linea.BlockParameter.Companion.toBlockParameter +import net.consensys.linea.async.toSafeFuture import net.consensys.zkevm.ethereum.coordination.blockcreation.SafeBlockProvider +import org.web3j.protocol.Web3j import org.web3j.protocol.core.DefaultBlockParameterName -import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1 import tech.pegasys.teku.infrastructure.async.SafeFuture class GethCliqueSafeBlockProvider( - private val extendedWeb3j: ExtendedWeb3J, + private val web3j: Web3j, private val config: Config ) : SafeBlockProvider { data class Config( val blocksToFinalization: Long ) - override fun getLatestSafeBlock(): SafeFuture { - return SafeFuture.of( - extendedWeb3j.web3jClient - .ethGetBlockByNumber(DefaultBlockParameterName.LATEST, false).sendAsync() - ) + override fun getLatestSafeBlock(): SafeFuture { + return web3j + .ethGetBlockByNumber(DefaultBlockParameterName.LATEST, false).sendAsync() + .toSafeFuture() .thenCompose { block -> val safeBlockNumber = (block.block.number.toLong() - config.blocksToFinalization).coerceAtLeast(0) - extendedWeb3j.ethGetExecutionPayloadByNumber(safeBlockNumber) + web3j.ethGetBlockByNumber(safeBlockNumber.toBlockParameter().toWeb3j(), true).sendAsync().toSafeFuture() } + .thenApply { it.block.toDomain() } } } diff --git a/coordinator/app/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/MaxLongTracker.kt b/coordinator/app/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/MaxLongTracker.kt index c5c4dda77..fed87a9c1 100644 --- a/coordinator/app/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/MaxLongTracker.kt +++ b/coordinator/app/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/MaxLongTracker.kt @@ -34,7 +34,7 @@ class HighestProvenBatchTracker(initialProvenBlockNumber: ULong) : class HighestConflationTracker(initialProvenBlockNumber: ULong) : MaxLongTracker(initialProvenBlockNumber.toLong()) { override fun convertToLong(trackable: BlocksConflation): Long { - return trackable.blocks.last().blockNumber.longValue() + return trackable.blocks.last().number.toLong() } } diff --git a/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitorTest.kt b/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitorTest.kt deleted file mode 100644 index 81577394b..000000000 --- a/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitorTest.kt +++ /dev/null @@ -1,537 +0,0 @@ -package net.consensys.zkevm.coordinator.blockcreation - -import io.vertx.core.Vertx -import io.vertx.junit5.VertxExtension -import io.vertx.junit5.VertxTestContext -import net.consensys.ByteArrayExt -import net.consensys.decodeHex -import net.consensys.linea.async.get -import net.consensys.linea.web3j.ExtendedWeb3J -import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreated -import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreationListener -import org.apache.logging.log4j.Logger -import org.assertj.core.api.Assertions.assertThat -import org.awaitility.Awaitility.await -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.extension.ExtendWith -import org.mockito.ArgumentMatchers.eq -import org.mockito.Mockito.atLeastOnce -import org.mockito.kotlin.any -import org.mockito.kotlin.atLeast -import org.mockito.kotlin.atMost -import org.mockito.kotlin.doReturn -import org.mockito.kotlin.mock -import org.mockito.kotlin.never -import org.mockito.kotlin.times -import org.mockito.kotlin.verify -import org.mockito.kotlin.whenever -import org.web3j.protocol.Web3j -import org.web3j.protocol.core.Request -import org.web3j.protocol.core.methods.response.EthBlock -import tech.pegasys.teku.ethereum.executionclient.schema.executionPayloadV1 -import tech.pegasys.teku.infrastructure.async.SafeFuture -import java.math.BigInteger -import java.util.concurrent.Executors -import java.util.concurrent.TimeUnit -import kotlin.time.Duration -import kotlin.time.Duration.Companion.milliseconds -import kotlin.time.Duration.Companion.seconds -import kotlin.time.toJavaDuration - -@ExtendWith(VertxExtension::class) -class BlockCreationMonitorTest { - private val parentHash = "0x1000000000000000000000000000000000000000000000000000000000000000".decodeHex() - private val startingBlockNumberInclusive: Long = 100 - private val blocksToFetch: Long = 5L - private val lastBlockNumberInclusiveToProcess: ULong = startingBlockNumberInclusive.toULong() + 10uL - private lateinit var log: Logger - private lateinit var web3jNativeClientMock: Web3j - private lateinit var web3jClient: ExtendedWeb3J - private lateinit var blockCreationListener: BlockCreationListener - private var lastProvenBlock: Long = startingBlockNumberInclusive - private var config: BlockCreationMonitor.Config = - BlockCreationMonitor.Config( - pollingInterval = 100.milliseconds, - blocksToFinalization = 2L, - blocksFetchLimit = blocksToFetch, - lastL2BlockNumberToProcessInclusive = lastBlockNumberInclusiveToProcess - ) - private val executor = Executors.newSingleThreadScheduledExecutor() - private lateinit var monitor: BlockCreationMonitor - - @BeforeEach - fun beforeEach(vertx: Vertx) { - val ethGetBlockByNumberMock: Request = mock { - on { sendAsync() } doReturn SafeFuture.completedFuture(EthBlock()) - on { sendAsync() } doReturn SafeFuture.completedFuture(EthBlock()) - on { sendAsync() } doReturn SafeFuture.completedFuture( - EthBlock().apply { - result = EthBlock.Block() - .apply { - setNumber("0x63") - hash = "0x1000000000000000000000000000000000000000000000000000000000000000" - } - } - ) - } - - web3jNativeClientMock = mock { - on { ethGetBlockByNumber(any(), any()) } doReturn ethGetBlockByNumberMock - } - web3jClient = mock { - on { web3jClient } doReturn web3jNativeClientMock - } - blockCreationListener = - mock { on { acceptBlock(any()) } doReturn SafeFuture.completedFuture(Unit) } - - log = mock() - - val lastProvenBlockNumberProviderAsync = object : LastProvenBlockNumberProviderAsync { - override fun getLastProvenBlockNumber(): SafeFuture { - return SafeFuture.completedFuture(lastProvenBlock) - } - } - - monitor = - BlockCreationMonitor( - vertx, - web3jClient, - startingBlockNumberExclusive = startingBlockNumberInclusive - 1, - blockCreationListener, - lastProvenBlockNumberProviderAsync, - config - ) - } - - @AfterEach - fun afterEach(vertx: Vertx) { - monitor.stop() - vertx.close().get() - } - - @Test - fun `skip blocks after lastBlockNumberInclusiveToProcess`(vertx: Vertx, testContext: VertxTestContext) { - val lastProvenBlockNumberProviderAsync = mock() - whenever(lastProvenBlockNumberProviderAsync.getLastProvenBlockNumber()).thenAnswer { - SafeFuture.completedFuture((lastBlockNumberInclusiveToProcess - 2uL).toLong()) - } - - monitor = - BlockCreationMonitor( - vertx, - web3jClient, - startingBlockNumberExclusive = (lastBlockNumberInclusiveToProcess - 2uL).toLong(), - blockCreationListener, - lastProvenBlockNumberProviderAsync, - config - ) - val payload = - executionPayloadV1(blockNumber = lastBlockNumberInclusiveToProcess.toLong() - 1, parentHash = parentHash) - val payload2 = - executionPayloadV1( - blockNumber = lastBlockNumberInclusiveToProcess.toLong(), - parentHash = payload.blockHash.toArray() - ) - val payload3 = - executionPayloadV1( - blockNumber = lastBlockNumberInclusiveToProcess.toLong() + 1, - parentHash = payload2.blockHash.toArray() - ) - - val headBlockNumber = lastBlockNumberInclusiveToProcess.toLong() + config.blocksToFinalization - whenever(web3jClient.ethBlockNumber()) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber))) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 1))) - whenever(web3jClient.ethGetExecutionPayloadByNumber(any())) - .thenReturn(SafeFuture.completedFuture(payload)) - .thenReturn(SafeFuture.completedFuture(payload2)) - .thenReturn(SafeFuture.completedFuture(payload3)) - whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit)) - - monitor.start().thenApply { - await() - .untilAsserted { - verify(lastProvenBlockNumberProviderAsync, atLeast(3)).getLastProvenBlockNumber() - verify(web3jClient).ethGetExecutionPayloadByNumber(eq(lastBlockNumberInclusiveToProcess.toLong() - 1)) - verify(web3jClient).ethGetExecutionPayloadByNumber(eq(lastBlockNumberInclusiveToProcess.toLong())) - verify(web3jClient, never()).ethGetExecutionPayloadByNumber( - eq(lastBlockNumberInclusiveToProcess.toLong() + 1) - ) - verify(blockCreationListener, times(1)).acceptBlock(BlockCreated(payload)) - verify(blockCreationListener, times(1)).acceptBlock(BlockCreated(payload2)) - verify(blockCreationListener, never()).acceptBlock(BlockCreated(payload3)) - assertThat(monitor.nexBlockNumberToFetch).isEqualTo(lastBlockNumberInclusiveToProcess.toLong() + 1) - } - testContext.completeNow() - } - .whenException(testContext::failNow) - } - - @Test - fun `notifies listener after block is finalized sync`(vertx: Vertx, testContext: VertxTestContext) { - val payload = - executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash) - val payload2 = - executionPayloadV1( - blockNumber = startingBlockNumberInclusive + 1, - parentHash = payload.blockHash.toArray() - ) - val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization - whenever(web3jClient.ethBlockNumber()) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber))) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 1))) - whenever(web3jClient.ethGetExecutionPayloadByNumber(any())) - .thenReturn(SafeFuture.completedFuture(payload)) - .thenReturn(SafeFuture.completedFuture(payload2)) - whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit)) - - val lastProvenBlockNumberProviderAsync = mock() - - val monitor = - BlockCreationMonitor( - vertx, - web3jClient, - startingBlockNumberExclusive = startingBlockNumberInclusive - 1, - blockCreationListener, - lastProvenBlockNumberProviderAsync, - config - ) - whenever(lastProvenBlockNumberProviderAsync.getLastProvenBlockNumber()).thenAnswer { - SafeFuture.completedFuture(lastProvenBlock) - } - monitor.start().thenApply { - await() - .untilAsserted { - verify(lastProvenBlockNumberProviderAsync, atLeast(3)).getLastProvenBlockNumber() - verify(web3jClient).ethGetExecutionPayloadByNumber(eq(startingBlockNumberInclusive)) - verify(web3jClient).ethGetExecutionPayloadByNumber(eq(startingBlockNumberInclusive + 1)) - verify(blockCreationListener).acceptBlock(BlockCreated(payload)) - verify(blockCreationListener).acceptBlock(BlockCreated(payload2)) - assertThat(monitor.nexBlockNumberToFetch).isEqualTo(startingBlockNumberInclusive + 2) - } - testContext.completeNow() - } - .whenException(testContext::failNow) - } - - @Test - fun `does not notify listener when block is not safely finalized`(testContext: VertxTestContext) { - val payload = - executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash) - val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization - 1 - whenever(web3jClient.ethBlockNumber()) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber))) - whenever(web3jClient.ethGetExecutionPayloadByNumber(any())) - .thenReturn(SafeFuture.completedFuture(payload)) - whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit)) - - monitor.start().thenApply { - await() - .timeout(1.seconds.toJavaDuration()) - .untilAsserted { - verify(web3jClient, never()).ethGetExecutionPayloadByNumber(any()) - verify(blockCreationListener, never()).acceptBlock(BlockCreated(payload)) - assertThat(monitor.nexBlockNumberToFetch).isEqualTo(startingBlockNumberInclusive) - } - testContext.completeNow() - } - .whenException(testContext::failNow) - } - - @Test - fun `when listener throws retries on the next tick and moves on`(testContext: VertxTestContext) { - val payload = - executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash) - val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization + 1 - whenever(web3jClient.ethBlockNumber()) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber))) - whenever(web3jClient.ethGetExecutionPayloadByNumber(any())) - .thenReturn(SafeFuture.completedFuture(payload)) - - whenever(blockCreationListener.acceptBlock(any())) - .thenReturn(SafeFuture.failedFuture(Exception("Notification 1 Error"))) - .thenThrow(RuntimeException("Notification 2 Error")) - .thenReturn(SafeFuture.failedFuture(Exception("Notification 3 Error"))) - .thenReturn(SafeFuture.completedFuture(Unit)) - - monitor.start().thenApply { - await() - .timeout(5.seconds.toJavaDuration()) - .untilAsserted { - verify(blockCreationListener, atLeast(4)).acceptBlock(BlockCreated(payload)) - assertThat(monitor.nexBlockNumberToFetch).isEqualTo(startingBlockNumberInclusive + 1) - } - testContext.completeNow() - } - .whenException(testContext::failNow) - } - - @Test - fun `is resilient to connection failures`(testContext: VertxTestContext) { - val payload = - executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash) - val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization - whenever(web3jClient.ethBlockNumber()) - .thenReturn(SafeFuture.failedFuture(Exception("ethBlockNumber Error 1"))) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber))) - whenever(web3jClient.ethGetExecutionPayloadByNumber(any())) - .thenReturn(SafeFuture.failedFuture(Exception("ethGetExecutionPayloadByNumber Error 1"))) - .thenReturn(SafeFuture.completedFuture(payload)) - whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit)) - - monitor.start().thenApply { - await() - .timeout(1.seconds.toJavaDuration()) - .untilAsserted { - verify(blockCreationListener, times(1)).acceptBlock(BlockCreated(payload)) - assertThat(monitor.nexBlockNumberToFetch).isEqualTo(startingBlockNumberInclusive + 1) - } - testContext.completeNow() - } - .whenException(testContext::failNow) - } - - @Test - fun `should stop when reorg is detected above blocksToFinalization limit - manual intervention necessary`( - testContext: VertxTestContext - ) { - val payload = - executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash) - val payload2 = - executionPayloadV1( - blockNumber = startingBlockNumberInclusive + 1, - parentHash = ByteArrayExt.random32() - ) - val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization - whenever(web3jClient.ethBlockNumber()) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber))) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 1))) - whenever(web3jClient.ethGetExecutionPayloadByNumber(any())) - .thenReturn(SafeFuture.completedFuture(payload)) - .thenReturn(SafeFuture.completedFuture(payload2)) - whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit)) - - monitor.start().thenApply { - await() - .timeout(1.seconds.toJavaDuration()) - .untilAsserted { - verify(blockCreationListener, times(1)).acceptBlock(BlockCreated(payload)) - verify(blockCreationListener, never()).acceptBlock(BlockCreated(payload2)) - assertThat(monitor.nexBlockNumberToFetch).isEqualTo(startingBlockNumberInclusive + 1) - } - testContext.completeNow() - } - .whenException(testContext::failNow) - } - - private fun delay(delay: Duration, action: () -> SafeFuture): SafeFuture { - val future = SafeFuture() - executor.schedule( - { action().propagateTo(future) }, - delay.inWholeMilliseconds, - TimeUnit.MILLISECONDS - ) - return future - } - - @Test - fun `should poll in order when response takes longer that polling interval`(testContext: VertxTestContext) { - val payload = - executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash) - val payload2 = - executionPayloadV1( - blockNumber = startingBlockNumberInclusive + 1, - parentHash = payload.blockHash.toArray() - ) - val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization - - whenever(web3jClient.ethBlockNumber()) - .then { - delay(config.pollingInterval.times(2)) { - SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber)) - } - } - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 1))) - whenever(web3jClient.ethGetExecutionPayloadByNumber(any())) - .then { delay(config.pollingInterval.times(2)) { SafeFuture.completedFuture(payload) } } - .thenReturn(SafeFuture.completedFuture(payload2)) - whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit)) - - monitor.start().thenApply { - await() - .untilAsserted { - verify(blockCreationListener).acceptBlock(BlockCreated(payload)) - verify(blockCreationListener).acceptBlock(BlockCreated(payload2)) - assertThat(monitor.nexBlockNumberToFetch).isEqualTo(startingBlockNumberInclusive + 2) - } - testContext.completeNow() - } - .whenException(testContext::failNow) - } - - @Test - fun `start allow 2nd call when already started`() { - monitor.start().get() - monitor.start().get() - } - - @Test - fun `stop should be idempotent`(testContext: VertxTestContext) { - val payload = - executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash) - val payload2 = - executionPayloadV1( - blockNumber = startingBlockNumberInclusive + 1, - parentHash = payload.blockHash.toArray() - ) - val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization - whenever(web3jClient.ethBlockNumber()) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber))) - .then { - delay(config.pollingInterval.times(30)) { - SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 1)) - } - } - whenever(web3jClient.ethGetExecutionPayloadByNumber(any())) - .thenReturn(SafeFuture.completedFuture(payload)) - .then { delay(config.pollingInterval.times(30)) { SafeFuture.completedFuture(payload2) } } - whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit)) - - monitor.start().thenApply { - await() - .timeout(1.seconds.toJavaDuration()) - .untilAsserted { - verify(blockCreationListener, times(1)).acceptBlock(any()) - } - } - .whenException(testContext::failNow) - - monitor.stop().thenApply { - await() - .timeout(1.seconds.toJavaDuration()) - .untilAsserted { - verify(blockCreationListener, times(1)).acceptBlock(any()) - } - testContext.completeNow() - } - .whenException(testContext::failNow) - } - - @Test - fun `block shouldn't be fetched when block gap is greater than fetch limit`(testContext: VertxTestContext) { - val payload = executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash) - val payload2 = - executionPayloadV1(blockNumber = startingBlockNumberInclusive + 1, parentHash = payload.blockHash.toArray()) - val payload3 = - executionPayloadV1(blockNumber = startingBlockNumberInclusive + 2, parentHash = payload2.blockHash.toArray()) - val payload4 = - executionPayloadV1(blockNumber = startingBlockNumberInclusive + 3, parentHash = payload3.blockHash.toArray()) - val payload5 = - executionPayloadV1(blockNumber = startingBlockNumberInclusive + 4, parentHash = payload4.blockHash.toArray()) - val payload6 = - executionPayloadV1(blockNumber = startingBlockNumberInclusive + 5, parentHash = payload5.blockHash.toArray()) - val payload7 = - executionPayloadV1(blockNumber = startingBlockNumberInclusive + 6, parentHash = payload6.blockHash.toArray()) - - val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization - whenever(web3jClient.ethGetExecutionPayloadByNumber(any())) - .thenReturn(SafeFuture.completedFuture(payload)) - .then { SafeFuture.completedFuture(payload2) } - .then { SafeFuture.completedFuture(payload3) } - .then { SafeFuture.completedFuture(payload4) } - .then { SafeFuture.completedFuture(payload5) } - .then { SafeFuture.completedFuture(payload6) } - .then { SafeFuture.completedFuture(payload7) } - - whenever(web3jClient.ethBlockNumber()) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber))) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 2))) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 3))) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 4))) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 5))) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 6))) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 7))) - whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit)) - - monitor.start().thenApply { - await() - .timeout(4.seconds.toJavaDuration()) - .untilAsserted { - verify(blockCreationListener, atLeastOnce()).acceptBlock(any()) - // Number of invocations should remain at 5 as the blocks are now above the limit - verify(blockCreationListener, atMost(6)).acceptBlock(any()) - } - testContext.completeNow() - } - .whenException(testContext::failNow) - } - - @Test - fun `last block not fetched until finalization catches up to limit`(vertx: Vertx, testContext: VertxTestContext) { - val payload = executionPayloadV1(blockNumber = startingBlockNumberInclusive, parentHash = parentHash) - val payload2 = executionPayloadV1(blockNumber = startingBlockNumberInclusive + 1, parentHash = payload.blockHash) - val payload3 = executionPayloadV1(blockNumber = startingBlockNumberInclusive + 2, parentHash = payload2.blockHash) - val payload4 = executionPayloadV1(blockNumber = startingBlockNumberInclusive + 3, parentHash = payload3.blockHash) - val payload5 = executionPayloadV1(blockNumber = startingBlockNumberInclusive + 4, parentHash = payload4.blockHash) - val payload6 = executionPayloadV1(blockNumber = startingBlockNumberInclusive + 5, parentHash = payload5.blockHash) - val payload7 = executionPayloadV1(blockNumber = startingBlockNumberInclusive + 6, parentHash = payload6.blockHash) - - val headBlockNumber = startingBlockNumberInclusive + config.blocksToFinalization + config.blocksFetchLimit - whenever(web3jClient.ethGetExecutionPayloadByNumber(any())) - .thenReturn(SafeFuture.completedFuture(payload)) - .thenReturn(SafeFuture.completedFuture(payload2)) - .thenReturn(SafeFuture.completedFuture(payload3)) - .thenReturn(SafeFuture.completedFuture(payload4)) - .thenReturn(SafeFuture.completedFuture(payload5)) - .thenReturn(SafeFuture.completedFuture(payload6)) - .thenReturn(SafeFuture.completedFuture(payload7)) - - whenever(web3jClient.ethBlockNumber()) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber))) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 1))) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 2))) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 3))) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 4))) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 5))) - .thenReturn(SafeFuture.completedFuture(BigInteger.valueOf(headBlockNumber + 6))) - whenever(blockCreationListener.acceptBlock(any())).thenReturn(SafeFuture.completedFuture(Unit)) - - val lastProvenBlockNumberProviderAsync = mock() - - val monitor = - BlockCreationMonitor( - vertx, - web3jClient, - startingBlockNumberExclusive = startingBlockNumberInclusive - 1, - blockCreationListener, - lastProvenBlockNumberProviderAsync, - config - ) - - whenever(lastProvenBlockNumberProviderAsync.getLastProvenBlockNumber()).thenAnswer { - SafeFuture.completedFuture(lastProvenBlock) - } - - monitor.start().thenApply { - await() - .timeout(4.seconds.toJavaDuration()) - .untilAsserted { - verify(blockCreationListener, atLeastOnce()).acceptBlock(any()) - verify(blockCreationListener, times(6)).acceptBlock(any()) - } - }.thenApply { - whenever(lastProvenBlockNumberProviderAsync.getLastProvenBlockNumber()).thenAnswer { - SafeFuture.completedFuture(lastProvenBlock + 1) - } - await() - .timeout(2.seconds.toJavaDuration()) - .untilAsserted { - verify(blockCreationListener, times(7)).acceptBlock(any()) - } - testContext.completeNow() - } - .whenException(testContext::failNow) - } -} diff --git a/coordinator/clients/prover-client/file-based-client/build.gradle b/coordinator/clients/prover-client/file-based-client/build.gradle index 8b865ae80..b44155ff9 100644 --- a/coordinator/clients/prover-client/file-based-client/build.gradle +++ b/coordinator/clients/prover-client/file-based-client/build.gradle @@ -16,16 +16,13 @@ dependencies { implementation "io.vertx:vertx-core" // Block dependencies - implementation "org.hyperledger.besu:besu-datatypes:${libs.versions.besu.get()}" - implementation "org.hyperledger.besu.internal:rlp:${libs.versions.besu.get()}" + implementation project(':jvm-libs:linea:besu-libs') implementation "com.fasterxml.jackson.core:jackson-annotations:${libs.versions.jackson.get()}" implementation "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson.get()}" implementation "com.fasterxml.jackson.module:jackson-module-kotlin:${libs.versions.jackson.get()}" implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${libs.versions.jackson.get()}") - testImplementation project(':jvm-libs:linea:testing:teku-helper') + testImplementation testFixtures(project(':jvm-libs:linea:core:domain-models')) testImplementation "io.vertx:vertx-junit5" - testImplementation "tech.pegasys.teku.internal:spec:${libs.versions.teku.get()}" - testImplementation "tech.pegasys.teku.internal:spec:${libs.versions.teku.get()}:test-fixtures" } diff --git a/coordinator/clients/prover-client/file-based-client/src/main/kotlin/net/consensys/zkevm/coordinator/clients/prover/FileBasedExecutionProverClientV2.kt b/coordinator/clients/prover-client/file-based-client/src/main/kotlin/net/consensys/zkevm/coordinator/clients/prover/FileBasedExecutionProverClientV2.kt index 91c83ac15..c1f34bd22 100644 --- a/coordinator/clients/prover-client/file-based-client/src/main/kotlin/net/consensys/zkevm/coordinator/clients/prover/FileBasedExecutionProverClientV2.kt +++ b/coordinator/clients/prover-client/file-based-client/src/main/kotlin/net/consensys/zkevm/coordinator/clients/prover/FileBasedExecutionProverClientV2.kt @@ -3,6 +3,7 @@ package net.consensys.zkevm.coordinator.clients.prover import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.node.ArrayNode import io.vertx.core.Vertx +import linea.encoding.BlockRLPEncoder import net.consensys.encodeHex import net.consensys.linea.async.toSafeFuture import net.consensys.toBigInteger @@ -13,11 +14,9 @@ import net.consensys.zkevm.coordinator.clients.L2MessageServiceLogsClient import net.consensys.zkevm.coordinator.clients.prover.serialization.JsonSerialization import net.consensys.zkevm.domain.ProofIndex import net.consensys.zkevm.domain.RlpBridgeLogsData -import net.consensys.zkevm.encoding.ExecutionPayloadV1Encoder -import net.consensys.zkevm.encoding.ExecutionPayloadV1RLPEncoderByBesuImplementation +import net.consensys.zkevm.encoding.BlockEncoder import net.consensys.zkevm.fileio.FileReader import net.consensys.zkevm.fileio.FileWriter -import net.consensys.zkevm.toULong import org.apache.logging.log4j.LogManager import org.web3j.protocol.Web3j import org.web3j.protocol.core.DefaultBlockParameter @@ -37,7 +36,7 @@ data class BatchExecutionProofRequestDto( internal class ExecutionProofRequestDataDecorator( private val l2MessageServiceLogsClient: L2MessageServiceLogsClient, private val l2Web3jClient: Web3j, - private val encoder: ExecutionPayloadV1Encoder = ExecutionPayloadV1RLPEncoderByBesuImplementation + private val encoder: BlockEncoder = BlockRLPEncoder ) : (BatchExecutionProofRequestV1) -> SafeFuture { private fun getBlockStateRootHash(blockNumber: ULong): SafeFuture { return l2Web3jClient @@ -52,13 +51,13 @@ internal class ExecutionProofRequestDataDecorator( override fun invoke(request: BatchExecutionProofRequestV1): SafeFuture { val bridgeLogsSfList = request.blocks.map { block -> - l2MessageServiceLogsClient.getBridgeLogs(blockNumber = block.blockNumber.longValue()) + l2MessageServiceLogsClient.getBridgeLogs(blockNumber = block.number.toLong()) .thenApply { block to it } } return SafeFuture.collectAll(bridgeLogsSfList.stream()) .thenCombine( - getBlockStateRootHash(request.blocks.first().blockNumber.toULong() - 1UL) + getBlockStateRootHash(request.blocks.first().number.toULong() - 1UL) ) { blocksAndBridgeLogs, previousKeccakStateRootHash -> BatchExecutionProofRequestDto( zkParentStateRootHash = request.type2StateData.zkParentStateRootHash.encodeHex(), diff --git a/coordinator/clients/prover-client/file-based-client/src/test/kotlin/net/consensys/zkevm/coordinator/clients/prover/ExecutionProofRequestDataDecoratorTest.kt b/coordinator/clients/prover-client/file-based-client/src/test/kotlin/net/consensys/zkevm/coordinator/clients/prover/ExecutionProofRequestDataDecoratorTest.kt index 839a2e127..f5db49f45 100644 --- a/coordinator/clients/prover-client/file-based-client/src/test/kotlin/net/consensys/zkevm/coordinator/clients/prover/ExecutionProofRequestDataDecoratorTest.kt +++ b/coordinator/clients/prover-client/file-based-client/src/test/kotlin/net/consensys/zkevm/coordinator/clients/prover/ExecutionProofRequestDataDecoratorTest.kt @@ -2,13 +2,15 @@ package net.consensys.zkevm.coordinator.clients.prover import build.linea.clients.GetZkEVMStateMerkleProofResponse import com.fasterxml.jackson.databind.node.ArrayNode +import linea.domain.Block +import linea.domain.createBlock import net.consensys.ByteArrayExt import net.consensys.encodeHex import net.consensys.zkevm.coordinator.clients.BatchExecutionProofRequestV1 import net.consensys.zkevm.coordinator.clients.GenerateTracesResponse import net.consensys.zkevm.coordinator.clients.L2MessageServiceLogsClient import net.consensys.zkevm.domain.RlpBridgeLogsData -import net.consensys.zkevm.encoding.ExecutionPayloadV1Encoder +import net.consensys.zkevm.encoding.BlockEncoder import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -21,8 +23,6 @@ import org.mockito.kotlin.spy import org.mockito.kotlin.whenever import org.web3j.protocol.Web3j import org.web3j.protocol.core.methods.response.EthBlock -import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1 -import tech.pegasys.teku.ethereum.executionclient.schema.executionPayloadV1 import tech.pegasys.teku.infrastructure.async.SafeFuture import kotlin.random.Random @@ -30,11 +30,11 @@ class ExecutionProofRequestDataDecoratorTest { private lateinit var l2MessageServiceLogsClient: L2MessageServiceLogsClient private lateinit var l2Web3jClient: Web3j - private lateinit var encoder: ExecutionPayloadV1Encoder + private lateinit var encoder: BlockEncoder private lateinit var requestDatDecorator: ExecutionProofRequestDataDecorator - private val fakeEncoder: ExecutionPayloadV1Encoder = object : ExecutionPayloadV1Encoder { - override fun encode(payload: ExecutionPayloadV1): ByteArray { - return payload.blockNumber.toString().toByteArray() + private val fakeEncoder: BlockEncoder = object : BlockEncoder { + override fun encode(block: Block): ByteArray { + return block.number.toString().toByteArray() } } @@ -48,8 +48,8 @@ class ExecutionProofRequestDataDecoratorTest { @Test fun `should decorate data with bridge logs and parent stateRootHash`() { - val executionPayload1 = executionPayloadV1(blockNumber = 123, gasLimit = 20_000_000UL) - val executionPayload2 = executionPayloadV1(blockNumber = 124, gasLimit = 20_000_000UL) + val block1 = createBlock(number = 123UL) + val block2 = createBlock(number = 124UL) val type2StateResponse = GetZkEVMStateMerkleProofResponse( zkStateMerkleProof = ArrayNode(null), zkParentStateRootHash = ByteArrayExt.random32(), @@ -61,7 +61,7 @@ class ExecutionProofRequestDataDecoratorTest { tracesEngineVersion = "1.0.0" ) val request = BatchExecutionProofRequestV1( - blocks = listOf(executionPayload1, executionPayload2), + blocks = listOf(block1, block2), tracesResponse = generateTracesResponse, type2StateData = type2StateResponse ) @@ -74,9 +74,9 @@ class ExecutionProofRequestDataDecoratorTest { SafeFuture.completedFuture(mockedEthBlock) } - whenever(l2MessageServiceLogsClient.getBridgeLogs(eq(executionPayload1.blockNumber.longValue()))) + whenever(l2MessageServiceLogsClient.getBridgeLogs(eq(block1.number.toLong()))) .thenReturn(SafeFuture.completedFuture(listOf(CommonTestData.bridgeLogs[0]))) - whenever(l2MessageServiceLogsClient.getBridgeLogs(eq(executionPayload2.blockNumber.longValue()))) + whenever(l2MessageServiceLogsClient.getBridgeLogs(eq(block2.number.toLong()))) .thenReturn(SafeFuture.completedFuture(listOf(CommonTestData.bridgeLogs[1]))) val requestDto = requestDatDecorator.invoke(request).get() diff --git a/coordinator/clients/shomei-client/build.gradle b/coordinator/clients/shomei-client/build.gradle index 7117d606d..e16dc42f9 100644 --- a/coordinator/clients/shomei-client/build.gradle +++ b/coordinator/clients/shomei-client/build.gradle @@ -8,9 +8,6 @@ dependencies { implementation project(':jvm-libs:generic:json-rpc') implementation project(':jvm-libs:linea:metrics:micrometer') implementation project(':jvm-libs:linea:core:traces') - implementation project(":jvm-libs:linea:teku-execution-client") - implementation "tech.pegasys.teku.internal:unsigned:${libs.versions.teku.get()}" - api "io.vertx:vertx-core" testImplementation 'org.junit.jupiter:junit-jupiter' diff --git a/coordinator/core/build.gradle b/coordinator/core/build.gradle index 19a0c1b0f..17b494707 100644 --- a/coordinator/core/build.gradle +++ b/coordinator/core/build.gradle @@ -17,7 +17,6 @@ dependencies { api project(':jvm-libs:generic:extensions:futures') api "tech.pegasys.teku.internal:unsigned:${libs.versions.teku.get()}" api "org.jetbrains.kotlinx:kotlinx-datetime:${libs.versions.kotlinxDatetime.get()}" - implementation project(":jvm-libs:linea:teku-execution-client") implementation "io.vertx:vertx-core" // jackson shall never be used in the core module // however, it is used already :( but was as transitive through Teku Execution Client @@ -30,10 +29,10 @@ dependencies { } testFixturesApi "org.jetbrains.kotlinx:kotlinx-datetime:${libs.versions.kotlinxDatetime.get()}" + testFixturesApi testFixtures(project(':jvm-libs:linea:core:domain-models')) testFixturesImplementation("org.web3j:core:${libs.versions.web3j.get()}") { exclude group: 'org.slf4j', module: 'slf4j-nop' } - testImplementation project(":jvm-libs:linea:testing:teku-helper") testImplementation project(':jvm-libs:linea:metrics:micrometer') testImplementation(testFixtures(project(':jvm-libs:linea:core:traces'))) testImplementation(testFixtures(project(':jvm-libs:generic:extensions:kotlin'))) diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/coordinator/clients/BatchExecutionProverRequestResponse.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/coordinator/clients/BatchExecutionProverRequestResponse.kt index 8ddd48320..d3af4a259 100644 --- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/coordinator/clients/BatchExecutionProverRequestResponse.kt +++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/coordinator/clients/BatchExecutionProverRequestResponse.kt @@ -2,18 +2,17 @@ package net.consensys.zkevm.coordinator.clients import build.linea.clients.GetZkEVMStateMerkleProofResponse import build.linea.domain.BlockInterval -import net.consensys.zkevm.toULong -import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1 +import linea.domain.Block data class BatchExecutionProofRequestV1( - val blocks: List, + val blocks: List, val tracesResponse: GenerateTracesResponse, val type2StateData: GetZkEVMStateMerkleProofResponse ) : BlockInterval { override val startBlockNumber: ULong - get() = blocks.first().blockNumber.toULong() + get() = blocks.first().number override val endBlockNumber: ULong - get() = blocks.last().blockNumber.toULong() + get() = blocks.last().number } data class BatchExecutionProofResponse( diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/domain/Conflation.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/domain/Conflation.kt index d10bec59c..9ffe14eab 100644 --- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/domain/Conflation.kt +++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/domain/Conflation.kt @@ -2,24 +2,23 @@ package net.consensys.zkevm.domain import build.linea.domain.BlockInterval import kotlinx.datetime.Instant +import linea.domain.Block import net.consensys.isSortedBy import net.consensys.linea.CommonDomainFunctions import net.consensys.linea.traces.TracesCounters -import net.consensys.zkevm.toULong -import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1 data class BlocksConflation( - val blocks: List, + val blocks: List, val conflationResult: ConflationCalculationResult ) : BlockInterval { init { - require(blocks.isSortedBy { it.blockNumber }) { "Blocks list must be sorted by blockNumber" } + require(blocks.isSortedBy { it.number }) { "Blocks list must be sorted by blockNumber" } } override val startBlockNumber: ULong - get() = blocks.first().blockNumber.toULong() + get() = blocks.first().number.toULong() override val endBlockNumber: ULong - get() = blocks.last().blockNumber.toULong() + get() = blocks.last().number.toULong() } data class Batch( diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/encoding/BlockEncoder.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/encoding/BlockEncoder.kt new file mode 100644 index 000000000..63fc97687 --- /dev/null +++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/encoding/BlockEncoder.kt @@ -0,0 +1,7 @@ +package net.consensys.zkevm.encoding + +import linea.domain.Block + +fun interface BlockEncoder { + fun encode(block: Block): ByteArray +} diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/encoding/ExecutionPayloadV1Encoder.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/encoding/ExecutionPayloadV1Encoder.kt deleted file mode 100644 index 28a269979..000000000 --- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/encoding/ExecutionPayloadV1Encoder.kt +++ /dev/null @@ -1,7 +0,0 @@ -package net.consensys.zkevm.encoding - -import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1 - -fun interface ExecutionPayloadV1Encoder { - fun encode(payload: ExecutionPayloadV1): ByteArray -} diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/blockcreation/BlockCreationCoordinator.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/blockcreation/BlockCreationCoordinator.kt index 140518a4a..6482a815a 100644 --- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/blockcreation/BlockCreationCoordinator.kt +++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/blockcreation/BlockCreationCoordinator.kt @@ -1,10 +1,10 @@ package net.consensys.zkevm.ethereum.coordination.blockcreation -import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1 +import linea.domain.Block import tech.pegasys.teku.infrastructure.async.SafeFuture data class BlockCreated( - val executionPayload: ExecutionPayloadV1 + val block: Block ) fun interface BlockCreationListener { diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/blockcreation/SafeBlockProvider.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/blockcreation/SafeBlockProvider.kt index 88f0ed6ba..85dd4e890 100644 --- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/blockcreation/SafeBlockProvider.kt +++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/blockcreation/SafeBlockProvider.kt @@ -1,49 +1,12 @@ package net.consensys.zkevm.ethereum.coordination.blockcreation -import kotlinx.datetime.Instant -import net.consensys.zkevm.toULong -import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1 +import linea.domain.Block +import linea.domain.BlockHeaderSummary import tech.pegasys.teku.infrastructure.async.SafeFuture -data class BlockHeaderSummary( - val number: ULong, - val hash: ByteArray, - val timestamp: Instant -) { - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (javaClass != other?.javaClass) return false - - other as BlockHeaderSummary - - if (number != other.number) return false - if (!hash.contentEquals(other.hash)) return false - if (timestamp != other.timestamp) return false - - return true - } - - override fun hashCode(): Int { - var result = number.hashCode() - result = 31 * result + hash.contentHashCode() - result = 31 * result + timestamp.hashCode() - return result - } - - override fun toString(): String { - return "BlockHeaderSummary(number=$number, hash=${hash.contentToString()}, timestamp=$timestamp)" - } -} - interface SafeBlockProvider { - fun getLatestSafeBlock(): SafeFuture + fun getLatestSafeBlock(): SafeFuture fun getLatestSafeBlockHeader(): SafeFuture { - return getLatestSafeBlock().thenApply { - BlockHeaderSummary( - it.blockNumber.toULong(), - it.blockHash.toArray(), - Instant.fromEpochSeconds(it.timestamp.longValue()) - ) - } + return getLatestSafeBlock().thenApply { it.headerSummary } } } diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/BlockToBatchSubmissionCoordinator.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/BlockToBatchSubmissionCoordinator.kt index 001fe2ef5..ca3d66315 100644 --- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/BlockToBatchSubmissionCoordinator.kt +++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/BlockToBatchSubmissionCoordinator.kt @@ -4,17 +4,16 @@ import com.github.michaelbull.result.Err import com.github.michaelbull.result.Ok import io.vertx.core.Vertx import kotlinx.datetime.Instant -import net.consensys.linea.BlockNumberAndHash +import linea.domain.Block import net.consensys.linea.async.toSafeFuture import net.consensys.linea.errors.ErrorResponse import net.consensys.zkevm.coordinator.clients.GetTracesCountersResponse import net.consensys.zkevm.coordinator.clients.TracesCountersClientV1 import net.consensys.zkevm.coordinator.clients.TracesServiceErrorType import net.consensys.zkevm.domain.BlockCounters -import net.consensys.zkevm.encoding.ExecutionPayloadV1Encoder +import net.consensys.zkevm.encoding.BlockEncoder import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreated import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreationListener -import net.consensys.zkevm.toULong import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger import tech.pegasys.teku.infrastructure.async.SafeFuture @@ -24,19 +23,14 @@ class BlockToBatchSubmissionCoordinator( private val conflationService: ConflationService, private val tracesCountersClient: TracesCountersClientV1, private val vertx: Vertx, - private val payloadEncoder: ExecutionPayloadV1Encoder, + private val encoder: BlockEncoder, private val log: Logger = LogManager.getLogger(BlockToBatchSubmissionCoordinator::class.java) ) : BlockCreationListener { private fun getTracesCounters( - blockEvent: BlockCreated + block: Block ): SafeFuture { return tracesCountersClient - .rollupGetTracesCounters( - BlockNumberAndHash( - blockEvent.executionPayload.blockNumber.toULong(), - blockEvent.executionPayload.blockHash.toArray() - ) - ) + .rollupGetTracesCounters(block.numberAndHash) .thenCompose { result -> when (result) { is Err> -> { @@ -51,31 +45,37 @@ class BlockToBatchSubmissionCoordinator( } override fun acceptBlock(blockEvent: BlockCreated): SafeFuture { - log.debug("Accepting new block={}", blockEvent.executionPayload.blockNumber) - vertx.executeBlocking( - Callable { - payloadEncoder.encode(blockEvent.executionPayload) - } - ).toSafeFuture().thenCombine(getTracesCounters(blockEvent)) { blockRLPEncoded, traces -> - conflationService.newBlock( - blockEvent.executionPayload, - BlockCounters( - blockNumber = blockEvent.executionPayload.blockNumber.toULong(), - blockTimestamp = Instant.fromEpochSeconds(blockEvent.executionPayload.timestamp.longValue()), - tracesCounters = traces.tracesCounters, - blockRLPEncoded = blockRLPEncoded + log.debug("accepting new block={}", blockEvent.block.number) + encodeBlock(blockEvent.block) + .thenCombine(getTracesCounters(blockEvent.block)) { blockRLPEncoded, traces -> + conflationService.newBlock( + blockEvent.block, + BlockCounters( + blockNumber = blockEvent.block.number, + blockTimestamp = Instant.fromEpochSeconds(blockEvent.block.timestamp.toLong()), + tracesCounters = traces.tracesCounters, + blockRLPEncoded = blockRLPEncoded + ) ) - ) - }.whenException { th -> - log.error( - "Failed to conflate block={} errorMessage={}", - blockEvent.executionPayload.blockNumber, - th.message, - th - ) - } + }.whenException { th -> + log.error( + "Failed to conflate block={} errorMessage={}", + blockEvent.block.number, + th.message, + th + ) + } // This is to parallelize `getTracesCounters` requests which would otherwise be sent sequentially return SafeFuture.completedFuture(Unit) } + + private fun encodeBlock(block: Block): SafeFuture { + return vertx.executeBlocking( + Callable { + encoder.encode(block) + } + ) + .toSafeFuture() + } } diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationServiceImpl.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationServiceImpl.kt index 437616843..80fa89628 100644 --- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationServiceImpl.kt +++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationServiceImpl.kt @@ -1,14 +1,13 @@ package net.consensys.zkevm.ethereum.coordination.conflation +import linea.domain.Block import net.consensys.linea.metrics.LineaMetricsCategory import net.consensys.linea.metrics.MetricsFacade import net.consensys.zkevm.domain.BlockCounters import net.consensys.zkevm.domain.BlocksConflation import net.consensys.zkevm.domain.ConflationCalculationResult -import net.consensys.zkevm.toULong import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger -import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1 import tech.pegasys.teku.infrastructure.async.SafeFuture import java.util.concurrent.PriorityBlockingQueue import java.util.concurrent.TimeUnit @@ -20,14 +19,14 @@ class ConflationServiceImpl( ConflationService { private val log: Logger = LogManager.getLogger(this::class.java) private var listener: ConflationHandler = ConflationHandler { SafeFuture.completedFuture(null) } - private val blocksInProgress: MutableList = mutableListOf() + private val blocksInProgress: MutableList = mutableListOf() data class PayloadAndBlockCounters( - val executionPayload: ExecutionPayloadV1, + val block: Block, val blockCounters: BlockCounters ) : Comparable { override fun compareTo(other: PayloadAndBlockCounters): Int { - return this.executionPayload.blockNumber.compareTo(other.executionPayload.blockNumber) + return this.block.number.compareTo(other.block.number) } } @@ -66,8 +65,8 @@ class ConflationServiceImpl( ) val blocksToConflate = blocksInProgress - .filter { it.blockNumber.toULong() in conflation.blocksRange } - .sortedBy { it.blockNumber } + .filter { it.number in conflation.blocksRange } + .sortedBy { it.number } blocksInProgress.removeAll(blocksToConflate) return listener.handleConflatedBatch(BlocksConflation(blocksToConflate, conflation)) @@ -82,21 +81,21 @@ class ConflationServiceImpl( } @Synchronized - override fun newBlock(block: ExecutionPayloadV1, blockCounters: BlockCounters) { - require(block.blockNumber.toULong() == blockCounters.blockNumber) { - "Payload blockNumber ${block.blockNumber} does not match blockCounters.blockNumber=${blockCounters.blockNumber}" + override fun newBlock(block: Block, blockCounters: BlockCounters) { + require(block.number == blockCounters.blockNumber) { + "block=${block.number} does not match blockCounters.blockNumber=${blockCounters.blockNumber}" } blocksCounter.increment() log.trace( "newBlock={} calculatorLastBlockNumber={} blocksToConflateSize={} blocksInProgressSize={}", - block.blockNumber, + block.number, calculator.lastBlockNumber, blocksToConflate.size, blocksInProgress.size ) blocksToConflate.add(PayloadAndBlockCounters(block, blockCounters)) blocksInProgress.add(block) - log.trace("block {} added to conflation queue", block.blockNumber) + log.trace("block {} added to conflation queue", block.number) sendBlocksInOrderToTracesCounter() } @@ -104,14 +103,14 @@ class ConflationServiceImpl( var nextBlockNumberToConflate = calculator.lastBlockNumber + 1u var nextAvailableBlock = blocksToConflate.peek() - while (nextAvailableBlock?.executionPayload?.blockNumber?.toULong() == nextBlockNumberToConflate) { + while (nextAvailableBlock?.block?.number == nextBlockNumberToConflate) { nextAvailableBlock = blocksToConflate.poll(100, TimeUnit.MILLISECONDS) log.trace( "block {} removed from conflation queue and sent to calculator", - nextAvailableBlock?.executionPayload?.blockNumber + nextAvailableBlock?.block?.number ) calculator.newBlock(nextAvailableBlock.blockCounters) - nextBlockNumberToConflate = nextAvailableBlock.executionPayload.blockNumber.toULong() + 1u + nextBlockNumberToConflate = nextAvailableBlock.block.number + 1u nextAvailableBlock = blocksToConflate.peek() } } diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ProofGeneratingConflationHandlerImpl.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ProofGeneratingConflationHandlerImpl.kt index 01539ca61..0a258556d 100644 --- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ProofGeneratingConflationHandlerImpl.kt +++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ProofGeneratingConflationHandlerImpl.kt @@ -3,12 +3,10 @@ package net.consensys.zkevm.ethereum.coordination.conflation import com.github.michaelbull.result.getOrElse import com.github.michaelbull.result.runCatching import io.vertx.core.Vertx -import net.consensys.linea.BlockNumberAndHash import net.consensys.linea.async.AsyncRetryer import net.consensys.zkevm.domain.BlocksConflation import net.consensys.zkevm.ethereum.coordination.proofcreation.BatchProofHandler import net.consensys.zkevm.ethereum.coordination.proofcreation.ZkProofCreationCoordinator -import net.consensys.zkevm.toULong import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger import tech.pegasys.teku.infrastructure.async.SafeFuture @@ -57,9 +55,7 @@ class ProofGeneratingConflationHandlerImpl( } private fun conflationToProofCreation(conflation: BlocksConflation): SafeFuture<*> { - val blockNumbersAndHash = conflation.blocks.map { - BlockNumberAndHash(it.blockNumber.toULong(), it.blockHash.toArray()) - } + val blockNumbersAndHash = conflation.blocks.map { it.numberAndHash } val blockIntervalString = conflation.conflationResult.intervalString() return tracesProductionCoordinator .conflateExecutionTraces(blockNumbersAndHash) diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/TracesConflationCalculator.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/TracesConflationCalculator.kt index 796385581..ee261436e 100644 --- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/TracesConflationCalculator.kt +++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/TracesConflationCalculator.kt @@ -1,10 +1,10 @@ package net.consensys.zkevm.ethereum.coordination.conflation +import linea.domain.Block import net.consensys.zkevm.domain.Blob import net.consensys.zkevm.domain.BlockCounters import net.consensys.zkevm.domain.BlocksConflation import net.consensys.zkevm.domain.ConflationCalculationResult -import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1 import tech.pegasys.teku.infrastructure.async.SafeFuture fun interface BlobCreationHandler { @@ -23,6 +23,6 @@ interface TracesConflationCalculator { } interface ConflationService { - fun newBlock(block: ExecutionPayloadV1, blockCounters: BlockCounters) + fun newBlock(block: Block, blockCounters: BlockCounters) fun onConflatedBatch(consumer: ConflationHandler) } diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/upgrade/SwitchAwareConflationHandler.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/upgrade/SwitchAwareConflationHandler.kt index 55e7dc155..7cb3d983d 100644 --- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/upgrade/SwitchAwareConflationHandler.kt +++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/upgrade/SwitchAwareConflationHandler.kt @@ -2,7 +2,6 @@ package net.consensys.zkevm.ethereum.coordination.conflation.upgrade import net.consensys.zkevm.domain.BlocksConflation import net.consensys.zkevm.ethereum.coordination.conflation.ConflationHandler -import net.consensys.zkevm.toULong import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger import tech.pegasys.teku.infrastructure.async.SafeFuture @@ -20,8 +19,8 @@ class SwitchAwareConflationHandler( override fun handleConflatedBatch(conflation: BlocksConflation): SafeFuture<*> { return switchProvider.getSwitch(newVersion).thenCompose { switchBlock -> - val conflationStartBlockNumber = conflation.blocks.first().blockNumber.toULong() - val conflationEndBlockNumber = conflation.blocks.last().blockNumber.toULong() + val conflationStartBlockNumber = conflation.blocks.first().number.toULong() + val conflationEndBlockNumber = conflation.blocks.last().number.toULong() if (switchBlock == null || conflationStartBlockNumber < switchBlock) { log.debug("Handing conflation [$conflationStartBlockNumber, $conflationEndBlockNumber] over to old handler") oldHandler.handleConflatedBatch(conflation) diff --git a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/proofcreation/ZkProofCreationCoordinatorImpl.kt b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/proofcreation/ZkProofCreationCoordinatorImpl.kt index d6011dbe6..418120cab 100644 --- a/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/proofcreation/ZkProofCreationCoordinatorImpl.kt +++ b/coordinator/core/src/main/kotlin/net/consensys/zkevm/ethereum/coordination/proofcreation/ZkProofCreationCoordinatorImpl.kt @@ -5,7 +5,6 @@ import net.consensys.zkevm.coordinator.clients.ExecutionProverClientV2 import net.consensys.zkevm.domain.Batch import net.consensys.zkevm.domain.BlocksConflation import net.consensys.zkevm.ethereum.coordination.conflation.BlocksTracesConflated -import net.consensys.zkevm.toULong import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger import tech.pegasys.teku.infrastructure.async.SafeFuture @@ -19,8 +18,8 @@ class ZkProofCreationCoordinatorImpl( blocksConflation: BlocksConflation, traces: BlocksTracesConflated ): SafeFuture { - val startBlockNumber = blocksConflation.blocks.first().blockNumber.toULong() - val endBlockNumber = blocksConflation.blocks.last().blockNumber.toULong() + val startBlockNumber = blocksConflation.blocks.first().number.toULong() + val endBlockNumber = blocksConflation.blocks.last().number.toULong() val blocksConflationInterval = blocksConflation.intervalString() return executionProverClient diff --git a/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/AggregationTriggerCalculatorByDeadlineTest.kt b/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/AggregationTriggerCalculatorByDeadlineTest.kt index 7bc2b05a0..9aaf2eecd 100644 --- a/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/AggregationTriggerCalculatorByDeadlineTest.kt +++ b/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/AggregationTriggerCalculatorByDeadlineTest.kt @@ -2,10 +2,10 @@ package net.consensys.zkevm.ethereum.coordination.aggregation import kotlinx.datetime.Clock import kotlinx.datetime.Instant +import linea.domain.BlockHeaderSummary import net.consensys.ByteArrayExt import net.consensys.zkevm.domain.BlobCounters import net.consensys.zkevm.domain.BlobsToAggregate -import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockHeaderSummary import net.consensys.zkevm.ethereum.coordination.blockcreation.SafeBlockProvider import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test diff --git a/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/GlobalAggregationCalculatorTest.kt b/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/GlobalAggregationCalculatorTest.kt index b75be3890..101d2165f 100644 --- a/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/GlobalAggregationCalculatorTest.kt +++ b/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/aggregation/GlobalAggregationCalculatorTest.kt @@ -2,13 +2,13 @@ package net.consensys.zkevm.ethereum.coordination.aggregation import io.micrometer.core.instrument.simple.SimpleMeterRegistry import kotlinx.datetime.Instant +import linea.domain.BlockHeaderSummary import net.consensys.ByteArrayExt import net.consensys.FakeFixedClock import net.consensys.linea.metrics.MetricsFacade import net.consensys.linea.metrics.micrometer.MicrometerMetricsFacade import net.consensys.zkevm.domain.BlobCounters import net.consensys.zkevm.domain.BlobsToAggregate -import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockHeaderSummary import net.consensys.zkevm.ethereum.coordination.blockcreation.SafeBlockProvider import org.assertj.core.api.Assertions import org.assertj.core.api.Assertions.assertThat diff --git a/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/BlockToBatchSubmissionCoordinatorTest.kt b/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/BlockToBatchSubmissionCoordinatorTest.kt index 17451a100..4d158a496 100644 --- a/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/BlockToBatchSubmissionCoordinatorTest.kt +++ b/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/BlockToBatchSubmissionCoordinatorTest.kt @@ -3,12 +3,11 @@ package net.consensys.zkevm.ethereum.coordination.conflation import com.github.michaelbull.result.Ok import io.vertx.core.Vertx import io.vertx.junit5.VertxExtension -import net.consensys.linea.BlockNumberAndHash +import linea.domain.createBlock import net.consensys.linea.traces.TracesCountersV1 import net.consensys.zkevm.coordinator.clients.GetTracesCountersResponse import net.consensys.zkevm.coordinator.clients.TracesCountersClientV1 import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreated -import net.consensys.zkevm.toULong import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger import org.assertj.core.api.Assertions @@ -22,7 +21,6 @@ import org.mockito.kotlin.eq import org.mockito.kotlin.times import org.mockito.kotlin.verify import org.mockito.kotlin.whenever -import tech.pegasys.teku.ethereum.executionclient.schema.randomExecutionPayload import tech.pegasys.teku.infrastructure.async.SafeFuture import kotlin.time.Duration.Companion.seconds import kotlin.time.toJavaDuration @@ -31,9 +29,8 @@ import kotlin.time.toJavaDuration class BlockToBatchSubmissionCoordinatorTest { companion object { private val defaultConflationService = ConflationServiceImpl(mock(), mock()) - private const val ARBITRARY_BLOCK_NUMBER = 100L - private val randomExecutionPayload = randomExecutionPayload(blockNumber = ARBITRARY_BLOCK_NUMBER) - private val baseBlock = BlockCreated(randomExecutionPayload) + private val randomBlock = createBlock(number = 100UL) + private val baseBlock = BlockCreated(randomBlock) private val blockRlpEncoded = ByteArray(0) private val tracesCounters = TracesCountersV1.EMPTY_TRACES_COUNT } @@ -45,22 +42,14 @@ class BlockToBatchSubmissionCoordinatorTest { ): BlockToBatchSubmissionCoordinator { val tracesCountersClient = mock().also { - whenever( - it.rollupGetTracesCounters( - BlockNumberAndHash( - randomExecutionPayload.blockNumber.toULong(), - randomExecutionPayload.blockHash.toArray() - ) - ) - ).thenReturn( - SafeFuture.completedFuture(Ok(GetTracesCountersResponse(tracesCounters, ""))) - ) + whenever(it.rollupGetTracesCounters(randomBlock.numberAndHash)) + .thenReturn(SafeFuture.completedFuture(Ok(GetTracesCountersResponse(tracesCounters, "")))) } return BlockToBatchSubmissionCoordinator( conflationService = conflationService, tracesCountersClient = tracesCountersClient, vertx = vertx, - payloadEncoder = { blockRlpEncoded }, + encoder = { blockRlpEncoded }, log = log ) } diff --git a/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationCalculatorByTimeDeadlineTest.kt b/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationCalculatorByTimeDeadlineTest.kt index abd7c6476..8b3b10208 100644 --- a/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationCalculatorByTimeDeadlineTest.kt +++ b/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationCalculatorByTimeDeadlineTest.kt @@ -2,10 +2,10 @@ package net.consensys.zkevm.ethereum.coordination.conflation import kotlinx.datetime.Clock import kotlinx.datetime.Instant +import linea.domain.BlockHeaderSummary import net.consensys.ByteArrayExt import net.consensys.linea.traces.fakeTracesCountersV1 import net.consensys.zkevm.domain.BlockCounters -import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockHeaderSummary import net.consensys.zkevm.ethereum.coordination.blockcreation.SafeBlockProvider import org.apache.logging.log4j.Logger import org.assertj.core.api.Assertions.assertThat diff --git a/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationServiceImplTest.kt b/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationServiceImplTest.kt index 13e6428d7..5993d578e 100644 --- a/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationServiceImplTest.kt +++ b/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/ConflationServiceImplTest.kt @@ -1,13 +1,13 @@ package net.consensys.zkevm.ethereum.coordination.conflation import kotlinx.datetime.Instant +import linea.domain.createBlock import net.consensys.linea.traces.TracesCountersV1 import net.consensys.linea.traces.fakeTracesCountersV1 import net.consensys.zkevm.domain.BlockCounters import net.consensys.zkevm.domain.BlocksConflation import net.consensys.zkevm.domain.ConflationCalculationResult import net.consensys.zkevm.domain.ConflationTrigger -import net.consensys.zkevm.toULong import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatThrownBy import org.awaitility.Awaitility @@ -17,7 +17,6 @@ import org.mockito.Mockito.RETURNS_DEEP_STUBS import org.mockito.kotlin.any import org.mockito.kotlin.mock import org.mockito.kotlin.whenever -import tech.pegasys.teku.ethereum.executionclient.schema.executionPayloadV1 import tech.pegasys.teku.infrastructure.async.SafeFuture import java.time.Duration import java.util.concurrent.Executors @@ -43,9 +42,9 @@ class ConflationServiceImplTest { @Test fun `emits event with blocks when calculator emits conflation`() { - val payload1 = executionPayloadV1(blockNumber = 1, gasLimit = 20_000_000UL) - val payload2 = executionPayloadV1(blockNumber = 2, gasLimit = 20_000_000UL) - val payload3 = executionPayloadV1(blockNumber = 3, gasLimit = 20_000_000UL) + val payload1 = createBlock(number = 1UL, gasLimit = 20_000_000UL) + val payload2 = createBlock(number = 2UL, gasLimit = 20_000_000UL) + val payload3 = createBlock(number = 3UL, gasLimit = 20_000_000UL) val payload1Time = Instant.parse("2021-01-01T00:00:00Z") val payloadCounters1 = BlockCounters( blockNumber = 1UL, @@ -100,7 +99,7 @@ class ConflationServiceImplTest { val moduleTracesCounter = 10u assertThat(numberOfBlocks % numberOfThreads).isEqualTo(0) val expectedConflations = numberOfBlocks / conflationBlockLimit.toInt() - 1 - val blocks = (1..numberOfBlocks).map { executionPayloadV1(blockNumber = it.toLong(), gasLimit = 20_000_000UL) } + val blocks = (1UL..numberOfBlocks.toULong()).map { createBlock(number = it, gasLimit = 20_000_000UL) } val fixedTracesCounters = fakeTracesCountersV1(moduleTracesCounter) val blockTime = Instant.parse("2021-01-01T00:00:00Z") val conflationEvents = mutableListOf() @@ -118,7 +117,7 @@ class ConflationServiceImplTest { conflationService.newBlock( it, BlockCounters( - blockNumber = it.blockNumber.toULong(), + blockNumber = it.number.toULong(), blockTimestamp = blockTime, tracesCounters = fixedTracesCounters, blockRLPEncoded = ByteArray(0) @@ -152,13 +151,13 @@ class ConflationServiceImplTest { val failingConflationCalculator: TracesConflationCalculator = mock() whenever(failingConflationCalculator.newBlock(any())).thenThrow(expectedException) conflationService = ConflationServiceImpl(failingConflationCalculator, mock(defaultAnswer = RETURNS_DEEP_STUBS)) - val block = executionPayloadV1(blockNumber = 1, gasLimit = 20_000_000UL) + val block = createBlock(number = 1UL, gasLimit = 20_000_000UL) assertThatThrownBy { conflationService.newBlock( block, BlockCounters( - blockNumber = block.blockNumber.toULong(), + blockNumber = block.number.toULong(), blockTimestamp = blockTime, tracesCounters = fixedTracesCounters, blockRLPEncoded = ByteArray(0) diff --git a/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/GlobalBlobAwareConflationCalculatorTest.kt b/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/GlobalBlobAwareConflationCalculatorTest.kt index 8d9fd5adc..6bf564d2f 100644 --- a/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/GlobalBlobAwareConflationCalculatorTest.kt +++ b/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/GlobalBlobAwareConflationCalculatorTest.kt @@ -1,6 +1,7 @@ package net.consensys.zkevm.ethereum.coordination.conflation import kotlinx.datetime.Instant +import linea.domain.BlockHeaderSummary import net.consensys.ByteArrayExt import net.consensys.FakeFixedClock import net.consensys.linea.traces.TracesCountersV1 @@ -11,7 +12,6 @@ import net.consensys.zkevm.domain.ConflationCalculationResult import net.consensys.zkevm.domain.ConflationTrigger import net.consensys.zkevm.ethereum.coordination.blob.BlobCompressor import net.consensys.zkevm.ethereum.coordination.blob.FakeBlobCompressor -import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockHeaderSummary import net.consensys.zkevm.ethereum.coordination.blockcreation.SafeBlockProvider import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeEach diff --git a/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/GlobalBlockConflationCalculatorIntTest.kt b/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/GlobalBlockConflationCalculatorIntTest.kt index 63cd0c0ce..8545dcc99 100644 --- a/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/GlobalBlockConflationCalculatorIntTest.kt +++ b/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/GlobalBlockConflationCalculatorIntTest.kt @@ -1,6 +1,7 @@ package net.consensys.zkevm.ethereum.coordination.conflation import kotlinx.datetime.Instant +import linea.domain.BlockHeaderSummary import net.consensys.ByteArrayExt import net.consensys.FakeFixedClock import net.consensys.linea.metrics.MetricsFacade @@ -10,7 +11,6 @@ import net.consensys.zkevm.domain.BlockCounters import net.consensys.zkevm.domain.ConflationCalculationResult import net.consensys.zkevm.domain.ConflationTrigger import net.consensys.zkevm.ethereum.coordination.blob.FakeBlobCompressor -import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockHeaderSummary import net.consensys.zkevm.ethereum.coordination.blockcreation.SafeBlockProvider import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeEach diff --git a/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/upgrade/SwitchAwareConflationHandlerTest.kt b/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/upgrade/SwitchAwareConflationHandlerTest.kt index fe15ed83a..4cfc74042 100644 --- a/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/upgrade/SwitchAwareConflationHandlerTest.kt +++ b/coordinator/core/src/test/kotlin/net/consensys/zkevm/ethereum/coordination/conflation/upgrade/SwitchAwareConflationHandlerTest.kt @@ -1,11 +1,11 @@ package net.consensys.zkevm.ethereum.coordination.conflation.upgrade +import linea.domain.createBlock import net.consensys.linea.traces.TracesCountersV1 import net.consensys.zkevm.domain.BlocksConflation import net.consensys.zkevm.domain.ConflationCalculationResult import net.consensys.zkevm.domain.ConflationTrigger import net.consensys.zkevm.ethereum.coordination.conflation.ConflationHandler -import net.consensys.zkevm.toULong import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.mockito.kotlin.any @@ -15,7 +15,6 @@ import org.mockito.kotlin.never import org.mockito.kotlin.times import org.mockito.kotlin.verify import org.mockito.kotlin.whenever -import tech.pegasys.teku.ethereum.executionclient.schema.executionPayloadV1 import tech.pegasys.teku.infrastructure.async.SafeFuture @Suppress("DEPRECATION") @@ -28,13 +27,12 @@ class SwitchAwareConflationHandlerTest { private val switchBlock = 100UL private fun generateArbitraryConflation(startBlockNumber: ULong, blocksLong: UInt): BlocksConflation { - val executionPayloads = (startBlockNumber..startBlockNumber + blocksLong).map { - executionPayloadV1(blockNumber = it.toLong(), gasLimit = 20_000_000UL) - } + val executionPayloads = (startBlockNumber..startBlockNumber + blocksLong) + .map { createBlock(number = it) } val conflationCalculationResult = ConflationCalculationResult( - startBlockNumber = executionPayloads.first().blockNumber.toULong(), - endBlockNumber = executionPayloads.last().blockNumber.toULong(), + startBlockNumber = executionPayloads.first().number.toULong(), + endBlockNumber = executionPayloads.last().number.toULong(), conflationTrigger = ConflationTrigger.TRACES_LIMIT, tracesCounters = TracesCountersV1.EMPTY_TRACES_COUNT ) diff --git a/coordinator/ethereum/blob-submitter/build.gradle b/coordinator/ethereum/blob-submitter/build.gradle index d616229b9..dc5430d65 100644 --- a/coordinator/ethereum/blob-submitter/build.gradle +++ b/coordinator/ethereum/blob-submitter/build.gradle @@ -20,7 +20,6 @@ dependencies { implementation("org.web3j:core:${libs.versions.web3j.get()}") { exclude group: "org.slf4j", module: "slf4j-nop" } - implementation project(":jvm-libs:linea:teku-execution-client") testImplementation(project(":jvm-libs:linea:testing:l1-blob-and-proof-submission")) testImplementation(project(":coordinator:persistence:aggregation")) diff --git a/coordinator/ethereum/gas-pricing/dynamic-cap/src/test/kotlin/net/consensys/linea/ethereum/gaspricing/dynamiccap/GasPriceCapProviderImplTest.kt b/coordinator/ethereum/gas-pricing/dynamic-cap/src/test/kotlin/net/consensys/linea/ethereum/gaspricing/dynamiccap/GasPriceCapProviderImplTest.kt index 1c15faef4..375f6f987 100644 --- a/coordinator/ethereum/gas-pricing/dynamic-cap/src/test/kotlin/net/consensys/linea/ethereum/gaspricing/dynamiccap/GasPriceCapProviderImplTest.kt +++ b/coordinator/ethereum/gas-pricing/dynamic-cap/src/test/kotlin/net/consensys/linea/ethereum/gaspricing/dynamiccap/GasPriceCapProviderImplTest.kt @@ -37,8 +37,6 @@ class GasPriceCapProviderImplTest { private val adjustmentConstant = 25U private val finalizationTargetMaxDelay = 6.hours private val gasPriceCapsCoefficient = 1.0.div(1.1) - private val historicBaseFeePerBlobGasLowerBound = 200000000uL // 0.2GWei - private val initialFixedAvgReward = 100000000uL // 0.1GWei private val gasPriceCapCalculator = GasPriceCapCalculatorImpl() private lateinit var targetBlockTime: Instant diff --git a/coordinator/ethereum/models-helper/build.gradle b/coordinator/ethereum/models-helper/build.gradle index 10cd8f109..e697a462d 100644 --- a/coordinator/ethereum/models-helper/build.gradle +++ b/coordinator/ethereum/models-helper/build.gradle @@ -4,13 +4,6 @@ plugins { dependencies { api (project(":coordinator:core")) - api project(":jvm-libs:linea:teku-execution-client") - implementation "tech.pegasys.teku.internal:bytes:${libs.versions.teku.get()}" - implementation "org.hyperledger.besu:besu-datatypes:${libs.versions.besu.get()}" - implementation "org.hyperledger.besu:evm:${libs.versions.besu.get()}" - implementation "org.hyperledger.besu.internal:rlp:${libs.versions.besu.get()}" - implementation "org.hyperledger.besu.internal:core:${libs.versions.besu.get()}" - implementation "org.hyperledger.besu:plugin-api:${libs.versions.besu.get()}" - - testImplementation project(":jvm-libs:linea:testing:teku-helper") + api project(":jvm-libs:linea:besu-libs") + api project(":jvm-libs:linea:besu-rlp-and-mappers") } diff --git a/coordinator/ethereum/models-helper/src/main/kotlin/linea/encoding/BlockRLPEncoder.kt b/coordinator/ethereum/models-helper/src/main/kotlin/linea/encoding/BlockRLPEncoder.kt new file mode 100644 index 000000000..aa0290c3b --- /dev/null +++ b/coordinator/ethereum/models-helper/src/main/kotlin/linea/encoding/BlockRLPEncoder.kt @@ -0,0 +1,8 @@ +package linea.encoding + +import linea.rlp.RLP +import net.consensys.zkevm.encoding.BlockEncoder + +object BlockRLPEncoder : BlockEncoder { + override fun encode(block: linea.domain.Block): ByteArray = RLP.encode(block) +} diff --git a/coordinator/ethereum/models-helper/src/main/kotlin/net/consensys/zkevm/encoding/ExecutionPayloadV1RLPEncoderByBesuImplementation.kt b/coordinator/ethereum/models-helper/src/main/kotlin/net/consensys/zkevm/encoding/ExecutionPayloadV1RLPEncoderByBesuImplementation.kt deleted file mode 100644 index e855f72c5..000000000 --- a/coordinator/ethereum/models-helper/src/main/kotlin/net/consensys/zkevm/encoding/ExecutionPayloadV1RLPEncoderByBesuImplementation.kt +++ /dev/null @@ -1,44 +0,0 @@ -package net.consensys.zkevm.encoding - -import org.hyperledger.besu.datatypes.Address -import org.hyperledger.besu.datatypes.Hash -import org.hyperledger.besu.datatypes.Wei -import org.hyperledger.besu.ethereum.core.Block -import org.hyperledger.besu.ethereum.core.BlockBody -import org.hyperledger.besu.ethereum.core.BlockHeaderBuilder -import org.hyperledger.besu.ethereum.core.Difficulty -import org.hyperledger.besu.ethereum.core.encoding.EncodingContext -import org.hyperledger.besu.ethereum.core.encoding.TransactionDecoder -import org.hyperledger.besu.ethereum.mainnet.BodyValidation -import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions -import org.hyperledger.besu.evm.log.LogsBloomFilter -import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1 - -object ExecutionPayloadV1RLPEncoderByBesuImplementation : ExecutionPayloadV1Encoder { - override fun encode(payload: ExecutionPayloadV1): ByteArray { - val parsedTransactions = payload.transactions - .map { TransactionDecoder.decodeOpaqueBytes(it, EncodingContext.BLOCK_BODY) } - val parsedBody = BlockBody(parsedTransactions, emptyList()) - val blockHeader = - BlockHeaderBuilder.create() - .parentHash(Hash.wrap(payload.parentHash)) - .ommersHash(Hash.EMPTY_LIST_HASH) - .coinbase(Address.wrap(payload.feeRecipient.wrappedBytes)) - .stateRoot(Hash.wrap(payload.stateRoot)) - .transactionsRoot(BodyValidation.transactionsRoot(parsedBody.transactions)) - .receiptsRoot(Hash.wrap(payload.receiptsRoot)) - .logsBloom(LogsBloomFilter(payload.logsBloom)) - .difficulty(Difficulty.ZERO) - .number(payload.blockNumber.longValue()) - .gasLimit(payload.gasLimit.longValue()) - .gasUsed(payload.gasLimit.longValue()) - .timestamp(payload.timestamp.longValue()) - .extraData(payload.extraData) - .baseFee(Wei.wrap(payload.baseFeePerGas.toBytes())) - .mixHash(Hash.wrap(payload.prevRandao)) - .nonce(0) // this works because Linea is not using PoW - .blockHeaderFunctions(MainnetBlockHeaderFunctions()) - .buildBlockHeader() - return Block(blockHeader, parsedBody).toRlp().toArray() - } -} diff --git a/coordinator/ethereum/models-helper/src/test/kotlin/net/consensys/zkevm/encoding/ExecutionPayloadV1RLPEncoderByBesuImplementationTest.kt b/coordinator/ethereum/models-helper/src/test/kotlin/net/consensys/zkevm/encoding/ExecutionPayloadV1RLPEncoderByBesuImplementationTest.kt deleted file mode 100644 index 979269917..000000000 --- a/coordinator/ethereum/models-helper/src/test/kotlin/net/consensys/zkevm/encoding/ExecutionPayloadV1RLPEncoderByBesuImplementationTest.kt +++ /dev/null @@ -1,33 +0,0 @@ -package net.consensys.zkevm.encoding - -import net.consensys.zkevm.toULong -import org.apache.tuweni.bytes.Bytes -import org.assertj.core.api.Assertions.assertThat -import org.hyperledger.besu.ethereum.core.Block -import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions -import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput -import org.junit.jupiter.api.Test -import tech.pegasys.teku.ethereum.executionclient.schema.randomExecutionPayload - -class ExecutionPayloadV1RLPEncoderByBesuImplementationTest { - - @Test - fun encode() { - val payload = randomExecutionPayload() - val rlpEncodedPayload = ExecutionPayloadV1RLPEncoderByBesuImplementation.encode(payload) - val block = Block.readFrom(BytesValueRLPInput(Bytes.wrap(rlpEncodedPayload), false), MainnetBlockHeaderFunctions()) - - assertThat(block.header.number.toULong()).isEqualTo(payload.blockNumber.toULong()) - // we cannot assert oh block hash because Besu will calculate real Hash whereas random payload has random bytes - // assertThat(block.header.blockHash.toHexString()).isEqualTo(payload.blockHash.toHexString()) - assertThat(block.header.gasLimit.toULong()).isEqualTo(payload.gasLimit.toULong()) - assertThat(block.header.logsBloom.toArray()).isEqualTo(payload.logsBloom.toArray()) - assertThat(block.header.parentHash.toArray()).isEqualTo(payload.parentHash.toArray()) - assertThat(block.header.prevRandao.get().toArray()).isEqualTo(payload.prevRandao.toArray()) - assertThat(block.header.stateRoot.toArray()) - .isEqualTo(payload.stateRoot.toArray()) - assertThat(payload.transactions).isEmpty() - - // FIXME: add remaining fields assertions - } -} diff --git a/coordinator/persistence/blob/build.gradle b/coordinator/persistence/blob/build.gradle index 8ad52ed6e..332e252a4 100644 --- a/coordinator/persistence/blob/build.gradle +++ b/coordinator/persistence/blob/build.gradle @@ -14,7 +14,6 @@ dependencies { testImplementation("com.fasterxml.jackson.core:jackson-annotations:${libs.versions.jackson.get()}") testImplementation("com.fasterxml.jackson.module:jackson-module-kotlin:${libs.versions.jackson.get()}") testImplementation "io.tmio:tuweni-units:${libs.versions.tuweni.get()}" - testImplementation("tech.pegasys.teku.internal:executionclient:${libs.versions.teku.get()}") testImplementation(project(":coordinator:persistence:db-common")) testImplementation(testFixtures(project(":coordinator:core"))) testImplementation(testFixtures(project(":jvm-libs:generic:extensions:kotlin"))) diff --git a/coordinator/persistence/feehistory/build.gradle b/coordinator/persistence/feehistory/build.gradle index 2c7aa94a8..585b8cbdd 100644 --- a/coordinator/persistence/feehistory/build.gradle +++ b/coordinator/persistence/feehistory/build.gradle @@ -16,7 +16,6 @@ dependencies { testImplementation("com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson.get()}") testImplementation("com.fasterxml.jackson.core:jackson-annotations:${libs.versions.jackson.get()}") testImplementation("com.fasterxml.jackson.module:jackson-module-kotlin:${libs.versions.jackson.get()}") - testImplementation("tech.pegasys.teku.internal:executionclient:${libs.versions.teku.get()}") testImplementation(testFixtures(project(":jvm-libs:generic:persistence:db"))) testImplementation(testFixtures(project(":jvm-libs:generic:extensions:kotlin"))) testImplementation("io.vertx:vertx-junit5") diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ede6c9b3d..4102e4cc2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -17,6 +17,7 @@ junit = "5.10.1" kotlinxDatetime = "0.6.1" ktlint = "0.47.0" log4j = "2.20.0" +slf4j = "1.7.30" micrometer = "1.8.4" netty = "4.1.92.Final" picoli = "4.7.1" diff --git a/jvm-libs/generic/extensions/futures/build.gradle b/jvm-libs/generic/extensions/futures/build.gradle index ae64c1924..c80bd5713 100644 --- a/jvm-libs/generic/extensions/futures/build.gradle +++ b/jvm-libs/generic/extensions/futures/build.gradle @@ -6,7 +6,7 @@ plugins { description = "Utilities related to futures used in Linea" dependencies { - implementation "io.vertx:vertx-core" + api "io.vertx:vertx-core" testImplementation("io.vertx:vertx-junit5") } diff --git a/jvm-libs/generic/extensions/kotlin/src/main/kotlin/net/consensys/StringExtensions.kt b/jvm-libs/generic/extensions/kotlin/src/main/kotlin/net/consensys/StringExtensions.kt index 68f846f90..e7dadd7df 100644 --- a/jvm-libs/generic/extensions/kotlin/src/main/kotlin/net/consensys/StringExtensions.kt +++ b/jvm-libs/generic/extensions/kotlin/src/main/kotlin/net/consensys/StringExtensions.kt @@ -1,5 +1,6 @@ package net.consensys +import java.math.BigInteger import java.util.HexFormat fun String.decodeHex(): ByteArray { @@ -10,3 +11,8 @@ fun String.decodeHex(): ByteArray { fun String.containsAny(strings: List, ignoreCase: Boolean): Boolean { return strings.any { this.contains(it, ignoreCase) } } + +fun String.toIntFromHex(): Int = removePrefix("0x").toInt(16) +fun String.toLongFromHex(): Long = removePrefix("0x").toLong(16) +fun String.toULongFromHex(): ULong = BigInteger(removePrefix("0x"), 16).toULong() +fun String.toBigIntegerFromHex(): BigInteger = BigInteger(removePrefix("0x"), 16) diff --git a/jvm-libs/generic/extensions/kotlin/src/main/kotlin/net/consensys/TypingsExtensions.kt b/jvm-libs/generic/extensions/kotlin/src/main/kotlin/net/consensys/TypingsExtensions.kt index 6ec6c7487..650de984b 100644 --- a/jvm-libs/generic/extensions/kotlin/src/main/kotlin/net/consensys/TypingsExtensions.kt +++ b/jvm-libs/generic/extensions/kotlin/src/main/kotlin/net/consensys/TypingsExtensions.kt @@ -64,7 +64,7 @@ fun ULong.toGWei(): Double = this.toDouble().toGWei() * Parses an hexadecimal string as [ULong] number and returns the result. * @throws NumberFormatException if the string is not a valid hexadecimal representation of a number. */ -fun ULong.Companion.fromHexString(value: String): ULong = value.replace("0x", "").toULong(16) +fun ULong.Companion.fromHexString(value: String): ULong = value.removePrefix("0x").toULong(16) fun > ClosedRange.toIntervalString(): String { val size = if (start <= endInclusive) { diff --git a/jvm-libs/generic/extensions/kotlin/src/test/kotlin/net/consensys/StringExtensionsTest.kt b/jvm-libs/generic/extensions/kotlin/src/test/kotlin/net/consensys/StringExtensionsTest.kt index 28549f425..14854e3e6 100644 --- a/jvm-libs/generic/extensions/kotlin/src/test/kotlin/net/consensys/StringExtensionsTest.kt +++ b/jvm-libs/generic/extensions/kotlin/src/test/kotlin/net/consensys/StringExtensionsTest.kt @@ -27,4 +27,30 @@ class StringExtensionsTest { assertThat("this includes lorem ipsum".containsAny(stringList, ignoreCase = true)).isTrue() assertThat("this string won't match".containsAny(stringList, ignoreCase = true)).isFalse() } + + @OptIn(ExperimentalStdlibApi::class) + @Test + fun `String#toIntFromHex`() { + assertThat("0x00".toIntFromHex()).isEqualTo(0) + assertThat("0x01".toIntFromHex()).isEqualTo(1) + assertThat("0x123456".toIntFromHex()).isEqualTo(1193046) + assertThat("0x7FFFFFFF".toIntFromHex()).isEqualTo(Int.MAX_VALUE) + } + + @OptIn(ExperimentalStdlibApi::class) + @Test + fun `String#toLongFromHex`() { + assertThat("0x00".toLongFromHex()).isEqualTo(0L) + assertThat("0x01".toLongFromHex()).isEqualTo(1L) + assertThat("0x123456".toLongFromHex()).isEqualTo(1193046L) + assertThat("0x7FFFFFFFFFFFFFFF".toLongFromHex()).isEqualTo(Long.MAX_VALUE) + } + + @Test + fun `String#toULongFromHex`() { + assertThat("0x00".toULongFromHex()).isEqualTo(0UL) + assertThat("0x01".toULongFromHex()).isEqualTo(1UL) + assertThat("0x123456".toULongFromHex()).isEqualTo(1193046UL) + assertThat("0xffffffffffffffff".toULongFromHex()).isEqualTo(ULong.MAX_VALUE) + } } diff --git a/jvm-libs/linea/besu-libs/build.gradle b/jvm-libs/linea/besu-libs/build.gradle index bcbebac1f..4e009a0d2 100644 --- a/jvm-libs/linea/besu-libs/build.gradle +++ b/jvm-libs/linea/besu-libs/build.gradle @@ -20,4 +20,16 @@ dependencies { api("org.hyperledger.besu:plugin-api:${libs.versions.besu.get()}") { transitive = false } + + api("org.hyperledger.besu.internal:rlp:${libs.versions.besu.get()}") { + transitive = false + } + + api("io.tmio:tuweni-bytes:${libs.versions.tuweni.get()}") { + transitive = false + } + + api("io.tmio:tuweni-units:${libs.versions.tuweni.get()}") { + transitive = false + } } diff --git a/jvm-libs/linea/besu-rlp-and-mappers/build.gradle b/jvm-libs/linea/besu-rlp-and-mappers/build.gradle new file mode 100644 index 000000000..3e1576a6b --- /dev/null +++ b/jvm-libs/linea/besu-rlp-and-mappers/build.gradle @@ -0,0 +1,11 @@ +plugins { + id 'net.consensys.zkevm.kotlin-library-conventions' +} + +dependencies { + api(project(':jvm-libs:generic:extensions:kotlin')) + api(project(':jvm-libs:generic:extensions:futures')) + api(project(':jvm-libs:linea:core:domain-models')) + api(project(':jvm-libs:linea:besu-libs')) + api "io.vertx:vertx-core" +} diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperBesuToLineaDomain.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperBesuToLineaDomain.kt new file mode 100644 index 000000000..ae3dd619a --- /dev/null +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperBesuToLineaDomain.kt @@ -0,0 +1,57 @@ +package linea.domain + +import linea.domain.MapperBesuToLineaDomain.mapToDomain +import net.consensys.toULong +import org.hyperledger.besu.ethereum.core.Transaction +import kotlin.jvm.optionals.getOrNull + +fun org.hyperledger.besu.ethereum.core.Block.toDomain(): Block { + return mapToDomain(this) +} + +object MapperBesuToLineaDomain { + fun mapToDomain(besuBlock: org.hyperledger.besu.ethereum.core.Block): Block { + val block = Block( + number = besuBlock.header.getNumber().toULong(), + hash = besuBlock.header.hash.toArray(), + parentHash = besuBlock.header.parentHash.toArray(), + ommersHash = besuBlock.header.ommersHash.toArray(), + miner = besuBlock.header.coinbase.toArray(), + stateRoot = besuBlock.header.stateRoot.toArray(), + transactionsRoot = besuBlock.header.transactionsRoot.toArray(), + receiptsRoot = besuBlock.header.receiptsRoot.toArray(), + logsBloom = besuBlock.header.logsBloom.toArray(), + difficulty = besuBlock.header.difficulty.toBigInteger().toULong(), + gasLimit = besuBlock.header.gasLimit.toULong(), + gasUsed = besuBlock.header.gasUsed.toULong(), + timestamp = besuBlock.header.timestamp.toULong(), + extraData = besuBlock.header.extraData.toArray(), + mixHash = besuBlock.header.mixHash.toArray(), + nonce = besuBlock.header.nonce.toULong(), + baseFeePerGas = besuBlock.header.baseFee.getOrNull()?.toBigInteger()?.toULong(), + ommers = besuBlock.body.ommers.map { it.hash.toArray() }, + transactions = besuBlock.body.transactions.map(MapperBesuToLineaDomain::mapToDomain) + ) + + return block + } + + fun mapToDomain(transaction: Transaction): linea.domain.Transaction { + return Transaction( + nonce = transaction.nonce.toULong(), + gasPrice = transaction.getGasPrice().getOrNull()?.toBigInteger()?.toULong(), + gasLimit = transaction.gasLimit.toULong(), + to = transaction.to?.getOrNull()?.toArray(), + value = transaction.value.toBigInteger(), + input = transaction.payload.toArray(), + r = transaction.signature.getR(), + s = transaction.signature.getS(), + v = transaction.getV().toULong(), + yParity = transaction.yParity?.toULong(), + type = transaction.type.toDomain(), + chainId = transaction.chainId?.getOrNull()?.toULong(), + maxFeePerGas = transaction.maxFeePerGas?.getOrNull()?.toBigInteger()?.toULong(), + maxPriorityFeePerGas = transaction.maxPriorityFeePerGas?.getOrNull()?.toBigInteger()?.toULong() + ) + } +} diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperLineaDomainToBesu.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperLineaDomainToBesu.kt new file mode 100644 index 000000000..6583ec492 --- /dev/null +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperLineaDomainToBesu.kt @@ -0,0 +1,122 @@ +package linea.domain + +import linea.domain.MapperLineaDomainToBesu.mapToBesu +import net.consensys.encodeHex +import net.consensys.toBigInteger +import org.apache.tuweni.bytes.Bytes +import org.apache.tuweni.bytes.Bytes32 +import org.hyperledger.besu.crypto.SECP256K1 +import org.hyperledger.besu.datatypes.AccessListEntry +import org.hyperledger.besu.datatypes.Address +import org.hyperledger.besu.datatypes.Hash +import org.hyperledger.besu.datatypes.Wei +import org.hyperledger.besu.ethereum.core.BlockBody +import org.hyperledger.besu.ethereum.core.BlockHeaderBuilder +import org.hyperledger.besu.ethereum.core.Difficulty +import org.hyperledger.besu.ethereum.core.Transaction +import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions +import org.hyperledger.besu.evm.log.LogsBloomFilter +import java.math.BigInteger + +fun Block.toBesu(): org.hyperledger.besu.ethereum.core.Block = mapToBesu(this) +fun linea.domain.Transaction.toBesu(): Transaction = mapToBesu(this) + +object MapperLineaDomainToBesu { + private val secp256k1 = SECP256K1() + private val blockHeaderFunctions = MainnetBlockHeaderFunctions() + + fun recIdFromV(v: BigInteger): Pair { + val recId: Byte + var chainId: BigInteger? = null + if (v == Transaction.REPLAY_UNPROTECTED_V_BASE || v == Transaction.REPLAY_UNPROTECTED_V_BASE_PLUS_1) { + recId = v.subtract(Transaction.REPLAY_UNPROTECTED_V_BASE).byteValueExact() + } else if (v > Transaction.REPLAY_PROTECTED_V_MIN) { + chainId = v.subtract(Transaction.REPLAY_PROTECTED_V_BASE).divide(Transaction.TWO) + recId = v.subtract(Transaction.TWO.multiply(chainId).add(Transaction.REPLAY_PROTECTED_V_BASE)).byteValueExact() + } else { + throw RuntimeException("An unsupported encoded `v` value of $v was found") + } + return Pair(recId, chainId) + } + + fun getRecIdAndChainId(tx: linea.domain.Transaction): Pair { + if (tx.type == TransactionType.FRONTIER) { + return recIdFromV(tx.v.toBigInteger()) + } else { + return tx.v.toByte() to tx.chainId?.toBigInteger() + } + } + + fun mapToBesu(block: Block): org.hyperledger.besu.ethereum.core.Block { + runCatching { + val header = BlockHeaderBuilder.create() + .parentHash(Hash.wrap(Bytes32.wrap(block.parentHash))) + .ommersHash(Hash.wrap(Bytes32.wrap(block.ommersHash))) + .coinbase(Address.wrap(Bytes.wrap(block.miner))) + .stateRoot(Hash.wrap(Bytes32.wrap(block.stateRoot))) + .transactionsRoot(Hash.wrap(Bytes32.wrap(block.transactionsRoot))) + .receiptsRoot(Hash.wrap(Bytes32.wrap(block.receiptsRoot))) + .logsBloom(LogsBloomFilter.fromHexString(block.logsBloom.encodeHex())) + .difficulty(Difficulty.fromHexOrDecimalString(block.difficulty.toString())) + .number(block.number.toLong()) + .gasLimit(block.gasLimit.toLong()) + .gasUsed(block.gasUsed.toLong()) + .timestamp(block.timestamp.toLong()) + .extraData(Bytes.wrap(block.extraData)) + .mixHash(Hash.wrap(Bytes32.wrap(block.mixHash))) + .nonce(block.nonce.toLong()) + .baseFee(block.baseFeePerGas?.toWei()) + .blockHeaderFunctions(blockHeaderFunctions) + .buildBlockHeader() + + val transactions = block.transactions.map(MapperLineaDomainToBesu::mapToBesu) + // linea does not support uncles, so we are not converting them + // throwing an exception just in case we get one and we can fix it + if (block.ommers.isNotEmpty()) { + throw IllegalStateException("Uncles are not supported: block=${block.number}") + } + + val body = BlockBody(transactions, emptyList()) + + return org.hyperledger.besu.ethereum.core.Block(header, body) + }.getOrElse { + throw IllegalStateException("Error mapping block to Besu: block=${block.number}", it) + } + } + + fun mapToBesu(tx: linea.domain.Transaction): Transaction { + val (recId, chainId) = getRecIdAndChainId(tx) + val signature = secp256k1.createSignature( + tx.r, + tx.s, + recId + ) + + return Transaction.builder() + .type(tx.type.toBesu()) + .nonce(tx.nonce.toLong()) + .apply { tx.gasPrice?.let { gasPrice(it.toWei()) } } + .gasLimit(tx.gasLimit.toLong()) + .to(tx.to?.let { Address.wrap(Bytes.wrap(it)) }) + .value(tx.value.toWei()) + .payload(Bytes.wrap(tx.input)) + .chainId(tx.chainId?.toBigInteger() ?: chainId) + .maxPriorityFeePerGas(tx.maxPriorityFeePerGas?.toWei()) + .maxFeePerGas(tx.maxFeePerGas?.toWei()) + .apply { + if (!tx.accessList.isEmpty()) { + tx.accessList.map { entry -> + AccessListEntry( + Address.wrap(Bytes.wrap(entry.address)), + entry.storageKeys.map { Bytes32.wrap(it) } + ) + } + } + } + .signature(signature) + .build() + } + + fun ULong.toWei(): Wei = Wei.of(this.toBigInteger()) + fun BigInteger.toWei(): Wei = Wei.of(this) +} diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/TransactionTypeMapper.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/TransactionTypeMapper.kt new file mode 100644 index 000000000..bb706789d --- /dev/null +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/TransactionTypeMapper.kt @@ -0,0 +1,23 @@ +package linea.domain + +import org.hyperledger.besu.datatypes.TransactionType + +fun TransactionType.toDomain(): linea.domain.TransactionType { + return when (this) { + TransactionType.FRONTIER -> linea.domain.TransactionType.FRONTIER + TransactionType.EIP1559 -> linea.domain.TransactionType.EIP1559 + TransactionType.ACCESS_LIST -> linea.domain.TransactionType.ACCESS_LIST + TransactionType.BLOB -> linea.domain.TransactionType.BLOB + TransactionType.DELEGATE_CODE -> linea.domain.TransactionType.DELEGATE_CODE + } +} + +fun linea.domain.TransactionType.toBesu(): TransactionType { + return when (this) { + linea.domain.TransactionType.FRONTIER -> TransactionType.FRONTIER + linea.domain.TransactionType.EIP1559 -> TransactionType.EIP1559 + linea.domain.TransactionType.ACCESS_LIST -> TransactionType.ACCESS_LIST + linea.domain.TransactionType.BLOB -> TransactionType.BLOB + linea.domain.TransactionType.DELEGATE_CODE -> TransactionType.DELEGATE_CODE + } +} diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRLPBlockEncoder.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRLPBlockEncoder.kt new file mode 100644 index 000000000..0e0db2917 --- /dev/null +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRLPBlockEncoder.kt @@ -0,0 +1,40 @@ +package linea.rlp + +import io.vertx.core.Vertx +import linea.domain.Block +import net.consensys.linea.async.toSafeFuture +import tech.pegasys.teku.infrastructure.async.SafeFuture +import java.util.concurrent.Callable + +class BesuRLPBlockEncoder( + val vertx: Vertx +) : RLPBlockEncoderAsync { + + override fun encodeAsync(block: Block): SafeFuture { + return vertx.executeBlocking( + Callable { + RLP.encode(block) + }, + false + ) + .toSafeFuture() + } + + override fun encodeAsync(blocks: List): SafeFuture> { + return SafeFuture.collectAll(blocks.map { encodeAsync(it) }.stream()) + } + + override fun decodeAsync(block: ByteArray): SafeFuture { + return vertx.executeBlocking( + Callable { + RLP.decode(block) + }, + false + ) + .toSafeFuture() + } + + override fun decodeAsync(blocks: List): SafeFuture> { + return SafeFuture.collectAll(blocks.map { decodeAsync(it) }.stream()) + } +} diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/RLP.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/RLP.kt new file mode 100644 index 000000000..1922b6a89 --- /dev/null +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/RLP.kt @@ -0,0 +1,53 @@ +package linea.rlp + +import linea.domain.toBesu +import linea.domain.toDomain +import org.apache.tuweni.bytes.Bytes +import org.hyperledger.besu.ethereum.core.Block +import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput +import org.hyperledger.besu.ethereum.rlp.RLP + +object RLP : RLPBlockEncoder { + override fun encode(block: linea.domain.Block): ByteArray { + return encodeBlock(block.toBesu()) + } + + override fun decode(block: ByteArray): linea.domain.Block { + return decodeBlock(block).toDomain() + } + + fun encodeBlock(besuBlock: org.hyperledger.besu.ethereum.core.Block): ByteArray { + return besuBlock.toRlp().toArray() + } + + fun decodeBlock(block: ByteArray): org.hyperledger.besu.ethereum.core.Block { + return Block.readFrom( + RLP.input(Bytes.wrap(block)), + MainnetBlockHeaderFunctions() + ) + } + + fun encodeList(list: List): ByteArray { + val encoder = BytesValueRLPOutput() + encoder.startList() + list.forEach { + encoder.writeBytes(Bytes.wrap(it)) + } + encoder.endList() + return encoder.encoded().toArray() + } + + fun decodeList( + bytes: ByteArray + ): List { + val items = mutableListOf() + val rlpInput = RLP.input(Bytes.wrap(bytes), false) + rlpInput.enterList() + while (!rlpInput.isEndOfCurrentList) { + items.add(rlpInput.readBytes().toArray()) + } + rlpInput.leaveList() + return items + } +} diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/RLPBlockEncoder.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/RLPBlockEncoder.kt new file mode 100644 index 000000000..c6a0506c9 --- /dev/null +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/RLPBlockEncoder.kt @@ -0,0 +1,18 @@ +package linea.rlp + +import linea.domain.Block +import tech.pegasys.teku.infrastructure.async.SafeFuture + +interface RLPBlockEncoder { + fun encode(block: Block): ByteArray + fun encode(blocks: List): List = blocks.map { encode(it) } + fun decode(block: ByteArray): Block + fun decode(blocks: List): List = blocks.map { decode(it) } +} + +interface RLPBlockEncoderAsync { + fun encodeAsync(block: Block): SafeFuture + fun encodeAsync(blocks: List): SafeFuture> + fun decodeAsync(block: ByteArray): SafeFuture + fun decodeAsync(blocks: List): SafeFuture> +} diff --git a/jvm-libs/linea/core/domain-models/build.gradle b/jvm-libs/linea/core/domain-models/build.gradle index 9f0f46350..903cb0d26 100644 --- a/jvm-libs/linea/core/domain-models/build.gradle +++ b/jvm-libs/linea/core/domain-models/build.gradle @@ -1,9 +1,11 @@ plugins { id 'net.consensys.zkevm.kotlin-common-conventions' + id 'java-test-fixtures' } dependencies { implementation project(":jvm-libs:generic:extensions:kotlin") + testFixturesApi "org.jetbrains.kotlinx:kotlinx-datetime:${libs.versions.kotlinxDatetime.get()}" } jar { diff --git a/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/Block.kt b/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/Block.kt new file mode 100644 index 000000000..ba8c87a87 --- /dev/null +++ b/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/Block.kt @@ -0,0 +1,146 @@ +package linea.domain + +import kotlinx.datetime.Instant +import net.consensys.encodeHex +import net.consensys.linea.BlockNumberAndHash + +data class Block( + val number: ULong, + val hash: ByteArray, + val parentHash: ByteArray, + val ommersHash: ByteArray, + val miner: ByteArray, + val stateRoot: ByteArray, + val transactionsRoot: ByteArray, + val receiptsRoot: ByteArray, + val logsBloom: ByteArray, + val difficulty: ULong, + val gasLimit: ULong, + val gasUsed: ULong, + val timestamp: ULong, + val extraData: ByteArray, + val mixHash: ByteArray, + val nonce: ULong, + val baseFeePerGas: ULong? = null, // Optional field for EIP-1559 blocks + val transactions: List = emptyList(), // List of transaction hashes + val ommers: List = emptyList() // List of uncle block hashes +) { + companion object { + // companion object to allow static extension functions + } + + val numberAndHash = BlockNumberAndHash(this.number, this.hash) + val headerSummary = BlockHeaderSummary(this.number, this.hash, Instant.fromEpochSeconds(this.timestamp.toLong())) + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as Block + + if (number != other.number) return false + if (!hash.contentEquals(other.hash)) return false + if (!parentHash.contentEquals(other.parentHash)) return false + if (!ommersHash.contentEquals(other.ommersHash)) return false + if (!miner.contentEquals(other.miner)) return false + if (!stateRoot.contentEquals(other.stateRoot)) return false + if (!transactionsRoot.contentEquals(other.transactionsRoot)) return false + if (!receiptsRoot.contentEquals(other.receiptsRoot)) return false + if (!logsBloom.contentEquals(other.logsBloom)) return false + if (difficulty != other.difficulty) return false + if (gasLimit != other.gasLimit) return false + if (gasUsed != other.gasUsed) return false + if (timestamp != other.timestamp) return false + if (!extraData.contentEquals(other.extraData)) return false + if (!mixHash.contentEquals(other.mixHash)) return false + if (nonce != other.nonce) return false + if (baseFeePerGas != other.baseFeePerGas) return false + if (transactions != other.transactions) return false + if (ommers != other.ommers) return false + if (numberAndHash != other.numberAndHash) return false + if (headerSummary != other.headerSummary) return false + + return true + } + + override fun hashCode(): Int { + var result = number.hashCode() + result = 31 * result + hash.contentHashCode() + result = 31 * result + parentHash.contentHashCode() + result = 31 * result + ommersHash.contentHashCode() + result = 31 * result + miner.contentHashCode() + result = 31 * result + stateRoot.contentHashCode() + result = 31 * result + transactionsRoot.contentHashCode() + result = 31 * result + receiptsRoot.contentHashCode() + result = 31 * result + logsBloom.contentHashCode() + result = 31 * result + difficulty.hashCode() + result = 31 * result + gasLimit.hashCode() + result = 31 * result + gasUsed.hashCode() + result = 31 * result + timestamp.hashCode() + result = 31 * result + extraData.contentHashCode() + result = 31 * result + mixHash.contentHashCode() + result = 31 * result + nonce.hashCode() + result = 31 * result + (baseFeePerGas?.hashCode() ?: 0) + result = 31 * result + transactions.hashCode() + result = 31 * result + ommers.hashCode() + result = 31 * result + numberAndHash.hashCode() + result = 31 * result + headerSummary.hashCode() + return result + } + + override fun toString(): String { + return "Block(" + + "number=$number, " + + "hash=${hash.encodeHex()}, " + + "parentHash=${parentHash.encodeHex()}, " + + "ommersHash=${ommersHash.encodeHex()}, " + + "miner=${miner.encodeHex()}, " + + "stateRoot=${stateRoot.encodeHex()}, " + + "transactionsRoot=${transactionsRoot.encodeHex()}, " + + "receiptsRoot=${receiptsRoot.encodeHex()}, " + + "logsBloom=${logsBloom.encodeHex()}, " + + "difficulty=$difficulty, " + + "gasLimit=$gasLimit, " + + "gasUsed=$gasUsed, " + + "timestamp=$timestamp, " + + "extraData=${extraData.encodeHex()}, " + + "mixHash=${mixHash.encodeHex()}, " + + "nonce=$nonce, " + + "baseFeePerGas=$baseFeePerGas, " + + "transactions=$transactions, " + + "ommers=$ommers" + ")" + } +} + +data class BlockHeaderSummary( + val number: ULong, + val hash: ByteArray, + val timestamp: Instant +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as BlockHeaderSummary + + if (number != other.number) return false + if (!hash.contentEquals(other.hash)) return false + if (timestamp != other.timestamp) return false + + return true + } + + override fun hashCode(): Int { + var result = number.hashCode() + result = 31 * result + hash.contentHashCode() + result = 31 * result + timestamp.hashCode() + return result + } + + override fun toString(): String { + return "BlockHeaderSummary(" + + "number=$number, " + + "hash=${hash.contentToString()}, " + + "timestamp=$timestamp)" + } +} diff --git a/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/Transaction.kt b/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/Transaction.kt new file mode 100644 index 000000000..75f8652c5 --- /dev/null +++ b/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/Transaction.kt @@ -0,0 +1,158 @@ +package linea.domain + +import net.consensys.encodeHex +import java.math.BigInteger +import java.util.EnumSet + +enum class TransactionType(private val typeValue: Int) { + FRONTIER(248), + ACCESS_LIST(1), + EIP1559(2), + BLOB(3), // Not supported by Linea atm, but here for completeness + DELEGATE_CODE(4); // Not supported by Linea atm, but here for completeness + + val serializedType: Byte + get() = typeValue.toByte() + + val ethSerializedType: Byte + get() = if (this == FRONTIER) 0 else serializedType + + fun compareTo(b: Byte?): Int { + return serializedType.compareTo(b!!) + } + + companion object { + private val ACCESS_LIST_SUPPORTED_TRANSACTION_TYPES: Set = + EnumSet.of(ACCESS_LIST, EIP1559, BLOB, DELEGATE_CODE) + private val LEGACY_FEE_MARKET_TRANSACTION_TYPES: Set = EnumSet.of(FRONTIER, ACCESS_LIST) + + fun fromSerializedValue(serializedTypeValue: Int): TransactionType { + return entries + .firstOrNull { type: TransactionType -> type.typeValue == serializedTypeValue } + ?: throw IllegalArgumentException( + String.format( + "Unsupported transaction type %x", + serializedTypeValue + ) + ) + } + + fun fromEthApiSerializedValue(serializedTypeValue: Int): TransactionType { + if (serializedTypeValue == 0) { + return FRONTIER + } + return fromSerializedValue(serializedTypeValue) + } + } +} + +data class Transaction( + val nonce: ULong, + val gasPrice: ULong?, + val gasLimit: ULong, + val to: ByteArray?, // Nullable for contract creation transactions + val value: BigInteger, + val input: ByteArray, + val r: BigInteger, + val s: BigInteger, + val v: ULong, + val yParity: ULong?, + val type: TransactionType, + val chainId: ULong? = null, // Optional field for EIP-155 transactions + val maxPriorityFeePerGas: ULong? = null, // null for non EIP-1559 transactions + val maxFeePerGas: ULong? = null, // null for EIP-1559 transactions + val accessList: List = emptyList() // null non for EIP-2930 transactions +) { + companion object { + // companion object to allow static extension functions + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as Transaction + + if (nonce != other.nonce) return false + if (gasPrice != other.gasPrice) return false + if (gasLimit != other.gasLimit) return false + if (to != null) { + if (other.to == null) return false + if (!to.contentEquals(other.to)) return false + } else if (other.to != null) return false + if (value != other.value) return false + if (!input.contentEquals(other.input)) return false + if (r != other.r) return false + if (s != other.s) return false + if (v != other.v) return false + if (yParity != other.yParity) return false + if (type != other.type) return false + if (chainId != other.chainId) return false + if (maxPriorityFeePerGas != other.maxPriorityFeePerGas) return false + if (maxFeePerGas != other.maxFeePerGas) return false + if (accessList != other.accessList) return false + + return true + } + + override fun hashCode(): Int { + var result = nonce.hashCode() + result = 31 * result + gasPrice.hashCode() + result = 31 * result + gasLimit.hashCode() + result = 31 * result + (to?.contentHashCode() ?: 0) + result = 31 * result + value.hashCode() + result = 31 * result + input.contentHashCode() + result = 31 * result + r.hashCode() + result = 31 * result + s.hashCode() + result = 31 * result + v.hashCode() + result = 31 * result + yParity.hashCode() + result = 31 * result + type.hashCode() + result = 31 * result + (chainId?.hashCode() ?: 0) + result = 31 * result + (maxPriorityFeePerGas?.hashCode() ?: 0) + result = 31 * result + (maxFeePerGas?.hashCode() ?: 0) + result = 31 * result + accessList.hashCode() + return result + } + + override fun toString(): String { + return "Transaction(" + + "nonce=$nonce, " + + "gasPrice=$gasPrice, " + + "gasLimit=$gasLimit, " + + "to=${to?.encodeHex()}, " + + "value=$value, " + + "input=${input.encodeHex()}, " + + "r=$r, " + + "s=$s, " + + "v=$v, " + + "yParity=$yParity, " + + "type=$type, " + + "chainId=$chainId, " + + "maxPriorityFeePerGas=$maxPriorityFeePerGas, " + + "maxFeePerGas=$maxFeePerGas, " + + "accessList=$accessList)" + } +} + +data class AccessListEntry( + val address: ByteArray, + val storageKeys: List +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as AccessListEntry + + if (!address.contentEquals(other.address)) return false + if (storageKeys != other.storageKeys) return false + + return true + } + + override fun hashCode(): Int { + var result = address.contentHashCode() + result = 31 * result + storageKeys.hashCode() + return result + } +} diff --git a/jvm-libs/linea/core/domain-models/src/testFixtures/kotlin/linea/domain/BlockFactory.kt b/jvm-libs/linea/core/domain-models/src/testFixtures/kotlin/linea/domain/BlockFactory.kt new file mode 100644 index 000000000..1526a7cda --- /dev/null +++ b/jvm-libs/linea/core/domain-models/src/testFixtures/kotlin/linea/domain/BlockFactory.kt @@ -0,0 +1,47 @@ +package linea.domain + +import kotlinx.datetime.Clock +import kotlinx.datetime.Instant +import net.consensys.ByteArrayExt + +val zeroHash = ByteArray(32) { 0 } + +fun createBlock( + number: ULong = 0UL, + hash: ByteArray = ByteArrayExt.random32(), + gasLimit: ULong = 60_000_000UL, + gasUsed: ULong = 30_000_000UL, + difficulty: ULong = 2UL, + parentHash: ByteArray = ByteArrayExt.random32(), + stateRoot: ByteArray = ByteArrayExt.random32(), + receiptsRoot: ByteArray = ByteArrayExt.random32(), + logsBloom: ByteArray = ByteArrayExt.random32(), + ommersHash: ByteArray = ByteArrayExt.random32(), + timestamp: Instant = Clock.System.now(), + extraData: ByteArray = ByteArrayExt.random32(), + baseFeePerGas: ULong = 7UL, + transactionsRoot: ByteArray = ByteArrayExt.random32(), + transactions: List = emptyList() +): Block { + return Block( + number = number, + hash = hash, + parentHash = parentHash, + ommersHash = ommersHash, + miner = zeroHash, + stateRoot = stateRoot, + transactionsRoot = transactionsRoot, + receiptsRoot = receiptsRoot, + logsBloom = logsBloom, + difficulty = difficulty, + gasLimit = gasLimit, + gasUsed = gasUsed, + timestamp = timestamp.epochSeconds.toULong(), + extraData = extraData, + mixHash = zeroHash, + nonce = 0UL, + baseFeePerGas = baseFeePerGas, + transactions = transactions, + ommers = emptyList() + ) +} diff --git a/jvm-libs/linea/teku-execution-client/build.gradle b/jvm-libs/linea/teku-execution-client/build.gradle deleted file mode 100644 index e0071179c..000000000 --- a/jvm-libs/linea/teku-execution-client/build.gradle +++ /dev/null @@ -1,19 +0,0 @@ -plugins { - id 'net.consensys.zkevm.kotlin-library-conventions' - id 'java-library' -} - -dependencies { - api("tech.pegasys.teku.internal:executionclient:${libs.versions.teku.get()}") { - exclude group: 'org.hyperledger.besu' - exclude group: 'org.web3j' - exclude group: 'com.github.jnr' - exclude group: 'com.squareup.okhttp3' - exclude group: 'io.reactivex.rxjava2' - exclude group: 'org.java-websocket' - exclude group: 'com.fasterxml.jackson.core:jackson-databind' - exclude group: 'org.slf4j' - exclude group: 'tech.pegasys.teku.internal' - exclude group: 'io.jsonwebtoken' - } -} diff --git a/jvm-libs/linea/testing/teku-helper/build.gradle b/jvm-libs/linea/testing/teku-helper/build.gradle deleted file mode 100644 index 74e7ffbb0..000000000 --- a/jvm-libs/linea/testing/teku-helper/build.gradle +++ /dev/null @@ -1,14 +0,0 @@ -plugins { - id 'net.consensys.zkevm.kotlin-library-conventions' -} - -dependencies { - api project(":jvm-libs:linea:teku-execution-client") - api "tech.pegasys.teku.internal:unsigned:${libs.versions.teku.get()}" - api "tech.pegasys.teku.internal:bytes:${libs.versions.teku.get()}" - - implementation "tech.pegasys.teku.internal:spec:${libs.versions.teku.get()}" - implementation "tech.pegasys.teku.internal:spec:${libs.versions.teku.get()}:test-fixtures" - implementation "io.tmio:tuweni-units:${libs.versions.tuweni.get()}" - implementation project(':jvm-libs:generic:extensions:kotlin') -} diff --git a/jvm-libs/linea/testing/teku-helper/src/main/kotlin/tech/pegasys/teku/ethereum/executionclient/schema/ExecutionPayloadV1.kt b/jvm-libs/linea/testing/teku-helper/src/main/kotlin/tech/pegasys/teku/ethereum/executionclient/schema/ExecutionPayloadV1.kt deleted file mode 100644 index bf862d111..000000000 --- a/jvm-libs/linea/testing/teku-helper/src/main/kotlin/tech/pegasys/teku/ethereum/executionclient/schema/ExecutionPayloadV1.kt +++ /dev/null @@ -1,131 +0,0 @@ -package tech.pegasys.teku.ethereum.executionclient.schema - -import kotlinx.datetime.Clock -import kotlinx.datetime.Instant -import net.consensys.ByteArrayExt -import net.consensys.toBigInteger -import org.apache.tuweni.bytes.Bytes -import org.apache.tuweni.bytes.Bytes32 -import org.apache.tuweni.units.bigints.UInt256 -import tech.pegasys.teku.infrastructure.bytes.Bytes20 -import tech.pegasys.teku.infrastructure.unsigned.UInt64 -import tech.pegasys.teku.spec.TestSpecFactory -import tech.pegasys.teku.spec.util.DataStructureUtil -import java.math.BigInteger - -fun executionPayloadV1( - blockNumber: Long = 0, - parentHash: ByteArray = ByteArrayExt.random32(), - feeRecipient: ByteArray = ByteArrayExt.random(20), - stateRoot: ByteArray = ByteArrayExt.random32(), - receiptsRoot: ByteArray = ByteArrayExt.random32(), - logsBloom: ByteArray = ByteArrayExt.random32(), - prevRandao: ByteArray = ByteArrayExt.random32(), - gasLimit: ULong = 0UL, - gasUsed: ULong = 0UL, - timestamp: Instant = Clock.System.now(), - extraData: ByteArray = ByteArrayExt.random32(), - baseFeePerGas: BigInteger = BigInteger.valueOf(256), - blockHash: ByteArray = ByteArrayExt.random32(), - transactions: List = emptyList() -): ExecutionPayloadV1 { - return ExecutionPayloadV1( - Bytes32.wrap(parentHash), - Bytes20(Bytes.wrap(feeRecipient)), - Bytes32.wrap(stateRoot), - Bytes32.wrap(receiptsRoot), - Bytes.wrap(logsBloom), - Bytes32.wrap(prevRandao), - UInt64.valueOf(blockNumber), - UInt64.valueOf(gasLimit.toBigInteger()), - UInt64.valueOf(gasUsed.toBigInteger()), - UInt64.valueOf(timestamp.epochSeconds), - Bytes.wrap(extraData), - UInt256.valueOf(baseFeePerGas), - Bytes32.wrap(blockHash), - transactions.map { Bytes.wrap(it) } - ) -} - -fun executionPayloadV1( - blockNumber: Long = 0, - parentHash: Bytes32 = Bytes32.random(), - feeRecipient: Bytes20 = Bytes20(Bytes.random(20)), - stateRoot: Bytes32 = Bytes32.random(), - receiptsRoot: Bytes32 = Bytes32.random(), - logsBloom: Bytes = Bytes32.random(), - prevRandao: Bytes32 = Bytes32.random(), - gasLimit: UInt64 = UInt64.valueOf(0), - gasUsed: UInt64 = UInt64.valueOf(0), - timestamp: UInt64 = UInt64.valueOf(0), - extraData: Bytes = Bytes32.random(), - baseFeePerGas: UInt256 = UInt256.valueOf(256), - blockHash: Bytes32 = Bytes32.random(), - transactions: List = emptyList() -): ExecutionPayloadV1 { - return ExecutionPayloadV1( - parentHash, - feeRecipient, - stateRoot, - receiptsRoot, - logsBloom, - prevRandao, - UInt64.valueOf(blockNumber), - gasLimit, - gasUsed, - timestamp, - extraData, - baseFeePerGas, - blockHash, - transactions - ) -} - -fun randomExecutionPayload( - transactionsRlp: List = emptyList(), - blockNumber: Long? = null -): ExecutionPayloadV1 { - val executionPayload = dataStructureUtil.randomExecutionPayload() - return ExecutionPayloadV1( - /* parentHash = */ executionPayload.parentHash, - /* feeRecipient = */ - executionPayload.feeRecipient, - /* stateRoot = */ - executionPayload.stateRoot, - /* receiptsRoot = */ - executionPayload.receiptsRoot, - /* logsBloom = */ - executionPayload.logsBloom, - /* prevRandao = */ - executionPayload.prevRandao, - /* blockNumber = */ - blockNumber?.let(UInt64::valueOf) ?: executionPayload.blockNumber.cropToPositiveSignedLong(), - /* gasLimit = */ - executionPayload.gasLimit.cropToPositiveSignedLong(), - /* gasUsed = */ - executionPayload.gasUsed.cropToPositiveSignedLong(), - /* timestamp = */ - executionPayload.timestamp.cropToPositiveSignedLong(), - /* extraData = */ - executionPayload.extraData, - /* baseFeePerGas = */ - executionPayload.baseFeePerGas, - /* blockHash = */ - executionPayload.blockHash, - /* transactions = */ - transactionsRlp - ) -} - -val dataStructureUtil: DataStructureUtil = DataStructureUtil(TestSpecFactory.createMinimalBellatrix()) - -// Teku UInt64 has a bug allow negative number to be created -// random test payload creates such cases we need to fix it -private fun UInt64.cropToPositiveSignedLong(): UInt64 { - val longValue = this.longValue() - return if (longValue < 0) { - return UInt64.valueOf(-longValue) - } else { - this - } -} diff --git a/jvm-libs/linea/web3j-extensions/build.gradle b/jvm-libs/linea/web3j-extensions/build.gradle index bfaec3e97..327e13719 100644 --- a/jvm-libs/linea/web3j-extensions/build.gradle +++ b/jvm-libs/linea/web3j-extensions/build.gradle @@ -8,19 +8,16 @@ dependencies { api "org.web3j:core:${libs.versions.web3j.get()}" api project(':jvm-libs:linea:core:domain-models') api project(':jvm-libs:generic:logging') - // For domain mappers api project(':jvm-libs:linea:besu-libs') implementation project(":jvm-libs:generic:extensions:kotlin") implementation "tech.pegasys.teku.internal:bytes:${libs.versions.teku.get()}" implementation "tech.pegasys.teku.internal:jackson:${libs.versions.teku.get()}" - // Returned by domain mapper - api project(":jvm-libs:linea:teku-execution-client") implementation "tech.pegasys.teku.internal:unsigned:${libs.versions.teku.get()}" - implementation "org.hyperledger.besu:besu-datatypes:${libs.versions.besu.get()}" testImplementation "org.apache.logging.log4j:log4j-slf4j2-impl:${libs.versions.log4j.get()}" testImplementation "com.fasterxml.jackson.core:jackson-annotations:${libs.versions.jackson.get()}" testImplementation "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson.get()}" + testImplementation project(":jvm-libs:linea:besu-rlp-and-mappers") } jar { diff --git a/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/EthGetBlockToLineaBlockMappers.kt b/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/EthGetBlockToLineaBlockMappers.kt new file mode 100644 index 000000000..99b799ea1 --- /dev/null +++ b/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/EthGetBlockToLineaBlockMappers.kt @@ -0,0 +1,74 @@ +package linea.web3j + +import linea.domain.Block +import linea.domain.Transaction +import linea.domain.TransactionType +import net.consensys.decodeHex +import net.consensys.toIntFromHex +import net.consensys.toULong +import net.consensys.toULongFromHex +import org.web3j.protocol.core.methods.response.EthBlock + +fun EthBlock.Block.toDomain(): Block = mapToDomain(this) + +fun mapToDomain(web3jBlock: EthBlock.Block): Block { + val block = Block( + number = web3jBlock.number.toULong(), + hash = web3jBlock.hash.decodeHex(), + parentHash = web3jBlock.parentHash.decodeHex(), + ommersHash = web3jBlock.sha3Uncles.decodeHex(), + miner = web3jBlock.miner.decodeHex(), + nonce = web3jBlock.nonce.toULong(), + stateRoot = web3jBlock.stateRoot.decodeHex(), + transactionsRoot = web3jBlock.transactionsRoot.decodeHex(), + receiptsRoot = web3jBlock.receiptsRoot.decodeHex(), + logsBloom = web3jBlock.logsBloom.decodeHex(), + difficulty = web3jBlock.difficulty.toULong(), + gasLimit = web3jBlock.gasLimit.toULong(), + gasUsed = web3jBlock.gasUsed.toULong(), + timestamp = web3jBlock.timestamp.toULong(), + extraData = web3jBlock.extraData.decodeHex(), + mixHash = web3jBlock.mixHash.decodeHex(), + baseFeePerGas = web3jBlock.baseFeePerGas?.toULong(), // Optional field for EIP-1559 blocks + ommers = web3jBlock.uncles.map { it.decodeHex() }, // List of uncle block hashes + transactions = run { + if (web3jBlock.transactions.isNotEmpty() && web3jBlock.transactions[0] !is EthBlock.TransactionObject) { + throw IllegalArgumentException( + "Expected to be have full EthBlock.TransactionObject." + + "Got just transaction hashes." + ) + } + web3jBlock.transactions.map { (it as EthBlock.TransactionObject).toDomain() } + } + ) + return block +} + +fun EthBlock.TransactionObject.toDomain(): Transaction { + val maxFeePerGas = this.maxFeePerGas?.toULong() + // Web3J throws an exception if maxPriorityFeePerGas null, instead of a check like in maxFeePerGas + // we need to check if maxFeePerGas is null to avoid the exception + val maxPriorityFeePerGas = if (maxFeePerGas != null) this.maxPriorityFeePerGas?.toULong() else null + return Transaction( + nonce = this.nonce.toULong(), + gasPrice = this.gasPrice.toULong(), + gasLimit = this.gas.toULong(), + to = this.to?.decodeHex(), + value = this.value, + input = this.input.decodeHex(), + r = this.r.removePrefix("0x").toBigInteger(16), + s = this.s.removePrefix("0x").toBigInteger(16), + v = this.v.toULong(), + yParity = this.getyParity()?.toULongFromHex(), + type = mapType(this.type), // Optional field for EIP-2718 typed transactions + chainId = this.chainId?.toULong(), // Optional field for EIP-155 transactions + maxFeePerGas = maxFeePerGas, // Optional field for EIP-1559 transactions + maxPriorityFeePerGas = maxPriorityFeePerGas // Optional field for EIP-1559 transactions + ) +} + +fun mapType(type: String?): TransactionType { + return type + ?.let { TransactionType.fromEthApiSerializedValue(it.toIntFromHex()) } + ?: TransactionType.FRONTIER +} diff --git a/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/Web3JFactory.kt b/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/Web3JFactory.kt new file mode 100644 index 000000000..6ac151866 --- /dev/null +++ b/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/Web3JFactory.kt @@ -0,0 +1,33 @@ +package linea.web3j + +import net.consensys.linea.web3j.okHttpClientBuilder +import org.apache.logging.log4j.Level +import org.apache.logging.log4j.Logger +import org.web3j.protocol.Web3j +import org.web3j.protocol.http.HttpService +import org.web3j.utils.Async +import java.util.concurrent.ScheduledExecutorService +import kotlin.time.Duration +import kotlin.time.Duration.Companion.milliseconds + +fun createWeb3jHttpClient( + rpcUrl: String, + log: Logger, + pollingInterval: Duration = 500.milliseconds, + executorService: ScheduledExecutorService = Async.defaultExecutorService(), + requestResponseLogLevel: Level = Level.TRACE, + failuresLogLevel: Level = Level.DEBUG +): Web3j { + return Web3j.build( + HttpService( + rpcUrl, + okHttpClientBuilder( + logger = log, + requestResponseLogLevel = requestResponseLogLevel, + failuresLogLevel = failuresLogLevel + ).build() + ), + pollingInterval.inWholeMilliseconds, + executorService + ) +} diff --git a/jvm-libs/linea/web3j-extensions/src/main/kotlin/net/consensys/linea/web3j/DomainObjectMappers.kt b/jvm-libs/linea/web3j-extensions/src/main/kotlin/net/consensys/linea/web3j/DomainObjectMappers.kt deleted file mode 100644 index 32f39d6dd..000000000 --- a/jvm-libs/linea/web3j-extensions/src/main/kotlin/net/consensys/linea/web3j/DomainObjectMappers.kt +++ /dev/null @@ -1,138 +0,0 @@ -package net.consensys.linea.web3j - -import net.consensys.linea.bigIntFromPrefixedHex -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.Logger -import org.apache.tuweni.bytes.Bytes -import org.apache.tuweni.bytes.Bytes32 -import org.apache.tuweni.units.bigints.UInt256 -import org.bouncycastle.math.ec.custom.sec.SecP256K1Curve -import org.hyperledger.besu.crypto.SECPSignature -import org.hyperledger.besu.datatypes.AccessListEntry -import org.hyperledger.besu.datatypes.Address -import org.hyperledger.besu.datatypes.Wei -import org.hyperledger.besu.ethereum.core.Transaction -import org.hyperledger.besu.ethereum.core.encoding.EncodingContext -import org.hyperledger.besu.ethereum.core.encoding.TransactionEncoder -import org.web3j.protocol.core.methods.response.EthBlock -import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1 -import tech.pegasys.teku.infrastructure.bytes.Bytes20 -import tech.pegasys.teku.infrastructure.unsigned.UInt64 -import java.math.BigInteger - -private val log: Logger = LogManager.getLogger("DomainObjectMappers") -fun EthBlock.Block.toExecutionPayloadV1(): ExecutionPayloadV1 { - /** - * @JsonProperty("parentHash") Bytes32 parentHash, - * @JsonProperty("feeRecipient") Bytes20 feeRecipient, - * @JsonProperty("stateRoot") Bytes32 stateRoot, - * @JsonProperty("receiptsRoot") Bytes32 receiptsRoot, - * @JsonProperty("logsBloom") Bytes logsBloom, - * @JsonProperty("prevRandao") Bytes32 prevRandao, - * @JsonProperty("blockNumber") UInt64 blockNumber, - * @JsonProperty("gasLimit") UInt64 gasLimit, - * @JsonProperty("gasUsed") UInt64 gasUsed, - * @JsonProperty("timestamp") UInt64 timestamp, - * @JsonProperty("extraData") Bytes extraData, - * @JsonProperty("baseFeePerGas") UInt256 baseFeePerGas, - * @JsonProperty("blockHash") Bytes32 blockHash, - * @JsonProperty("transactions") List transactions) - */ - return ExecutionPayloadV1( - Bytes32.fromHexString(this.parentHash), - Bytes20.fromHexString(this.miner), - Bytes32.fromHexString(this.stateRoot), - Bytes32.fromHexString(this.receiptsRoot), - Bytes.fromHexString(this.logsBloom), - Bytes32.fromHexString(this.mixHash), - UInt64.valueOf(this.number), - UInt64.valueOf(this.gasLimit), - UInt64.valueOf(this.gasUsed), - UInt64.valueOf(this.timestamp), - Bytes.fromHexString(this.extraData), - UInt256.valueOf(this.baseFeePerGas), - Bytes32.fromHexString(this.hash), - this.transactions.map { - val transaction = it.get() as EthBlock.TransactionObject - kotlin.runCatching { - transaction.toBytes() - }.onFailure { th -> - log.error( - "Failed to encode transaction! blockNumber={} tx={} errorMessage={}", - this.number, - transaction.hash.toString(), - th.message, - th - ) - } - .getOrThrow() - } - ) -} - -fun recIdFromV(v: BigInteger): Pair { - val recId: Byte - var chainId: BigInteger? = null - if (v == Transaction.REPLAY_UNPROTECTED_V_BASE || v == Transaction.REPLAY_UNPROTECTED_V_BASE_PLUS_1) { - recId = v.subtract(Transaction.REPLAY_UNPROTECTED_V_BASE).byteValueExact() - } else if (v > Transaction.REPLAY_PROTECTED_V_MIN) { - chainId = v.subtract(Transaction.REPLAY_PROTECTED_V_BASE).divide(Transaction.TWO) - recId = v.subtract(Transaction.TWO.multiply(chainId).add(Transaction.REPLAY_PROTECTED_V_BASE)).byteValueExact() - } else { - throw RuntimeException("An unsupported encoded `v` value of $v was found") - } - return Pair(recId, chainId) -} - -// TODO: Test -fun EthBlock.TransactionObject.toBytes(): Bytes { - val isFrontier = this.type == "0x0" - val (recId, chainId) = if (isFrontier) { - recIdFromV(this.v.toBigInteger()) - } else { - Pair(this.v.toByte(), BigInteger.valueOf(this.chainId)) - } - val signature = SECPSignature.create( - this.r.bigIntFromPrefixedHex(), - this.s.bigIntFromPrefixedHex(), - recId, - SecP256K1Curve().order - ) - - val transaction = Transaction.builder() - .nonce(this.nonce.toLong()) - .also { builder -> - if (isFrontier || this.type == "0x1") { - builder.gasPrice(Wei.of(this.gasPrice)) - } else { - builder.maxPriorityFeePerGas(Wei.of(this.maxPriorityFeePerGas)) - builder.maxFeePerGas(Wei.of(this.maxFeePerGas)) - } - } - .gasLimit(this.gas.toLong()) - .to(Address.fromHexString(this.to)) - .value(Wei.of(this.value)) - .signature(signature) - .payload(Bytes.fromHexString(this.input)) - .also { builder -> - this.accessList?.also { accessList -> - builder.accessList( - accessList.map { entry -> - AccessListEntry.createAccessListEntry( - Address.fromHexString(entry.address), - entry.storageKeys - ) - } - ) - } - } - .sender(Address.fromHexString(this.from)) - .apply { - if (chainId != null) { - chainId(chainId) - } - } - .build() - - return TransactionEncoder.encodeOpaqueBytes(transaction, EncodingContext.BLOCK_BODY) -} diff --git a/jvm-libs/linea/web3j-extensions/src/main/kotlin/net/consensys/linea/web3j/ExtendedWeb3J.kt b/jvm-libs/linea/web3j-extensions/src/main/kotlin/net/consensys/linea/web3j/ExtendedWeb3J.kt index 816139867..94ebab2e9 100644 --- a/jvm-libs/linea/web3j-extensions/src/main/kotlin/net/consensys/linea/web3j/ExtendedWeb3J.kt +++ b/jvm-libs/linea/web3j-extensions/src/main/kotlin/net/consensys/linea/web3j/ExtendedWeb3J.kt @@ -3,7 +3,6 @@ package net.consensys.linea.web3j import org.web3j.protocol.Web3j import org.web3j.protocol.core.DefaultBlockParameter import org.web3j.protocol.core.Response -import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1 import tech.pegasys.teku.infrastructure.async.SafeFuture import java.math.BigInteger @@ -14,7 +13,6 @@ import java.math.BigInteger interface ExtendedWeb3J { val web3jClient: Web3j fun ethBlockNumber(): SafeFuture - fun ethGetExecutionPayloadByNumber(blockNumber: Long): SafeFuture fun ethGetBlockTimestampByNumber(blockNumber: Long): SafeFuture } @@ -33,26 +31,6 @@ class ExtendedWeb3JImpl(override val web3jClient: Web3j) : ExtendedWeb3J { } } - override fun ethGetExecutionPayloadByNumber(blockNumber: Long): SafeFuture { - return SafeFuture.of( - web3jClient - .ethGetBlockByNumber( - DefaultBlockParameter.valueOf(BigInteger.valueOf(blockNumber)), - true - ) - .sendAsync() - ) - .thenCompose { response -> - if (response.hasError()) { - SafeFuture.failedFuture(buildException(response.error)) - } else { - response.block?.let { - SafeFuture.completedFuture(response.block.toExecutionPayloadV1()) - } ?: SafeFuture.failedFuture(Exception("Block $blockNumber not found!")) - } - } - } - override fun ethGetBlockTimestampByNumber( blockNumber: Long ): SafeFuture { diff --git a/jvm-libs/linea/web3j-extensions/src/test/kotlin/linea/web3j/EthGetBlockToLineaBlockMapperTest.kt b/jvm-libs/linea/web3j-extensions/src/test/kotlin/linea/web3j/EthGetBlockToLineaBlockMapperTest.kt new file mode 100644 index 000000000..7a6033017 --- /dev/null +++ b/jvm-libs/linea/web3j-extensions/src/test/kotlin/linea/web3j/EthGetBlockToLineaBlockMapperTest.kt @@ -0,0 +1,112 @@ +package linea.web3j + +import io.vertx.core.json.JsonObject +import linea.domain.Transaction +import linea.domain.TransactionType +import linea.domain.toBesu +import net.consensys.decodeHex +import net.consensys.toBigInteger +import net.consensys.toBigIntegerFromHex +import net.consensys.toLongFromHex +import org.apache.tuweni.bytes.Bytes +import org.assertj.core.api.Assertions.assertThat +import org.hyperledger.besu.datatypes.Address +import org.hyperledger.besu.datatypes.Wei +import org.junit.jupiter.api.Test +import org.web3j.protocol.core.methods.response.EthBlock +import kotlin.jvm.optionals.getOrNull + +class EthGetBlockToLineaBlockMapperTest { + + @Test + fun `should map frontier transactions`() { + val transactionJson = JsonObject( + // example of eth_getBlockByNumber.transactions[0] response + // This seems to be easier to read given than build Java object directly + """ + { + "blockHash": "0x004257e560a5f82595dddb73f752b904efef4b73cb3ece1469f5e5091e3c9665", + "blockNumber": "0xe1d30", + "chainId": "0xe705", + "from": "0x228466f2c715cbec05deabfac040ce3619d7cf0b", + "gas": "0x5208", + "gasPrice": "0xee2d984", + "hash": "0x5d3b5e1ae3e4ea5612e6907cb09c4e0e5482171b4c2af794e17b77314547bb79", + "input": "0x", + "nonce": "0x97411", + "r": "0xdf28597129341d5d345c9043c7d0b0a22be82cac13988cfc1d8cbdaf3ab3f35b", + "s": "0x3189b2ff80d8f728d6fb7503b46734ee77a60a42db01d0b09db10bdc9d5caa44", + "to": "0x228466f2c715cbec05deabfac040ce3619d7cf0b", + "transactionIndex": "0x0", + "type": "0x0", + "v": "0x1ce2e", + "value": "0x186a0" + } + """.trimIndent() + ) + val web3jTx = EthBlock.TransactionObject( + /*hash*/ transactionJson.getString("hash"), + /*nonce*/ transactionJson.getString("nonce"), + /*blockHash*/ transactionJson.getString("blockHash"), + /*blockNumber*/ transactionJson.getString("blockNumber"), + /*chainId*/ transactionJson.getString("chainId"), + /*transactionIndex*/ transactionJson.getString("transactionIndex"), + /*from*/ transactionJson.getString("from"), + /*to*/ transactionJson.getString("to"), + /*value*/ transactionJson.getString("value"), + /*gasPrice*/ transactionJson.getString("gasPrice"), + /*gas*/ transactionJson.getString("gas"), + /*input*/ transactionJson.getString("input"), + /*creates*/ null, + /*publicKey*/null, + /*raw*/null, + /*r*/ transactionJson.getString("r"), + /*s*/ transactionJson.getString("s"), + /*v*/ transactionJson.getString("v").toLongFromHex(), + /*yParity*/null, + /*type*/ transactionJson.getString("type"), + /*maxFeePerGas*/null, + /*maxPriorityFeePerGas*/ null, + /*> accessList*/emptyList() + ) + + val domainTx = web3jTx.toDomain() + assertThat(domainTx).isEqualTo( + Transaction( + nonce = 0x97411UL, + gasPrice = 0xee2d984UL, + gasLimit = 0x5208UL, + to = "0x228466f2c715cbec05deabfac040ce3619d7cf0b".decodeHex(), + value = 0x186a0UL.toBigInteger(), + input = "0x".decodeHex(), + r = "0xdf28597129341d5d345c9043c7d0b0a22be82cac13988cfc1d8cbdaf3ab3f35b".toBigIntegerFromHex(), + s = "0x3189b2ff80d8f728d6fb7503b46734ee77a60a42db01d0b09db10bdc9d5caa44".toBigIntegerFromHex(), + v = 118318UL, + yParity = null, + type = TransactionType.FRONTIER, + chainId = 0xe705UL, + maxFeePerGas = null, + maxPriorityFeePerGas = null + ) + ) + domainTx.toBesu().also { besuTx -> + assertThat(besuTx.nonce).isEqualTo(0x97411L) + assertThat(besuTx.gasPrice.getOrNull()).isEqualTo(Wei.of(0xee2d984L)) + assertThat(besuTx.gasLimit).isEqualTo(0x5208L) + assertThat(besuTx.to.getOrNull()).isEqualTo(Address.fromHexString("0x228466f2c715cbec05deabfac040ce3619d7cf0b")) + assertThat(besuTx.value).isEqualTo(Wei.of(0x186a0L)) + assertThat(besuTx.payload).isEqualTo(Bytes.EMPTY) + assertThat(besuTx.signature.r).isEqualTo( + "0xdf28597129341d5d345c9043c7d0b0a22be82cac13988cfc1d8cbdaf3ab3f35b".toBigIntegerFromHex() + ) + assertThat(besuTx.signature.s).isEqualTo( + "0x3189b2ff80d8f728d6fb7503b46734ee77a60a42db01d0b09db10bdc9d5caa44".toBigIntegerFromHex() + ) + assertThat(besuTx.signature.recId).isEqualTo(1) + assertThat(besuTx.type).isEqualTo(org.hyperledger.besu.datatypes.TransactionType.FRONTIER) + assertThat(besuTx.chainId.getOrNull()).isEqualTo(0xe705L) + assertThat(besuTx.maxFeePerGas).isEmpty() + assertThat(besuTx.maxPriorityFeePerGas).isEmpty() + } + } +} diff --git a/jvm-libs/linea/web3j-extensions/src/test/kotlin/net/consensys/linea/web3j/DomainObjectMappersTest.kt b/jvm-libs/linea/web3j-extensions/src/test/kotlin/net/consensys/linea/web3j/DomainObjectMappersTest.kt index 7a7f4a180..14cd03c02 100644 --- a/jvm-libs/linea/web3j-extensions/src/test/kotlin/net/consensys/linea/web3j/DomainObjectMappersTest.kt +++ b/jvm-libs/linea/web3j-extensions/src/test/kotlin/net/consensys/linea/web3j/DomainObjectMappersTest.kt @@ -1,5 +1,6 @@ package net.consensys.linea.web3j +/* import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.web3j.protocol.core.methods.response.AccessListObject @@ -219,3 +220,4 @@ class DomainObjectMappersTest { assertThat(encodedTransaction.toString()).isEqualTo(signedContractCreation) } } +*/ diff --git a/settings.gradle b/settings.gradle index fa163a800..6a3fb0685 100644 --- a/settings.gradle +++ b/settings.gradle @@ -26,10 +26,9 @@ include 'jvm-libs:linea:core:traces' include 'jvm-libs:linea:linea-contracts:l1-rollup' include 'jvm-libs:linea:linea-contracts:l2-message-service' include 'jvm-libs:linea:metrics:micrometer' -include 'jvm-libs:linea:teku-execution-client' +include 'jvm-libs:linea:besu-rlp-and-mappers' include 'jvm-libs:linea:testing:file-system' include 'jvm-libs:linea:testing:l1-blob-and-proof-submission' -include 'jvm-libs:linea:testing:teku-helper' include 'jvm-libs:linea:web3j-extensions' include 'coordinator:app' diff --git a/transaction-decoder-tool/build.gradle b/transaction-decoder-tool/build.gradle index b364298e3..6346723aa 100644 --- a/transaction-decoder-tool/build.gradle +++ b/transaction-decoder-tool/build.gradle @@ -3,6 +3,6 @@ plugins { } dependencies { - implementation project(":jvm-libs:linea:teku-execution-client") + implementation project(':jvm-libs:linea:core:domain-models') implementation project(':jvm-libs:linea:web3j-extensions') } diff --git a/transaction-decoder-tool/src/main/kotlin/net/consensys/linea/BlockReader.kt b/transaction-decoder-tool/src/main/kotlin/net/consensys/linea/BlockReader.kt deleted file mode 100644 index ff5032b07..000000000 --- a/transaction-decoder-tool/src/main/kotlin/net/consensys/linea/BlockReader.kt +++ /dev/null @@ -1,23 +0,0 @@ -package net.consensys.linea - -import net.consensys.linea.web3j.ExtendedWeb3JImpl -import org.web3j.protocol.Web3j -import org.web3j.protocol.http.HttpService -import org.web3j.utils.Async -import tech.pegasys.teku.ethereum.executionclient.schema.ExecutionPayloadV1 -import tech.pegasys.teku.infrastructure.async.SafeFuture - -class BlockReader { - private val web3jClient: Web3j = Web3j.build( - HttpService("https://linea-sepolia.infura.io/v3/"), - 1000, - Async.defaultExecutorService() - ) - - private val asyncWeb3J = ExtendedWeb3JImpl(web3jClient) - - fun getBlockPayload(blockNumber: Long): SafeFuture { - val encodedPayload = asyncWeb3J.ethGetExecutionPayloadByNumber(blockNumber) - return encodedPayload - } -} diff --git a/transaction-decoder-tool/src/main/kotlin/net/consensys/linea/TransactionEncodingToolMain.kt b/transaction-decoder-tool/src/main/kotlin/net/consensys/linea/TransactionEncodingToolMain.kt index d4e02ea65..ae3a3de47 100644 --- a/transaction-decoder-tool/src/main/kotlin/net/consensys/linea/TransactionEncodingToolMain.kt +++ b/transaction-decoder-tool/src/main/kotlin/net/consensys/linea/TransactionEncodingToolMain.kt @@ -1,6 +1,12 @@ package net.consensys.linea +import build.linea.web3j.domain.toWeb3j +import linea.web3j.toDomain +import net.consensys.linea.BlockParameter.Companion.toBlockParameter import org.apache.logging.log4j.LogManager +import org.web3j.protocol.Web3j +import org.web3j.protocol.http.HttpService +import org.web3j.utils.Async class TransactionEncodingToolMain { @@ -13,8 +19,12 @@ class TransactionEncodingToolMain { } private fun startApp() { - val app = BlockReader() - app.getBlockPayload(924973) + val web3j: Web3j = Web3j.build( + HttpService("https://linea-sepolia.infura.io/v3/"), + 1000, + Async.defaultExecutorService() + ) + web3j.ethGetBlockByNumber(924973UL.toBlockParameter().toWeb3j(), true).sendAsync().get().block.toDomain() } } } From 1082b74b2c47384a6334902f76dd997ad996b728 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Mon, 9 Dec 2024 21:06:47 +0000 Subject: [PATCH 02/31] coordinator: fix tx mappings --- .../linea/domain/MapperBesuToLineaDomain.kt | 8 +- .../linea/domain/MapperLineaDomainToBesu.kt | 32 +- .../main/kotlin/linea/domain/Transaction.kt | 15 +- .../web3j/EthGetBlockToLineaBlockMappers.kt | 11 +- .../EthGetBlockToLineaBlockMapperTest.kt | 432 +++++++++++++++++- 5 files changed, 485 insertions(+), 13 deletions(-) diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperBesuToLineaDomain.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperBesuToLineaDomain.kt index ae3dd619a..fd81a61a8 100644 --- a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperBesuToLineaDomain.kt +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperBesuToLineaDomain.kt @@ -51,7 +51,13 @@ object MapperBesuToLineaDomain { type = transaction.type.toDomain(), chainId = transaction.chainId?.getOrNull()?.toULong(), maxFeePerGas = transaction.maxFeePerGas?.getOrNull()?.toBigInteger()?.toULong(), - maxPriorityFeePerGas = transaction.maxPriorityFeePerGas?.getOrNull()?.toBigInteger()?.toULong() + maxPriorityFeePerGas = transaction.maxPriorityFeePerGas?.getOrNull()?.toBigInteger()?.toULong(), + accessList = transaction.accessList.getOrNull()?.map { accessListEntry -> + AccessListEntry( + accessListEntry.address.toArray(), + accessListEntry.storageKeys.map { it.toArray() } + ) + } ) } } diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperLineaDomainToBesu.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperLineaDomainToBesu.kt index 6583ec492..e5669f0b9 100644 --- a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperLineaDomainToBesu.kt +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperLineaDomainToBesu.kt @@ -69,7 +69,10 @@ object MapperLineaDomainToBesu { .blockHeaderFunctions(blockHeaderFunctions) .buildBlockHeader() - val transactions = block.transactions.map(MapperLineaDomainToBesu::mapToBesu) + val transactions = + block.transactions.mapIndexed { index, transaction -> + mapToBesu(block.number, index, transaction) + } // linea does not support uncles, so we are not converting them // throwing an exception just in case we get one and we can fix it if (block.ommers.isNotEmpty()) { @@ -79,11 +82,25 @@ object MapperLineaDomainToBesu { val body = BlockBody(transactions, emptyList()) return org.hyperledger.besu.ethereum.core.Block(header, body) - }.getOrElse { - throw IllegalStateException("Error mapping block to Besu: block=${block.number}", it) + }.getOrElse { th -> + if (th.message?.startsWith("Error mapping transaction to Besu") ?: false) { + throw th + } else { + throw RuntimeException("Error mapping block to Besu: block=${block.number}", th) + } } } + fun mapToBesu(blockNumber: ULong, txIndex: Int, tx: linea.domain.Transaction): Transaction { + return runCatching { mapToBesu(tx) } + .getOrElse { th -> + throw RuntimeException( + "Error mapping transaction to Besu: block=$blockNumber txIndex=$txIndex transaction=$tx", + th + ) + } + } + fun mapToBesu(tx: linea.domain.Transaction): Transaction { val (recId, chainId) = getRecIdAndChainId(tx) val signature = secp256k1.createSignature( @@ -92,6 +109,8 @@ object MapperLineaDomainToBesu { recId ) + val besuType = tx.type.toBesu() + return Transaction.builder() .type(tx.type.toBesu()) .nonce(tx.nonce.toLong()) @@ -104,13 +123,14 @@ object MapperLineaDomainToBesu { .maxPriorityFeePerGas(tx.maxPriorityFeePerGas?.toWei()) .maxFeePerGas(tx.maxFeePerGas?.toWei()) .apply { - if (!tx.accessList.isEmpty()) { - tx.accessList.map { entry -> + if (besuType.supportsAccessList()) { + val accList = tx.accessList?.map { entry -> AccessListEntry( Address.wrap(Bytes.wrap(entry.address)), entry.storageKeys.map { Bytes32.wrap(it) } ) - } + } ?: emptyList() + accessList(accList) } } .signature(signature) diff --git a/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/Transaction.kt b/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/Transaction.kt index 75f8652c5..51e15c21c 100644 --- a/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/Transaction.kt +++ b/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/Transaction.kt @@ -61,7 +61,7 @@ data class Transaction( val chainId: ULong? = null, // Optional field for EIP-155 transactions val maxPriorityFeePerGas: ULong? = null, // null for non EIP-1559 transactions val maxFeePerGas: ULong? = null, // null for EIP-1559 transactions - val accessList: List = emptyList() // null non for EIP-2930 transactions + val accessList: List? // null non for EIP-2930 transactions ) { companion object { // companion object to allow static extension functions @@ -138,6 +138,14 @@ data class AccessListEntry( val address: ByteArray, val storageKeys: List ) { + + override fun toString(): String { + return "AccessListEntry(" + + "address=${address.encodeHex()}, " + + "storageKeys=[${storageKeys.joinToString(",") { it.encodeHex() }}]" + + ")" + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (javaClass != other?.javaClass) return false @@ -145,7 +153,10 @@ data class AccessListEntry( other as AccessListEntry if (!address.contentEquals(other.address)) return false - if (storageKeys != other.storageKeys) return false + if (storageKeys.size != other.storageKeys.size) return false + storageKeys.zip(other.storageKeys).forEach { (a, b) -> + if (!a.contentEquals(b)) return false + } return true } diff --git a/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/EthGetBlockToLineaBlockMappers.kt b/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/EthGetBlockToLineaBlockMappers.kt index 99b799ea1..01987c87c 100644 --- a/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/EthGetBlockToLineaBlockMappers.kt +++ b/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/EthGetBlockToLineaBlockMappers.kt @@ -1,5 +1,6 @@ package linea.web3j +import linea.domain.AccessListEntry import linea.domain.Block import linea.domain.Transaction import linea.domain.TransactionType @@ -49,6 +50,13 @@ fun EthBlock.TransactionObject.toDomain(): Transaction { // Web3J throws an exception if maxPriorityFeePerGas null, instead of a check like in maxFeePerGas // we need to check if maxFeePerGas is null to avoid the exception val maxPriorityFeePerGas = if (maxFeePerGas != null) this.maxPriorityFeePerGas?.toULong() else null + val accessList = this.accessList?.map { accessListEntry -> + AccessListEntry( + accessListEntry.address.decodeHex(), + accessListEntry.storageKeys.map { it.decodeHex() } + ) + } + return Transaction( nonce = this.nonce.toULong(), gasPrice = this.gasPrice.toULong(), @@ -63,7 +71,8 @@ fun EthBlock.TransactionObject.toDomain(): Transaction { type = mapType(this.type), // Optional field for EIP-2718 typed transactions chainId = this.chainId?.toULong(), // Optional field for EIP-155 transactions maxFeePerGas = maxFeePerGas, // Optional field for EIP-1559 transactions - maxPriorityFeePerGas = maxPriorityFeePerGas // Optional field for EIP-1559 transactions + maxPriorityFeePerGas = maxPriorityFeePerGas, // Optional field for EIP-1559 transactions, + accessList = accessList ) } diff --git a/jvm-libs/linea/web3j-extensions/src/test/kotlin/linea/web3j/EthGetBlockToLineaBlockMapperTest.kt b/jvm-libs/linea/web3j-extensions/src/test/kotlin/linea/web3j/EthGetBlockToLineaBlockMapperTest.kt index 7a6033017..ec0cc3f1c 100644 --- a/jvm-libs/linea/web3j-extensions/src/test/kotlin/linea/web3j/EthGetBlockToLineaBlockMapperTest.kt +++ b/jvm-libs/linea/web3j-extensions/src/test/kotlin/linea/web3j/EthGetBlockToLineaBlockMapperTest.kt @@ -1,6 +1,7 @@ package linea.web3j import io.vertx.core.json.JsonObject +import linea.domain.AccessListEntry import linea.domain.Transaction import linea.domain.TransactionType import linea.domain.toBesu @@ -9,11 +10,15 @@ import net.consensys.toBigInteger import net.consensys.toBigIntegerFromHex import net.consensys.toLongFromHex import org.apache.tuweni.bytes.Bytes +import org.apache.tuweni.bytes.Bytes32 import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.fail import org.hyperledger.besu.datatypes.Address import org.hyperledger.besu.datatypes.Wei import org.junit.jupiter.api.Test +import org.web3j.protocol.core.methods.response.AccessListObject import org.web3j.protocol.core.methods.response.EthBlock +import kotlin.jvm.optionals.getOrElse import kotlin.jvm.optionals.getOrNull class EthGetBlockToLineaBlockMapperTest { @@ -67,7 +72,7 @@ class EthGetBlockToLineaBlockMapperTest { /*type*/ transactionJson.getString("type"), /*maxFeePerGas*/null, /*maxPriorityFeePerGas*/ null, - /*> accessList*/emptyList() + /*> accessList*/null ) val domainTx = web3jTx.toDomain() @@ -86,10 +91,12 @@ class EthGetBlockToLineaBlockMapperTest { type = TransactionType.FRONTIER, chainId = 0xe705UL, maxFeePerGas = null, - maxPriorityFeePerGas = null + maxPriorityFeePerGas = null, + accessList = null ) ) domainTx.toBesu().also { besuTx -> + assertThat(besuTx.type).isEqualTo(org.hyperledger.besu.datatypes.TransactionType.FRONTIER) assertThat(besuTx.nonce).isEqualTo(0x97411L) assertThat(besuTx.gasPrice.getOrNull()).isEqualTo(Wei.of(0xee2d984L)) assertThat(besuTx.gasLimit).isEqualTo(0x5208L) @@ -103,10 +110,429 @@ class EthGetBlockToLineaBlockMapperTest { "0x3189b2ff80d8f728d6fb7503b46734ee77a60a42db01d0b09db10bdc9d5caa44".toBigIntegerFromHex() ) assertThat(besuTx.signature.recId).isEqualTo(1) - assertThat(besuTx.type).isEqualTo(org.hyperledger.besu.datatypes.TransactionType.FRONTIER) assertThat(besuTx.chainId.getOrNull()).isEqualTo(0xe705L) assertThat(besuTx.maxFeePerGas).isEmpty() assertThat(besuTx.maxPriorityFeePerGas).isEmpty() } } + + @Test + fun `should map transaction with AccessList`() { + val txJson = + """ + { + "accessList": [ + { + "address": "0x8d97689c9818892b700e27f316cc3e41e17fbeb9", + "storageKeys": [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0x0000000000000000000000000000000000000000000000000000000000000001" + ] + } + ], + "blockHash": "0x7480ae911853c1fba10145401a21ddca3943b5894d74cbbf7a6beec526d1f9c2", + "blockNumber": "0xa", + "chainId": "0x539", + "from": "0xce3b7d471fd1fdd10d788ae64e48a9c2f2361179", + "gas": "0x30d40", + "gasPrice": "0x1017df87", + "hash": "0x8ef620582ed8ba98c8496a42b27a30ff7b1de901b1ff7e65b22ea59a2d0668ce", + "input": "0x", + "nonce": "0x0", + "to": "0x8d97689c9818892b700e27f316cc3e41e17fbeb9", + "transactionIndex": "0x2", + "type": "0x1", + "value": "0x2386f26fc10000", + "yParity": "0x1", + "v": "0x1", + "r": "0x4f24ed24207bec8591c8172584dc3b57cdf3ee96afbd5e63905a90a704ff33f0", + "s": "0x6277bb9d2614843a4791ff2c192e70876438ec940c39d92deb504591b83dfeb3" + } + """.trimIndent().let { JsonObject(it) } + + val txWeb3J = EthBlock.TransactionObject( + /*hash*/ txJson.getString("hash"), + /*nonce*/ txJson.getString("nonce"), + /*blockHash*/ txJson.getString("blockHash"), + /*blockNumber*/ txJson.getString("blockNumber"), + /*chainId*/ txJson.getString("chainId"), + /*transactionIndex*/ txJson.getString("transactionIndex"), + /*from*/ txJson.getString("from"), + /*to*/ txJson.getString("to"), + /*value*/ txJson.getString("value"), + /*gasPrice*/ txJson.getString("gasPrice"), + /*gas*/ txJson.getString("gas"), + /*input*/ txJson.getString("input"), + /*creates*/ null, + /*publicKey*/null, + /*raw*/null, + /*r*/ txJson.getString("r"), + /*s*/ txJson.getString("s"), + /*v*/ txJson.getString("v").toLongFromHex(), + /*yParity*/ txJson.getString("yParity"), + /*type*/ txJson.getString("type"), + /*maxFeePerGas*/null, + /*maxPriorityFeePerGas*/ null, + /* accessList*/txJson.getJsonArray("accessList").map { accessListEntry -> + val accessList = accessListEntry as JsonObject + AccessListObject( + /*address*/ accessList.getString("address"), + /*storageKeys*/ accessList.getJsonArray("storageKeys").map { it as String } + ) + } + ) + + val domainTx = txWeb3J.toDomain() + assertThat(domainTx).isEqualTo( + Transaction( + nonce = 0UL, + gasPrice = 0x1017df87UL, + gasLimit = 0x30d40UL, + to = "0x8d97689c9818892b700e27f316cc3e41e17fbeb9".decodeHex(), + value = 0x2386f26fc10000UL.toBigInteger(), + input = "0x".decodeHex(), + r = "0x4f24ed24207bec8591c8172584dc3b57cdf3ee96afbd5e63905a90a704ff33f0".toBigIntegerFromHex(), + s = "0x6277bb9d2614843a4791ff2c192e70876438ec940c39d92deb504591b83dfeb3".toBigIntegerFromHex(), + v = 1UL, + yParity = 1UL, + type = TransactionType.ACCESS_LIST, + chainId = 0x539UL, + maxFeePerGas = null, + maxPriorityFeePerGas = null, + accessList = listOf( + AccessListEntry( + address = "0x8d97689c9818892b700e27f316cc3e41e17fbeb9".decodeHex(), + listOf( + "0x0000000000000000000000000000000000000000000000000000000000000000".decodeHex(), + "0x0000000000000000000000000000000000000000000000000000000000000001".decodeHex() + ) + ) + ) + ) + ) + + domainTx.toBesu().also { besuTx -> + assertThat(besuTx.nonce).isEqualTo(0L) + assertThat(besuTx.gasPrice.getOrNull()).isEqualTo(Wei.of(0x1017df87L)) + assertThat(besuTx.gasLimit).isEqualTo(0x30d40L) + assertThat(besuTx.to.getOrNull()).isEqualTo(Address.fromHexString("0x8d97689c9818892b700e27f316cc3e41e17fbeb9")) + assertThat(besuTx.value).isEqualTo(Wei.of(0x2386f26fc10000L)) + assertThat(besuTx.payload).isEqualTo(Bytes.EMPTY) + assertThat(besuTx.signature.r).isEqualTo( + "0x4f24ed24207bec8591c8172584dc3b57cdf3ee96afbd5e63905a90a704ff33f0".toBigIntegerFromHex() + ) + assertThat(besuTx.signature.s).isEqualTo( + "0x6277bb9d2614843a4791ff2c192e70876438ec940c39d92deb504591b83dfeb3".toBigIntegerFromHex() + ) + assertThat(besuTx.signature.recId).isEqualTo(1) + assertThat(besuTx.type).isEqualTo(org.hyperledger.besu.datatypes.TransactionType.ACCESS_LIST) + assertThat(besuTx.chainId.getOrNull()).isEqualTo(0x539L) + assertThat(besuTx.maxFeePerGas).isEmpty() + assertThat(besuTx.maxPriorityFeePerGas).isEmpty() + val accessList = besuTx.accessList.getOrElse { fail("AccessList is empty") } + + assertThat(accessList.get(0).address) + .isEqualTo(Address.fromHexString("0x8d97689c9818892b700e27f316cc3e41e17fbeb9")) + assertThat(accessList.get(0).storageKeys) + .containsExactly( + Bytes32.fromHexString("0x0000000000000000000000000000000000000000000000000000000000000000"), + Bytes32.fromHexString("0x0000000000000000000000000000000000000000000000000000000000000001") + ) + } + } + + @Test + fun `should map type accessList with empty list`() { + val txJson = + """ + { + "accessList": [], + "blockHash": "0x7480ae911853c1fba10145401a21ddca3943b5894d74cbbf7a6beec526d1f9c2", + "blockNumber": "0xa", + "chainId": "0x539", + "from": "0x5007b0259849a673d0d780611f9a2ed8821d9ebe", + "gas": "0x5208", + "gasPrice": "0x1017df87", + "hash": "0xa2334c8858bb44ef3e9ef7f3523ec058ab24a869cfad7333fdf7bf3bb76deec4", + "input": "0x", + "nonce": "0x0", + "to": "0x8d97689c9818892b700e27f316cc3e41e17fbeb9", + "transactionIndex": "0x3", + "type": "0x1", + "value": "0x2386f26fc10000", + "yParity": "0x1", + "v": "0x1", + "r": "0xc57273f9ba15320937d5d9dfd1dc0b18d1e678b34bd3a4bfd29a63e11a856292", + "s": "0x7aa875a64835ecc5f9ac1a9fe3ab38d2a62bb3643a2597ab585a5607641a0c57" + } + """.trimIndent().let { JsonObject(it) } + val txWeb3J = EthBlock.TransactionObject( + /*hash*/ txJson.getString("hash"), + /*nonce*/ txJson.getString("nonce"), + /*blockHash*/ txJson.getString("blockHash"), + /*blockNumber*/ txJson.getString("blockNumber"), + /*chainId*/ txJson.getString("chainId"), + /*transactionIndex*/ txJson.getString("transactionIndex"), + /*from*/ txJson.getString("from"), + /*to*/ txJson.getString("to"), + /*value*/ txJson.getString("value"), + /*gasPrice*/ txJson.getString("gasPrice"), + /*gas*/ txJson.getString("gas"), + /*input*/ txJson.getString("input"), + /*creates*/ null, + /*publicKey*/null, + /*raw*/null, + /*r*/ txJson.getString("r"), + /*s*/ txJson.getString("s"), + /*v*/ txJson.getString("v").toLongFromHex(), + /*yParity*/ txJson.getString("yParity"), + /*type*/ txJson.getString("type"), + /*maxFeePerGas*/null, + /*maxPriorityFeePerGas*/ null, + /* accessList*/ emptyList() + ) + + val domainTx = txWeb3J.toDomain() + assertThat(domainTx).isEqualTo( + Transaction( + nonce = 0UL, + gasPrice = 0x1017df87UL, + gasLimit = 0x5208UL, + to = "0x8d97689c9818892b700e27f316cc3e41e17fbeb9".decodeHex(), + value = 0x2386f26fc10000UL.toBigInteger(), + input = "0x".decodeHex(), + r = "0xc57273f9ba15320937d5d9dfd1dc0b18d1e678b34bd3a4bfd29a63e11a856292".toBigIntegerFromHex(), + s = "0x7aa875a64835ecc5f9ac1a9fe3ab38d2a62bb3643a2597ab585a5607641a0c57".toBigIntegerFromHex(), + v = 1UL, + yParity = 1UL, + type = TransactionType.ACCESS_LIST, + chainId = 0x539UL, + maxFeePerGas = null, + maxPriorityFeePerGas = null, + accessList = emptyList() + ) + ) + + domainTx.toBesu().also { besuTx -> + assertThat(besuTx.type).isEqualTo(org.hyperledger.besu.datatypes.TransactionType.ACCESS_LIST) + assertThat(besuTx.nonce).isEqualTo(0L) + assertThat(besuTx.gasPrice.getOrNull()).isEqualTo(Wei.of(0x1017df87L)) + assertThat(besuTx.gasLimit).isEqualTo(0x5208L) + assertThat(besuTx.to.getOrNull()).isEqualTo(Address.fromHexString("0x8d97689c9818892b700e27f316cc3e41e17fbeb9")) + assertThat(besuTx.value).isEqualTo(Wei.of(0x2386f26fc10000L)) + assertThat(besuTx.payload).isEqualTo(Bytes.EMPTY) + assertThat(besuTx.signature.r).isEqualTo( + "0xc57273f9ba15320937d5d9dfd1dc0b18d1e678b34bd3a4bfd29a63e11a856292".toBigIntegerFromHex() + ) + assertThat(besuTx.signature.s).isEqualTo( + "0x7aa875a64835ecc5f9ac1a9fe3ab38d2a62bb3643a2597ab585a5607641a0c57".toBigIntegerFromHex() + ) + assertThat(besuTx.signature.recId).isEqualTo(1) + assertThat(besuTx.chainId.getOrNull()).isEqualTo(0x539L) + assertThat(besuTx.maxFeePerGas).isEmpty() + assertThat(besuTx.maxPriorityFeePerGas).isEmpty() + // it shall have an empty accessList + assertThat(besuTx.accessList.getOrNull()).isEmpty() + } + } + + @Test + fun `it should map EIP1559 tx`() { + val input = + """ + 0xdeb3cdf2000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000003e84d538bf9753309729adf92867d75bc2e58b566cb204b0d1b018fbf311e4b49bc52c1c450b2f412f5d9a44e01990d6127bc805ba8ef079ed2a897070378d706fbd2f5cf52b0e172541b7f11b9c2f0e0b91a67c5caf5908ddfdd2d340349e7398b698b3c336876c88e232f0e8f3197f2683e54a4439abb7d210d84cc1d3ad1bd48d0ba5dc30714d253743a734f17e88354eea550f7945df35d4c6316fcfaad09846f81f59b8127b037dbce6e5ffa45120fc69e6852f0c8ac2fcc9fd5e72503dd1c8d114ee079ffed84ddd6851e438cdd2a0ed1df9f0255481dc2a61a4808d856525619c948fbbd063bfd3db42504547db68c29990540eb7a36a1a8a0e483544eb634ae33f43f5bac2d991b9f6b36e23a7a299ade5b30ab96ad6dae27a9c374ae5f702fc689f596450c467722f24b7621ba5663ed6e08b620f04bc524338cac50e9ebc302d0b33dd9e2e563f05ce26303666a6c8c0f8dcd0b475f2219398ff4552533d28660c8d0d2843ce238ffb856dee06bc28e1ec0b92c3cb7b91378c07b049f3af20017dbdfdf48320ea7cf5f331bee27ba33d6a41351b3f044612a45f51451c068c23d6aec6784f623c6855acb95f07f213ad8605861fd8601ffa9a0282508a4d859769cb61247389020587a570cba1eac8b05576bd5b7a81b166f3c7b0f0ae0a8117f642d3fd0957e1cface4d10ebe6475a9a1f3bf6e3b1b7c16e50e529adcf0cf278aa64b9fecedcb0d894ab7ba6589e96ff56dff7b8636413f46cdc073c22521f6b89d7b68ca6f8af1ecc4e453137c801a9b35b5c4869883f59aaeaa7ee637d71c7e02f08894cffdb51a368b225fa1ab00e3ad2d91d1275d048aad5eb5d34438622c7aea1759b3fe747c2b0fddd62159de1d7cabbccde9c1e3511a34432e0c4e6dede019e38493fc29292ea321621629c1ffc62160747ee136171c96f55af7af6a29b8ca94f12d12b7c706974b1e586b3674a6aec7510f1025ba399f7a97f5911187b040b7a494e191bd761ad2a78daf427f5ee19cf24bc45fab34d32747de0f0a2c6bd33d2440d9f5d20da22da34e418d54d6894d42edd6d0c5a4f9b02d510a23db40cc455b7c423bd43b6fcd0f3655285e16ba8d9bdaf3f2147de572c33568353b5f5dd820f49dbddbc63297aed5e2f342b383a83319f9beed9d3d358a3dc7c0a010b85954fec3b34c3227a9b4447bd5d30b8b78c0ef36cd8197e867d37778b24e8ebfaadf08c42f3db6e5e46cb025bb4e98334ce0a7a59ba155eaf3968621f353d075e0d68f0787259e344a72e8938ebe3a81458ad20df917dc1392fe759210f045f7d87177ad39a13ec704301f1f0845b8c6cbc52f8f77c043bcc80adb513470d0e5a6b02df65259bcc3198efe01c555a5d28bf89f818ea1b984a64db220f487e230652000000000000000000000000000000000000000000000000 + """.trimIndent().trim() + val txJson = + """ + { + "accessList": [], + "blockHash": "0x7480ae911853c1fba10145401a21ddca3943b5894d74cbbf7a6beec526d1f9c2", + "blockNumber": "0xa", + "chainId": "0x539", + "from": "0x5007b0259849a673d0d780611f9a2ed8821d9ebe", + "gas": "0x5208", + "gasPrice": "0x1017df87", + "hash": "0xa2334c8858bb44ef3e9ef7f3523ec058ab24a869cfad7333fdf7bf3bb76deec4", + "input": "$input", + "nonce": "0x0", + "to": "0xe4392c8ecc46b304c83cdb5edaf742899b1bda93", + "transactionIndex": "0x3", + "type": "0x2", + "value": "0x2386f26fc10000", + "yParity": "0x1", + "v": "0x1", + "r": "0xeb4f70991ea4f14d23efb32591da3621d551406fd32bdfdd78bb677dec13160a", + "s": "0x783aaa89f73ef7535924da8fd5f12e15cae1a0811c4c4746d1c23abff1eacddf", + "maxFeePerGas": "0x1017df87", + "maxPriorityFeePerGas": "0x1017df87" + } + """.trimIndent().let { JsonObject(it) } + val txWeb3J = EthBlock.TransactionObject( + /*hash*/ txJson.getString("hash"), + /*nonce*/ txJson.getString("nonce"), + /*blockHash*/ txJson.getString("blockHash"), + /*blockNumber*/ txJson.getString("blockNumber"), + /*chainId*/ txJson.getString("chainId"), + /*transactionIndex*/ txJson.getString("transactionIndex"), + /*from*/ txJson.getString("from"), + /*to*/ txJson.getString("to"), + /*value*/ txJson.getString("value"), + /*gasPrice*/ txJson.getString("gasPrice"), + /*gas*/ txJson.getString("gas"), + /*input*/ txJson.getString("input"), + /*creates*/ null, + /*publicKey*/null, + /*raw*/null, + /*r*/ txJson.getString("r"), + /*s*/ txJson.getString("s"), + /*v*/ txJson.getString("v").toLongFromHex(), + /*yParity*/ txJson.getString("yParity"), + /*type*/ txJson.getString("type"), + /*maxFeePerGas*/ txJson.getString("maxFeePerGas"), + /*maxPriorityFeePerGas*/ txJson.getString("maxPriorityFeePerGas"), + /* accessList*/ emptyList() + ) + + val txDomain = txWeb3J.toDomain() + assertThat(txWeb3J.toDomain()).isEqualTo( + Transaction( + nonce = 0UL, + gasPrice = 0x1017df87UL, + gasLimit = 0x5208UL, + to = "0xe4392c8ecc46b304c83cdb5edaf742899b1bda93".decodeHex(), + value = 0x2386f26fc10000UL.toBigInteger(), + input = input.decodeHex(), + r = "0xeb4f70991ea4f14d23efb32591da3621d551406fd32bdfdd78bb677dec13160a".toBigIntegerFromHex(), + s = "0x783aaa89f73ef7535924da8fd5f12e15cae1a0811c4c4746d1c23abff1eacddf".toBigIntegerFromHex(), + v = 1UL, + yParity = 1UL, + type = TransactionType.EIP1559, + chainId = 0x539UL, + maxFeePerGas = 0x1017df87UL, + maxPriorityFeePerGas = 0x1017df87UL, + accessList = emptyList() + ) + ) + + txDomain.toBesu().also { txBesu -> + assertThat(txBesu.type).isEqualTo(org.hyperledger.besu.datatypes.TransactionType.EIP1559) + assertThat(txBesu.nonce).isEqualTo(0L) + assertThat(txBesu.gasPrice.getOrNull()).isEqualTo(Wei.of(0x1017df87L)) + assertThat(txBesu.gasLimit).isEqualTo(0x5208L) + assertThat(txBesu.to.getOrNull()).isEqualTo(Address.fromHexString("0xe4392c8ecc46b304c83cdb5edaf742899b1bda93")) + assertThat(txBesu.value).isEqualTo(Wei.of(0x2386f26fc10000L)) + assertThat(txBesu.payload).isEqualTo(Bytes.fromHexString(input)) + assertThat(txBesu.signature.r).isEqualTo( + "0xeb4f70991ea4f14d23efb32591da3621d551406fd32bdfdd78bb677dec13160a".toBigIntegerFromHex() + ) + assertThat(txBesu.signature.s).isEqualTo( + "0x783aaa89f73ef7535924da8fd5f12e15cae1a0811c4c4746d1c23abff1eacddf".toBigIntegerFromHex() + ) + assertThat(txBesu.signature.recId).isEqualTo(1) + assertThat(txBesu.chainId.getOrNull()).isEqualTo(0x539L) + assertThat(txBesu.maxFeePerGas.getOrNull()).isEqualTo(Wei.of(0x1017df87L)) + assertThat(txBesu.maxPriorityFeePerGas.getOrNull()).isEqualTo(Wei.of(0x1017df87L)) + assertThat(txBesu.accessList.getOrNull()).isEmpty() + } + } + + @Test + fun `shall decode tx with to=null`() { + val input = """ + 0x608060405234801561001057600080fd5b5061001a3361001f565b61006f565b600080546001600160a01b038381166001600160a01b0319831681178455604051919092169283917f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e09190a35050565b6108658061007e6000396000f3fe60806040526004361061007b5760003560e01c80639623609d1161004e5780639623609d1461012b57806399a88ec41461013e578063f2fde38b1461015e578063f3b7dead1461017e57600080fd5b8063204e1c7a14610080578063715018a6146100c95780637eff275e146100e05780638da5cb5b14610100575b600080fd5b34801561008c57600080fd5b506100a061009b366004610608565b61019e565b60405173ffffffffffffffffffffffffffffffffffffffff909116815260200160405180910390f35b3480156100d557600080fd5b506100de610255565b005b3480156100ec57600080fd5b506100de6100fb36600461062c565b610269565b34801561010c57600080fd5b5060005473ffffffffffffffffffffffffffffffffffffffff166100a0565b6100de610139366004610694565b6102f7565b34801561014a57600080fd5b506100de61015936600461062c565b61038c565b34801561016a57600080fd5b506100de610179366004610608565b6103e8565b34801561018a57600080fd5b506100a0610199366004610608565b6104a4565b60008060008373ffffffffffffffffffffffffffffffffffffffff166040516101ea907f5c60da1b00000000000000000000000000000000000000000000000000000000815260040190565b600060405180830381855afa9150503d8060008114610225576040519150601f19603f3d011682016040523d82523d6000602084013e61022a565b606091505b50915091508161023957600080fd5b8080602001905181019061024d9190610788565b949350505050565b61025d6104f0565b6102676000610571565b565b6102716104f0565b6040517f8f28397000000000000000000000000000000000000000000000000000000000815273ffffffffffffffffffffffffffffffffffffffff8281166004830152831690638f283970906024015b600060405180830381600087803b1580156102db57600080fd5b505af11580156102ef573d6000803e3d6000fd5b505050505050565b6102ff6104f0565b6040517f4f1ef28600000000000000000000000000000000000000000000000000000000815273ffffffffffffffffffffffffffffffffffffffff841690634f1ef28690349061035590869086906004016107a5565b6000604051808303818588803b15801561036e57600080fd5b505af1158015610382573d6000803e3d6000fd5b5050505050505050565b6103946104f0565b6040517f3659cfe600000000000000000000000000000000000000000000000000000000815273ffffffffffffffffffffffffffffffffffffffff8281166004830152831690633659cfe6906024016102c1565b6103f06104f0565b73ffffffffffffffffffffffffffffffffffffffff8116610498576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602660248201527f4f776e61626c653a206e6577206f776e657220697320746865207a65726f206160448201527f646472657373000000000000000000000000000000000000000000000000000060648201526084015b60405180910390fd5b6104a181610571565b50565b60008060008373ffffffffffffffffffffffffffffffffffffffff166040516101ea907ff851a44000000000000000000000000000000000000000000000000000000000815260040190565b60005473ffffffffffffffffffffffffffffffffffffffff163314610267576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e6572604482015260640161048f565b6000805473ffffffffffffffffffffffffffffffffffffffff8381167fffffffffffffffffffffffff0000000000000000000000000000000000000000831681178455604051919092169283917f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e09190a35050565b73ffffffffffffffffffffffffffffffffffffffff811681146104a157600080fd5b60006020828403121561061a57600080fd5b8135610625816105e6565b9392505050565b6000806040838503121561063f57600080fd5b823561064a816105e6565b9150602083013561065a816105e6565b809150509250929050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b6000806000606084860312156106a957600080fd5b83356106b4816105e6565b925060208401356106c4816105e6565b9150604084013567ffffffffffffffff808211156106e157600080fd5b818601915086601f8301126106f557600080fd5b81358181111561070757610707610665565b604051601f82017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0908116603f0116810190838211818310171561074d5761074d610665565b8160405282815289602084870101111561076657600080fd5b8260208601602083013760006020848301015280955050505050509250925092565b60006020828403121561079a57600080fd5b8151610625816105e6565b73ffffffffffffffffffffffffffffffffffffffff8316815260006020604081840152835180604085015260005b818110156107ef578581018301518582016060015282016107d3565b5060006060828601015260607fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0601f83011685010192505050939250505056fea2646970667358221220688ab5dd8d9528556ea321a4b4ef35edd0288c19274db8bf4057c8b61d9e438764736f6c63430008130033 + """.trimIndent().trim() + val txJson = + """ + { + "accessList": [], + "blockHash": "0xf9bf74ade4a723a5527badeb62ce58d478f1022df0effc2a091898ef068563b6", + "blockNumber": "0x1", + "chainId": "0x539", + "from": "0x1b9abeec3215d8ade8a33607f2cf0f4f60e5f0d0", + "gas": "0x83a3d", + "gasPrice": "0x7", + "maxPriorityFeePerGas": "0x0", + "maxFeePerGas": "0xe", + "hash": "0xc9647251765f5d679e024dd0e5c0f4700c431f129e50847c3f73e2aa2262e593", + "input": "$input", + "nonce": "0x1", + "to": null, + "transactionIndex": "0x1", + "type": "0x2", + "value": "0x0", + "yParity": "0x1", + "v": "0x1", + "r": "0xf7afccb560d0c52bea021ba522a27dbd6c3aba3512dd2d3b2f476ed8dd87d5f7", + "s": "0x5f47f6ddcf1c216eb33eb69db553d682de34c78f5a5ab97905a428c2182f32e" + } + """.trimIndent().let { JsonObject(it) } + + val txWeb3J = EthBlock.TransactionObject( + /*hash*/ txJson.getString("hash"), + /*nonce*/ txJson.getString("nonce"), + /*blockHash*/ txJson.getString("blockHash"), + /*blockNumber*/ txJson.getString("blockNumber"), + /*chainId*/ txJson.getString("chainId"), + /*transactionIndex*/ txJson.getString("transactionIndex"), + /*from*/ txJson.getString("from"), + /*to*/ null, + /*value*/ txJson.getString("value"), + /*gasPrice*/ txJson.getString("gasPrice"), + /*gas*/ txJson.getString("gas"), + /*input*/ txJson.getString("input"), + /*creates*/ null, + /*publicKey*/null, + /*raw*/null, + /*r*/ txJson.getString("r"), + /*s*/ txJson.getString("s"), + /*v*/ txJson.getString("v").toLongFromHex(), + /*yParity*/ txJson.getString("yParity"), + /*type*/ txJson.getString("type"), + /*maxFeePerGas*/ txJson.getString("maxFeePerGas"), + /*maxPriorityFeePerGas*/ txJson.getString("maxPriorityFeePerGas"), + /* accessList*/ emptyList() + ) + + val txDomain = txWeb3J.toDomain() + assertThat(txDomain).isEqualTo( + Transaction( + nonce = 1UL, + gasPrice = 0x7UL, + gasLimit = 0x83a3dUL, + to = null, + value = 0UL.toBigInteger(), + input = input.decodeHex(), + r = "0xf7afccb560d0c52bea021ba522a27dbd6c3aba3512dd2d3b2f476ed8dd87d5f7".toBigIntegerFromHex(), + s = "0x5f47f6ddcf1c216eb33eb69db553d682de34c78f5a5ab97905a428c2182f32e".toBigIntegerFromHex(), + v = 1UL, + yParity = 1UL, + type = TransactionType.EIP1559, + chainId = 0x539UL, + maxFeePerGas = 0xeUL, + maxPriorityFeePerGas = 0UL, + accessList = emptyList() + ) + ) + + txDomain.toBesu().let { txBesu -> + assertThat(txBesu.type).isEqualTo(org.hyperledger.besu.datatypes.TransactionType.EIP1559) + assertThat(txBesu.nonce).isEqualTo(1L) + assertThat(txBesu.gasPrice.getOrNull()).isEqualTo(Wei.of(0x7L)) + assertThat(txBesu.gasLimit).isEqualTo(0x83a3dL) + assertThat(txBesu.to.getOrNull()).isNull() + assertThat(txBesu.value).isEqualTo(Wei.ZERO) + assertThat(txBesu.payload).isEqualTo(Bytes.fromHexString(input)) + assertThat(txBesu.signature.r).isEqualTo( + "0xf7afccb560d0c52bea021ba522a27dbd6c3aba3512dd2d3b2f476ed8dd87d5f7".toBigIntegerFromHex() + ) + assertThat(txBesu.signature.s).isEqualTo( + "0x5f47f6ddcf1c216eb33eb69db553d682de34c78f5a5ab97905a428c2182f32e".toBigIntegerFromHex() + ) + assertThat(txBesu.signature.recId).isEqualTo(1) + assertThat(txBesu.chainId.getOrNull()).isEqualTo(0x539L) + assertThat(txBesu.maxFeePerGas.getOrNull()).isEqualTo(Wei.of(0xeL)) + assertThat(txBesu.maxPriorityFeePerGas.getOrNull()).isEqualTo(Wei.ZERO) + assertThat(txBesu.accessList.getOrNull()).isEmpty() + } + } } From 1044b2b9af9850db24cf51f5792d86debcb8ecdf Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Mon, 9 Dec 2024 22:13:19 +0000 Subject: [PATCH 03/31] coordinator: improve tests --- coordinator/app/build.gradle | 1 - .../EthGetBlockToLineaBlockMapperTest.kt | 347 ++++++------------ 2 files changed, 109 insertions(+), 239 deletions(-) diff --git a/coordinator/app/build.gradle b/coordinator/app/build.gradle index 4a0e208ae..89e778f6f 100644 --- a/coordinator/app/build.gradle +++ b/coordinator/app/build.gradle @@ -19,7 +19,6 @@ dependencies { implementation project(':jvm-libs:generic:vertx-helper') implementation project(':jvm-libs:generic:extensions:futures') implementation project(':jvm-libs:generic:persistence:db') - // implementation project(':jvm-libs:generic:serialization:jackson') implementation project(':jvm-libs:linea:web3j-extensions') implementation project(':jvm-libs:linea:core:metrics') implementation project(':jvm-libs:linea:metrics:micrometer') diff --git a/jvm-libs/linea/web3j-extensions/src/test/kotlin/linea/web3j/EthGetBlockToLineaBlockMapperTest.kt b/jvm-libs/linea/web3j-extensions/src/test/kotlin/linea/web3j/EthGetBlockToLineaBlockMapperTest.kt index ec0cc3f1c..d071db16f 100644 --- a/jvm-libs/linea/web3j-extensions/src/test/kotlin/linea/web3j/EthGetBlockToLineaBlockMapperTest.kt +++ b/jvm-libs/linea/web3j-extensions/src/test/kotlin/linea/web3j/EthGetBlockToLineaBlockMapperTest.kt @@ -1,6 +1,5 @@ package linea.web3j -import io.vertx.core.json.JsonObject import linea.domain.AccessListEntry import linea.domain.Transaction import linea.domain.TransactionType @@ -8,7 +7,6 @@ import linea.domain.toBesu import net.consensys.decodeHex import net.consensys.toBigInteger import net.consensys.toBigIntegerFromHex -import net.consensys.toLongFromHex import org.apache.tuweni.bytes.Bytes import org.apache.tuweni.bytes.Bytes32 import org.assertj.core.api.Assertions.assertThat @@ -16,18 +14,21 @@ import org.assertj.core.api.Assertions.fail import org.hyperledger.besu.datatypes.Address import org.hyperledger.besu.datatypes.Wei import org.junit.jupiter.api.Test -import org.web3j.protocol.core.methods.response.AccessListObject +import org.web3j.protocol.ObjectMapperFactory import org.web3j.protocol.core.methods.response.EthBlock import kotlin.jvm.optionals.getOrElse import kotlin.jvm.optionals.getOrNull class EthGetBlockToLineaBlockMapperTest { + // using raw JSON from eth_getBlockByNumber responses because realistically represens our use case + // Also it's very easy create new test cases + private fun serialize(json: String): EthBlock.TransactionObject { + return ObjectMapperFactory.getObjectMapper().readValue(json, EthBlock.TransactionObject::class.java) + } @Test fun `should map frontier transactions`() { - val transactionJson = JsonObject( - // example of eth_getBlockByNumber.transactions[0] response - // This seems to be easier to read given than build Java object directly + val txWeb3j = serialize( """ { "blockHash": "0x004257e560a5f82595dddb73f752b904efef4b73cb3ece1469f5e5091e3c9665", @@ -49,33 +50,7 @@ class EthGetBlockToLineaBlockMapperTest { } """.trimIndent() ) - val web3jTx = EthBlock.TransactionObject( - /*hash*/ transactionJson.getString("hash"), - /*nonce*/ transactionJson.getString("nonce"), - /*blockHash*/ transactionJson.getString("blockHash"), - /*blockNumber*/ transactionJson.getString("blockNumber"), - /*chainId*/ transactionJson.getString("chainId"), - /*transactionIndex*/ transactionJson.getString("transactionIndex"), - /*from*/ transactionJson.getString("from"), - /*to*/ transactionJson.getString("to"), - /*value*/ transactionJson.getString("value"), - /*gasPrice*/ transactionJson.getString("gasPrice"), - /*gas*/ transactionJson.getString("gas"), - /*input*/ transactionJson.getString("input"), - /*creates*/ null, - /*publicKey*/null, - /*raw*/null, - /*r*/ transactionJson.getString("r"), - /*s*/ transactionJson.getString("s"), - /*v*/ transactionJson.getString("v").toLongFromHex(), - /*yParity*/null, - /*type*/ transactionJson.getString("type"), - /*maxFeePerGas*/null, - /*maxPriorityFeePerGas*/ null, - /*> accessList*/null - ) - - val domainTx = web3jTx.toDomain() + val domainTx = txWeb3j.toDomain() assertThat(domainTx).isEqualTo( Transaction( nonce = 0x97411UL, @@ -118,71 +93,40 @@ class EthGetBlockToLineaBlockMapperTest { @Test fun `should map transaction with AccessList`() { - val txJson = + val txWeb3j = serialize( """ - { - "accessList": [ - { - "address": "0x8d97689c9818892b700e27f316cc3e41e17fbeb9", - "storageKeys": [ - "0x0000000000000000000000000000000000000000000000000000000000000000", - "0x0000000000000000000000000000000000000000000000000000000000000001" - ] - } - ], - "blockHash": "0x7480ae911853c1fba10145401a21ddca3943b5894d74cbbf7a6beec526d1f9c2", - "blockNumber": "0xa", - "chainId": "0x539", - "from": "0xce3b7d471fd1fdd10d788ae64e48a9c2f2361179", - "gas": "0x30d40", - "gasPrice": "0x1017df87", - "hash": "0x8ef620582ed8ba98c8496a42b27a30ff7b1de901b1ff7e65b22ea59a2d0668ce", - "input": "0x", - "nonce": "0x0", - "to": "0x8d97689c9818892b700e27f316cc3e41e17fbeb9", - "transactionIndex": "0x2", - "type": "0x1", - "value": "0x2386f26fc10000", - "yParity": "0x1", - "v": "0x1", - "r": "0x4f24ed24207bec8591c8172584dc3b57cdf3ee96afbd5e63905a90a704ff33f0", - "s": "0x6277bb9d2614843a4791ff2c192e70876438ec940c39d92deb504591b83dfeb3" - } - """.trimIndent().let { JsonObject(it) } - - val txWeb3J = EthBlock.TransactionObject( - /*hash*/ txJson.getString("hash"), - /*nonce*/ txJson.getString("nonce"), - /*blockHash*/ txJson.getString("blockHash"), - /*blockNumber*/ txJson.getString("blockNumber"), - /*chainId*/ txJson.getString("chainId"), - /*transactionIndex*/ txJson.getString("transactionIndex"), - /*from*/ txJson.getString("from"), - /*to*/ txJson.getString("to"), - /*value*/ txJson.getString("value"), - /*gasPrice*/ txJson.getString("gasPrice"), - /*gas*/ txJson.getString("gas"), - /*input*/ txJson.getString("input"), - /*creates*/ null, - /*publicKey*/null, - /*raw*/null, - /*r*/ txJson.getString("r"), - /*s*/ txJson.getString("s"), - /*v*/ txJson.getString("v").toLongFromHex(), - /*yParity*/ txJson.getString("yParity"), - /*type*/ txJson.getString("type"), - /*maxFeePerGas*/null, - /*maxPriorityFeePerGas*/ null, - /* accessList*/txJson.getJsonArray("accessList").map { accessListEntry -> - val accessList = accessListEntry as JsonObject - AccessListObject( - /*address*/ accessList.getString("address"), - /*storageKeys*/ accessList.getJsonArray("storageKeys").map { it as String } - ) + { + "accessList": [ + { + "address": "0x8d97689c9818892b700e27f316cc3e41e17fbeb9", + "storageKeys": [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0x0000000000000000000000000000000000000000000000000000000000000001" + ] + } + ], + "blockHash": "0x7480ae911853c1fba10145401a21ddca3943b5894d74cbbf7a6beec526d1f9c2", + "blockNumber": "0xa", + "chainId": "0x539", + "from": "0xce3b7d471fd1fdd10d788ae64e48a9c2f2361179", + "gas": "0x30d40", + "gasPrice": "0x1017df87", + "hash": "0x8ef620582ed8ba98c8496a42b27a30ff7b1de901b1ff7e65b22ea59a2d0668ce", + "input": "0x", + "nonce": "0x0", + "to": "0x8d97689c9818892b700e27f316cc3e41e17fbeb9", + "transactionIndex": "0x2", + "type": "0x1", + "value": "0x2386f26fc10000", + "yParity": "0x1", + "v": "0x1", + "r": "0x4f24ed24207bec8591c8172584dc3b57cdf3ee96afbd5e63905a90a704ff33f0", + "s": "0x6277bb9d2614843a4791ff2c192e70876438ec940c39d92deb504591b83dfeb3" } + """.trimIndent() ) - val domainTx = txWeb3J.toDomain() + val domainTx = txWeb3j.toDomain() assertThat(domainTx).isEqualTo( Transaction( nonce = 0UL, @@ -243,56 +187,32 @@ class EthGetBlockToLineaBlockMapperTest { @Test fun `should map type accessList with empty list`() { - val txJson = + val txWeb3j = serialize( """ { - "accessList": [], - "blockHash": "0x7480ae911853c1fba10145401a21ddca3943b5894d74cbbf7a6beec526d1f9c2", - "blockNumber": "0xa", - "chainId": "0x539", - "from": "0x5007b0259849a673d0d780611f9a2ed8821d9ebe", - "gas": "0x5208", - "gasPrice": "0x1017df87", - "hash": "0xa2334c8858bb44ef3e9ef7f3523ec058ab24a869cfad7333fdf7bf3bb76deec4", - "input": "0x", - "nonce": "0x0", - "to": "0x8d97689c9818892b700e27f316cc3e41e17fbeb9", - "transactionIndex": "0x3", - "type": "0x1", - "value": "0x2386f26fc10000", - "yParity": "0x1", - "v": "0x1", - "r": "0xc57273f9ba15320937d5d9dfd1dc0b18d1e678b34bd3a4bfd29a63e11a856292", - "s": "0x7aa875a64835ecc5f9ac1a9fe3ab38d2a62bb3643a2597ab585a5607641a0c57" - } - """.trimIndent().let { JsonObject(it) } - val txWeb3J = EthBlock.TransactionObject( - /*hash*/ txJson.getString("hash"), - /*nonce*/ txJson.getString("nonce"), - /*blockHash*/ txJson.getString("blockHash"), - /*blockNumber*/ txJson.getString("blockNumber"), - /*chainId*/ txJson.getString("chainId"), - /*transactionIndex*/ txJson.getString("transactionIndex"), - /*from*/ txJson.getString("from"), - /*to*/ txJson.getString("to"), - /*value*/ txJson.getString("value"), - /*gasPrice*/ txJson.getString("gasPrice"), - /*gas*/ txJson.getString("gas"), - /*input*/ txJson.getString("input"), - /*creates*/ null, - /*publicKey*/null, - /*raw*/null, - /*r*/ txJson.getString("r"), - /*s*/ txJson.getString("s"), - /*v*/ txJson.getString("v").toLongFromHex(), - /*yParity*/ txJson.getString("yParity"), - /*type*/ txJson.getString("type"), - /*maxFeePerGas*/null, - /*maxPriorityFeePerGas*/ null, - /* accessList*/ emptyList() + "accessList": [], + "blockHash": "0x7480ae911853c1fba10145401a21ddca3943b5894d74cbbf7a6beec526d1f9c2", + "blockNumber": "0xa", + "chainId": "0x539", + "from": "0x5007b0259849a673d0d780611f9a2ed8821d9ebe", + "gas": "0x5208", + "gasPrice": "0x1017df87", + "hash": "0xa2334c8858bb44ef3e9ef7f3523ec058ab24a869cfad7333fdf7bf3bb76deec4", + "input": "0x", + "nonce": "0x0", + "to": "0x8d97689c9818892b700e27f316cc3e41e17fbeb9", + "transactionIndex": "0x3", + "type": "0x1", + "value": "0x2386f26fc10000", + "yParity": "0x1", + "v": "0x1", + "r": "0xc57273f9ba15320937d5d9dfd1dc0b18d1e678b34bd3a4bfd29a63e11a856292", + "s": "0x7aa875a64835ecc5f9ac1a9fe3ab38d2a62bb3643a2597ab585a5607641a0c57" + } + """.trimIndent() ) - val domainTx = txWeb3J.toDomain() + val domainTx = txWeb3j.toDomain() assertThat(domainTx).isEqualTo( Transaction( nonce = 0UL, @@ -342,59 +262,35 @@ class EthGetBlockToLineaBlockMapperTest { """ 0xdeb3cdf2000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000003e84d538bf9753309729adf92867d75bc2e58b566cb204b0d1b018fbf311e4b49bc52c1c450b2f412f5d9a44e01990d6127bc805ba8ef079ed2a897070378d706fbd2f5cf52b0e172541b7f11b9c2f0e0b91a67c5caf5908ddfdd2d340349e7398b698b3c336876c88e232f0e8f3197f2683e54a4439abb7d210d84cc1d3ad1bd48d0ba5dc30714d253743a734f17e88354eea550f7945df35d4c6316fcfaad09846f81f59b8127b037dbce6e5ffa45120fc69e6852f0c8ac2fcc9fd5e72503dd1c8d114ee079ffed84ddd6851e438cdd2a0ed1df9f0255481dc2a61a4808d856525619c948fbbd063bfd3db42504547db68c29990540eb7a36a1a8a0e483544eb634ae33f43f5bac2d991b9f6b36e23a7a299ade5b30ab96ad6dae27a9c374ae5f702fc689f596450c467722f24b7621ba5663ed6e08b620f04bc524338cac50e9ebc302d0b33dd9e2e563f05ce26303666a6c8c0f8dcd0b475f2219398ff4552533d28660c8d0d2843ce238ffb856dee06bc28e1ec0b92c3cb7b91378c07b049f3af20017dbdfdf48320ea7cf5f331bee27ba33d6a41351b3f044612a45f51451c068c23d6aec6784f623c6855acb95f07f213ad8605861fd8601ffa9a0282508a4d859769cb61247389020587a570cba1eac8b05576bd5b7a81b166f3c7b0f0ae0a8117f642d3fd0957e1cface4d10ebe6475a9a1f3bf6e3b1b7c16e50e529adcf0cf278aa64b9fecedcb0d894ab7ba6589e96ff56dff7b8636413f46cdc073c22521f6b89d7b68ca6f8af1ecc4e453137c801a9b35b5c4869883f59aaeaa7ee637d71c7e02f08894cffdb51a368b225fa1ab00e3ad2d91d1275d048aad5eb5d34438622c7aea1759b3fe747c2b0fddd62159de1d7cabbccde9c1e3511a34432e0c4e6dede019e38493fc29292ea321621629c1ffc62160747ee136171c96f55af7af6a29b8ca94f12d12b7c706974b1e586b3674a6aec7510f1025ba399f7a97f5911187b040b7a494e191bd761ad2a78daf427f5ee19cf24bc45fab34d32747de0f0a2c6bd33d2440d9f5d20da22da34e418d54d6894d42edd6d0c5a4f9b02d510a23db40cc455b7c423bd43b6fcd0f3655285e16ba8d9bdaf3f2147de572c33568353b5f5dd820f49dbddbc63297aed5e2f342b383a83319f9beed9d3d358a3dc7c0a010b85954fec3b34c3227a9b4447bd5d30b8b78c0ef36cd8197e867d37778b24e8ebfaadf08c42f3db6e5e46cb025bb4e98334ce0a7a59ba155eaf3968621f353d075e0d68f0787259e344a72e8938ebe3a81458ad20df917dc1392fe759210f045f7d87177ad39a13ec704301f1f0845b8c6cbc52f8f77c043bcc80adb513470d0e5a6b02df65259bcc3198efe01c555a5d28bf89f818ea1b984a64db220f487e230652000000000000000000000000000000000000000000000000 """.trimIndent().trim() - val txJson = + val txWeb3j = serialize( """ { - "accessList": [], - "blockHash": "0x7480ae911853c1fba10145401a21ddca3943b5894d74cbbf7a6beec526d1f9c2", - "blockNumber": "0xa", - "chainId": "0x539", - "from": "0x5007b0259849a673d0d780611f9a2ed8821d9ebe", - "gas": "0x5208", - "gasPrice": "0x1017df87", - "hash": "0xa2334c8858bb44ef3e9ef7f3523ec058ab24a869cfad7333fdf7bf3bb76deec4", - "input": "$input", - "nonce": "0x0", - "to": "0xe4392c8ecc46b304c83cdb5edaf742899b1bda93", - "transactionIndex": "0x3", - "type": "0x2", - "value": "0x2386f26fc10000", - "yParity": "0x1", - "v": "0x1", - "r": "0xeb4f70991ea4f14d23efb32591da3621d551406fd32bdfdd78bb677dec13160a", - "s": "0x783aaa89f73ef7535924da8fd5f12e15cae1a0811c4c4746d1c23abff1eacddf", - "maxFeePerGas": "0x1017df87", - "maxPriorityFeePerGas": "0x1017df87" - } - """.trimIndent().let { JsonObject(it) } - val txWeb3J = EthBlock.TransactionObject( - /*hash*/ txJson.getString("hash"), - /*nonce*/ txJson.getString("nonce"), - /*blockHash*/ txJson.getString("blockHash"), - /*blockNumber*/ txJson.getString("blockNumber"), - /*chainId*/ txJson.getString("chainId"), - /*transactionIndex*/ txJson.getString("transactionIndex"), - /*from*/ txJson.getString("from"), - /*to*/ txJson.getString("to"), - /*value*/ txJson.getString("value"), - /*gasPrice*/ txJson.getString("gasPrice"), - /*gas*/ txJson.getString("gas"), - /*input*/ txJson.getString("input"), - /*creates*/ null, - /*publicKey*/null, - /*raw*/null, - /*r*/ txJson.getString("r"), - /*s*/ txJson.getString("s"), - /*v*/ txJson.getString("v").toLongFromHex(), - /*yParity*/ txJson.getString("yParity"), - /*type*/ txJson.getString("type"), - /*maxFeePerGas*/ txJson.getString("maxFeePerGas"), - /*maxPriorityFeePerGas*/ txJson.getString("maxPriorityFeePerGas"), - /* accessList*/ emptyList() + "accessList": [], + "blockHash": "0x7480ae911853c1fba10145401a21ddca3943b5894d74cbbf7a6beec526d1f9c2", + "blockNumber": "0xa", + "chainId": "0x539", + "from": "0x5007b0259849a673d0d780611f9a2ed8821d9ebe", + "gas": "0x5208", + "gasPrice": "0x1017df87", + "hash": "0xa2334c8858bb44ef3e9ef7f3523ec058ab24a869cfad7333fdf7bf3bb76deec4", + "input": "$input", + "nonce": "0x0", + "to": "0xe4392c8ecc46b304c83cdb5edaf742899b1bda93", + "transactionIndex": "0x3", + "type": "0x2", + "value": "0x2386f26fc10000", + "yParity": "0x1", + "v": "0x1", + "r": "0xeb4f70991ea4f14d23efb32591da3621d551406fd32bdfdd78bb677dec13160a", + "s": "0x783aaa89f73ef7535924da8fd5f12e15cae1a0811c4c4746d1c23abff1eacddf", + "maxFeePerGas": "0x1017df87", + "maxPriorityFeePerGas": "0x1017df87" + } + """.trimIndent() ) - val txDomain = txWeb3J.toDomain() - assertThat(txWeb3J.toDomain()).isEqualTo( + val txDomain = txWeb3j.toDomain() + assertThat(txDomain).isEqualTo( Transaction( nonce = 0UL, gasPrice = 0x1017df87UL, @@ -441,59 +337,34 @@ class EthGetBlockToLineaBlockMapperTest { val input = """ 0x608060405234801561001057600080fd5b5061001a3361001f565b61006f565b600080546001600160a01b038381166001600160a01b0319831681178455604051919092169283917f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e09190a35050565b6108658061007e6000396000f3fe60806040526004361061007b5760003560e01c80639623609d1161004e5780639623609d1461012b57806399a88ec41461013e578063f2fde38b1461015e578063f3b7dead1461017e57600080fd5b8063204e1c7a14610080578063715018a6146100c95780637eff275e146100e05780638da5cb5b14610100575b600080fd5b34801561008c57600080fd5b506100a061009b366004610608565b61019e565b60405173ffffffffffffffffffffffffffffffffffffffff909116815260200160405180910390f35b3480156100d557600080fd5b506100de610255565b005b3480156100ec57600080fd5b506100de6100fb36600461062c565b610269565b34801561010c57600080fd5b5060005473ffffffffffffffffffffffffffffffffffffffff166100a0565b6100de610139366004610694565b6102f7565b34801561014a57600080fd5b506100de61015936600461062c565b61038c565b34801561016a57600080fd5b506100de610179366004610608565b6103e8565b34801561018a57600080fd5b506100a0610199366004610608565b6104a4565b60008060008373ffffffffffffffffffffffffffffffffffffffff166040516101ea907f5c60da1b00000000000000000000000000000000000000000000000000000000815260040190565b600060405180830381855afa9150503d8060008114610225576040519150601f19603f3d011682016040523d82523d6000602084013e61022a565b606091505b50915091508161023957600080fd5b8080602001905181019061024d9190610788565b949350505050565b61025d6104f0565b6102676000610571565b565b6102716104f0565b6040517f8f28397000000000000000000000000000000000000000000000000000000000815273ffffffffffffffffffffffffffffffffffffffff8281166004830152831690638f283970906024015b600060405180830381600087803b1580156102db57600080fd5b505af11580156102ef573d6000803e3d6000fd5b505050505050565b6102ff6104f0565b6040517f4f1ef28600000000000000000000000000000000000000000000000000000000815273ffffffffffffffffffffffffffffffffffffffff841690634f1ef28690349061035590869086906004016107a5565b6000604051808303818588803b15801561036e57600080fd5b505af1158015610382573d6000803e3d6000fd5b5050505050505050565b6103946104f0565b6040517f3659cfe600000000000000000000000000000000000000000000000000000000815273ffffffffffffffffffffffffffffffffffffffff8281166004830152831690633659cfe6906024016102c1565b6103f06104f0565b73ffffffffffffffffffffffffffffffffffffffff8116610498576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602660248201527f4f776e61626c653a206e6577206f776e657220697320746865207a65726f206160448201527f646472657373000000000000000000000000000000000000000000000000000060648201526084015b60405180910390fd5b6104a181610571565b50565b60008060008373ffffffffffffffffffffffffffffffffffffffff166040516101ea907ff851a44000000000000000000000000000000000000000000000000000000000815260040190565b60005473ffffffffffffffffffffffffffffffffffffffff163314610267576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e6572604482015260640161048f565b6000805473ffffffffffffffffffffffffffffffffffffffff8381167fffffffffffffffffffffffff0000000000000000000000000000000000000000831681178455604051919092169283917f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e09190a35050565b73ffffffffffffffffffffffffffffffffffffffff811681146104a157600080fd5b60006020828403121561061a57600080fd5b8135610625816105e6565b9392505050565b6000806040838503121561063f57600080fd5b823561064a816105e6565b9150602083013561065a816105e6565b809150509250929050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b6000806000606084860312156106a957600080fd5b83356106b4816105e6565b925060208401356106c4816105e6565b9150604084013567ffffffffffffffff808211156106e157600080fd5b818601915086601f8301126106f557600080fd5b81358181111561070757610707610665565b604051601f82017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0908116603f0116810190838211818310171561074d5761074d610665565b8160405282815289602084870101111561076657600080fd5b8260208601602083013760006020848301015280955050505050509250925092565b60006020828403121561079a57600080fd5b8151610625816105e6565b73ffffffffffffffffffffffffffffffffffffffff8316815260006020604081840152835180604085015260005b818110156107ef578581018301518582016060015282016107d3565b5060006060828601015260607fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0601f83011685010192505050939250505056fea2646970667358221220688ab5dd8d9528556ea321a4b4ef35edd0288c19274db8bf4057c8b61d9e438764736f6c63430008130033 """.trimIndent().trim() - val txJson = + val txWeb3j = serialize( """ { - "accessList": [], - "blockHash": "0xf9bf74ade4a723a5527badeb62ce58d478f1022df0effc2a091898ef068563b6", - "blockNumber": "0x1", - "chainId": "0x539", - "from": "0x1b9abeec3215d8ade8a33607f2cf0f4f60e5f0d0", - "gas": "0x83a3d", - "gasPrice": "0x7", - "maxPriorityFeePerGas": "0x0", - "maxFeePerGas": "0xe", - "hash": "0xc9647251765f5d679e024dd0e5c0f4700c431f129e50847c3f73e2aa2262e593", - "input": "$input", - "nonce": "0x1", - "to": null, - "transactionIndex": "0x1", - "type": "0x2", - "value": "0x0", - "yParity": "0x1", - "v": "0x1", - "r": "0xf7afccb560d0c52bea021ba522a27dbd6c3aba3512dd2d3b2f476ed8dd87d5f7", - "s": "0x5f47f6ddcf1c216eb33eb69db553d682de34c78f5a5ab97905a428c2182f32e" - } - """.trimIndent().let { JsonObject(it) } - - val txWeb3J = EthBlock.TransactionObject( - /*hash*/ txJson.getString("hash"), - /*nonce*/ txJson.getString("nonce"), - /*blockHash*/ txJson.getString("blockHash"), - /*blockNumber*/ txJson.getString("blockNumber"), - /*chainId*/ txJson.getString("chainId"), - /*transactionIndex*/ txJson.getString("transactionIndex"), - /*from*/ txJson.getString("from"), - /*to*/ null, - /*value*/ txJson.getString("value"), - /*gasPrice*/ txJson.getString("gasPrice"), - /*gas*/ txJson.getString("gas"), - /*input*/ txJson.getString("input"), - /*creates*/ null, - /*publicKey*/null, - /*raw*/null, - /*r*/ txJson.getString("r"), - /*s*/ txJson.getString("s"), - /*v*/ txJson.getString("v").toLongFromHex(), - /*yParity*/ txJson.getString("yParity"), - /*type*/ txJson.getString("type"), - /*maxFeePerGas*/ txJson.getString("maxFeePerGas"), - /*maxPriorityFeePerGas*/ txJson.getString("maxPriorityFeePerGas"), - /* accessList*/ emptyList() + "accessList": [], + "blockHash": "0xf9bf74ade4a723a5527badeb62ce58d478f1022df0effc2a091898ef068563b6", + "blockNumber": "0x1", + "chainId": "0x539", + "from": "0x1b9abeec3215d8ade8a33607f2cf0f4f60e5f0d0", + "gas": "0x83a3d", + "gasPrice": "0x7", + "maxPriorityFeePerGas": "0x0", + "maxFeePerGas": "0xe", + "hash": "0xc9647251765f5d679e024dd0e5c0f4700c431f129e50847c3f73e2aa2262e593", + "input": "$input", + "nonce": "0x1", + "to": null, + "transactionIndex": "0x1", + "type": "0x2", + "value": "0x0", + "yParity": "0x1", + "v": "0x1", + "r": "0xf7afccb560d0c52bea021ba522a27dbd6c3aba3512dd2d3b2f476ed8dd87d5f7", + "s": "0x5f47f6ddcf1c216eb33eb69db553d682de34c78f5a5ab97905a428c2182f32e" + } + """.trimIndent() ) - val txDomain = txWeb3J.toDomain() + val txDomain = txWeb3j.toDomain() assertThat(txDomain).isEqualTo( Transaction( nonce = 1UL, From f4a0147657c78ad28d55e2d869068e6cf12e9aec Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Tue, 10 Dec 2024 10:59:24 +0000 Subject: [PATCH 04/31] Update coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitor.kt Co-authored-by: Roman Vaseev <4833306+Filter94@users.noreply.github.com> Signed-off-by: Pedro Novais <1478752+jpnovais@users.noreply.github.com> --- .../zkevm/coordinator/blockcreation/BlockCreationMonitor.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitor.kt b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitor.kt index 2219a49e8..7df27335e 100644 --- a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitor.kt +++ b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitor.kt @@ -147,7 +147,7 @@ class BlockCreationMonitor( block.number, block.hash.encodeHex(), block.parentHash.encodeHex(), - expectedParentBlockHash.get().encodeHex().subSequence(0, 8) + expectedParentBlockHash.get().encodeHex() ) SafeFuture.failedFuture(IllegalStateException("Reorg detected on block ${block.number}")) } From 796d7febcb762e45ca53d9493b2cf5f5f50369a0 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:16:46 +0000 Subject: [PATCH 05/31] coordinator: carveout compressor of coordinator --- jvm-libs/linea/blob-compressor/build.gradle | 2 + .../main/kotlin/linea/blob/BlobCompressor.kt | 109 ++++++++++++++++++ .../linea/blob/GoBackedBlobCompressorTest.kt | 90 +++++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 jvm-libs/linea/blob-compressor/src/main/kotlin/linea/blob/BlobCompressor.kt create mode 100644 jvm-libs/linea/blob-compressor/src/test/kotlin/linea/blob/GoBackedBlobCompressorTest.kt diff --git a/jvm-libs/linea/blob-compressor/build.gradle b/jvm-libs/linea/blob-compressor/build.gradle index 0c9cb129f..20561d11b 100644 --- a/jvm-libs/linea/blob-compressor/build.gradle +++ b/jvm-libs/linea/blob-compressor/build.gradle @@ -11,6 +11,8 @@ apply from: rootProject.file("gradle/publishing.gradle") dependencies { implementation "net.java.dev.jna:jna:${libs.versions.jna.get()}" implementation project(":jvm-libs:generic:extensions:kotlin") + implementation "org.apache.logging.log4j:log4j-api:${libs.versions.log4j.get()}" + implementation "org.apache.logging.log4j:log4j-core:${libs.versions.log4j.get()}" testImplementation project(":jvm-libs:linea:blob-shnarf-calculator") } diff --git a/jvm-libs/linea/blob-compressor/src/main/kotlin/linea/blob/BlobCompressor.kt b/jvm-libs/linea/blob-compressor/src/main/kotlin/linea/blob/BlobCompressor.kt new file mode 100644 index 000000000..92c114ec7 --- /dev/null +++ b/jvm-libs/linea/blob-compressor/src/main/kotlin/linea/blob/BlobCompressor.kt @@ -0,0 +1,109 @@ +package linea.blob + +import net.consensys.encodeHex +import net.consensys.linea.blob.BlobCompressorVersion +import net.consensys.linea.blob.GoNativeBlobCompressor +import net.consensys.linea.blob.GoNativeBlobCompressorFactory +import org.apache.logging.log4j.LogManager + +class BlobCompressionException(message: String) : RuntimeException(message) + +interface BlobCompressor { + + /** + * @Throws(BlobCompressionException::class) when blockRLPEncoded is invalid + */ + fun canAppendBlock(blockRLPEncoded: ByteArray): Boolean + + /** + * @Throws(BlobCompressionException::class) when blockRLPEncoded is invalid + */ + fun appendBlock(blockRLPEncoded: ByteArray): AppendResult + + fun startNewBatch() + fun getCompressedData(): ByteArray + fun reset() + + data class AppendResult( + // returns false if last chunk would go over dataLimit. Does not append last block. + val blockAppended: Boolean, + val compressedSizeBefore: Int, + // even when block is not appended, compressedSizeAfter should as if it was appended + val compressedSizeAfter: Int + ) +} + +class GoBackedBlobCompressor private constructor( + internal val goNativeBlobCompressor: GoNativeBlobCompressor +) : BlobCompressor { + + companion object { + @Volatile + private var instance: GoBackedBlobCompressor? = null + + fun getInstance( + compressorVersion: BlobCompressorVersion = BlobCompressorVersion.V0_1_0, + dataLimit: UInt + ): GoBackedBlobCompressor { + if (instance == null) { + synchronized(this) { + if (instance == null) { + val goNativeBlobCompressor = GoNativeBlobCompressorFactory.getInstance(compressorVersion) + val initialized = goNativeBlobCompressor.Init( + dataLimit.toInt(), + GoNativeBlobCompressorFactory.dictionaryPath.toString() + ) + if (!initialized) { + throw InstantiationException(goNativeBlobCompressor.Error()) + } + instance = GoBackedBlobCompressor(goNativeBlobCompressor) + } else { + throw IllegalStateException("Compressor singleton instance already created") + } + } + } else { + throw IllegalStateException("Compressor singleton instance already created") + } + return instance!! + } + } + + private val log = LogManager.getLogger(GoBackedBlobCompressor::class.java) + + override fun canAppendBlock(blockRLPEncoded: ByteArray): Boolean { + return goNativeBlobCompressor.CanWrite(blockRLPEncoded, blockRLPEncoded.size) + } + + override fun appendBlock(blockRLPEncoded: ByteArray): BlobCompressor.AppendResult { + val compressionSizeBefore = goNativeBlobCompressor.Len() + val appended = goNativeBlobCompressor.Write(blockRLPEncoded, blockRLPEncoded.size) + val compressedSizeAfter = goNativeBlobCompressor.Len() + log.trace( + "block compressed: blockRlpSize={} compressionDataBefore={} compressionDataAfter={} compressionRatio={}", + blockRLPEncoded.size, + compressionSizeBefore, + compressedSizeAfter, + 1.0 - ((compressedSizeAfter - compressionSizeBefore).toDouble() / blockRLPEncoded.size) + ) + val error = goNativeBlobCompressor.Error() + if (error != null) { + log.error("Failure while writing the following RLP encoded block: {}", blockRLPEncoded.encodeHex()) + throw BlobCompressionException(error) + } + return BlobCompressor.AppendResult(appended, compressionSizeBefore, compressedSizeAfter) + } + + override fun startNewBatch() { + goNativeBlobCompressor.StartNewBatch() + } + + override fun getCompressedData(): ByteArray { + val compressedData = ByteArray(goNativeBlobCompressor.Len()) + goNativeBlobCompressor.Bytes(compressedData) + return compressedData + } + + override fun reset() { + goNativeBlobCompressor.Reset() + } +} diff --git a/jvm-libs/linea/blob-compressor/src/test/kotlin/linea/blob/GoBackedBlobCompressorTest.kt b/jvm-libs/linea/blob-compressor/src/test/kotlin/linea/blob/GoBackedBlobCompressorTest.kt new file mode 100644 index 000000000..83e3cd30d --- /dev/null +++ b/jvm-libs/linea/blob-compressor/src/test/kotlin/linea/blob/GoBackedBlobCompressorTest.kt @@ -0,0 +1,90 @@ +package linea.blob + +import net.consensys.linea.blob.BlobCompressorVersion +import net.consensys.linea.nativecompressor.CompressorTestData +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.assertThrows +import kotlin.random.Random + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class GoBackedBlobCompressorTest { + companion object { + private const val DATA_LIMIT = 16 * 1024 + private val TEST_DATA = CompressorTestData.blocksRlpEncoded + private val compressor = GoBackedBlobCompressor.getInstance(BlobCompressorVersion.V0_1_0, DATA_LIMIT.toUInt()) + } + + @BeforeEach + fun before() { + compressor.reset() + } + + @Test + fun `test appendBlock with data within limit`() { + val blocks = TEST_DATA + val result = compressor.appendBlock(blocks.first()) + assertThat(result.blockAppended).isTrue + assertThat(result.compressedSizeBefore).isZero() + assertThat(result.compressedSizeAfter).isGreaterThan(0) + } + + @Test + fun `test invalid rlp block`() { + val block = Random.nextBytes(100) + assertThrows("rlp: expected input list for types.extblock") { + compressor.appendBlock(block) + } + } + + @Test + fun `test compression data limit exceeded`() { + val blocks = TEST_DATA.iterator() + var result = compressor.appendBlock(blocks.next()) + while (result.blockAppended && blocks.hasNext()) { + val blockRlp = blocks.next() + val canAppend = compressor.canAppendBlock(blockRlp) + result = compressor.appendBlock(blockRlp) + // assert consistency between canAppendBlock and appendBlock + assertThat(canAppend).isEqualTo(result.blockAppended) + } + assertThat(result.blockAppended).isFalse() + assertThat(result.compressedSizeBefore).isGreaterThan(0) + assertThat(result.compressedSizeAfter).isEqualTo(result.compressedSizeBefore) + } + + @Test + fun `test reset`() { + val blocks = TEST_DATA.iterator() + assertThat(compressor.goNativeBlobCompressor.Len()).isZero() + var res = compressor.appendBlock(blocks.next()) + assertThat(res.blockAppended).isTrue() + assertThat(res.compressedSizeBefore).isZero() + assertThat(res.compressedSizeAfter).isGreaterThan(0) + assertThat(res.compressedSizeAfter).isEqualTo(compressor.goNativeBlobCompressor.Len()) + + compressor.reset() + + assertThat(compressor.goNativeBlobCompressor.Len()).isZero() + res = compressor.appendBlock(blocks.next()) + assertThat(res.blockAppended).isTrue() + assertThat(res.compressedSizeBefore).isZero() + assertThat(res.compressedSizeAfter).isGreaterThan(0) + assertThat(res.compressedSizeAfter).isEqualTo(compressor.goNativeBlobCompressor.Len()) + } + + @Test + fun `test batches`() { + val blocks = TEST_DATA.iterator() + var res = compressor.appendBlock(blocks.next()) + assertThat(res.blockAppended).isTrue() + + compressor.startNewBatch() + + res = compressor.appendBlock(blocks.next()) + assertThat(res.blockAppended).isTrue() + assertThat(compressor.getCompressedData().size).isGreaterThan(0) + } +} From db99fa1353f6e2d1889f24b143714707e9b5f40d Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:07:19 +0000 Subject: [PATCH 06/31] coordinator: update maven release runner --- .github/workflows/maven-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven-release.yml b/.github/workflows/maven-release.yml index 9299194ab..763cd56d8 100644 --- a/.github/workflows/maven-release.yml +++ b/.github/workflows/maven-release.yml @@ -18,7 +18,7 @@ on: jobs: release: - runs-on: [self-hosted, ubuntu-20.04, X64, small] + runs-on: [self-hosted, ubuntu-22.04, X64, small] steps: - name: Checkout code uses: actions/checkout@v4 From 386e64df05b1f42911f4dcee6d5459b8d99845cd Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:48:46 +0000 Subject: [PATCH 07/31] coordinator: update maven release runner --- .github/workflows/maven-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven-release.yml b/.github/workflows/maven-release.yml index 763cd56d8..e0300f898 100644 --- a/.github/workflows/maven-release.yml +++ b/.github/workflows/maven-release.yml @@ -18,7 +18,7 @@ on: jobs: release: - runs-on: [self-hosted, ubuntu-22.04, X64, small] + runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med steps: - name: Checkout code uses: actions/checkout@v4 From 8daa2d81e8f9ec6b3867e73ea641f4ef38b05ef4 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:19:04 +0000 Subject: [PATCH 08/31] coordinator: improver block encoder/decoder interfaces --- .../kotlin/linea/encoding/BlockRLPEncoder.kt | 3 +- .../kotlin/linea/rlp/BesuRLPBlockEncoder.kt | 40 --------------- .../kotlin/linea/rlp/BesuRlpMainnetEncoder.kt | 50 +++++++++++++++++++ .../linea/rlp/BlockRlpEncoderInterfaces.kt | 30 +++++++++++ 4 files changed, 82 insertions(+), 41 deletions(-) delete mode 100644 jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRLPBlockEncoder.kt create mode 100644 jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRlpMainnetEncoder.kt create mode 100644 jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BlockRlpEncoderInterfaces.kt diff --git a/coordinator/ethereum/models-helper/src/main/kotlin/linea/encoding/BlockRLPEncoder.kt b/coordinator/ethereum/models-helper/src/main/kotlin/linea/encoding/BlockRLPEncoder.kt index aa0290c3b..e7e40918c 100644 --- a/coordinator/ethereum/models-helper/src/main/kotlin/linea/encoding/BlockRLPEncoder.kt +++ b/coordinator/ethereum/models-helper/src/main/kotlin/linea/encoding/BlockRLPEncoder.kt @@ -1,8 +1,9 @@ package linea.encoding +import linea.domain.toBesu import linea.rlp.RLP import net.consensys.zkevm.encoding.BlockEncoder object BlockRLPEncoder : BlockEncoder { - override fun encode(block: linea.domain.Block): ByteArray = RLP.encode(block) + override fun encode(block: linea.domain.Block): ByteArray = RLP.encodeBlock(block.toBesu()) } diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRLPBlockEncoder.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRLPBlockEncoder.kt deleted file mode 100644 index 0e0db2917..000000000 --- a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRLPBlockEncoder.kt +++ /dev/null @@ -1,40 +0,0 @@ -package linea.rlp - -import io.vertx.core.Vertx -import linea.domain.Block -import net.consensys.linea.async.toSafeFuture -import tech.pegasys.teku.infrastructure.async.SafeFuture -import java.util.concurrent.Callable - -class BesuRLPBlockEncoder( - val vertx: Vertx -) : RLPBlockEncoderAsync { - - override fun encodeAsync(block: Block): SafeFuture { - return vertx.executeBlocking( - Callable { - RLP.encode(block) - }, - false - ) - .toSafeFuture() - } - - override fun encodeAsync(blocks: List): SafeFuture> { - return SafeFuture.collectAll(blocks.map { encodeAsync(it) }.stream()) - } - - override fun decodeAsync(block: ByteArray): SafeFuture { - return vertx.executeBlocking( - Callable { - RLP.decode(block) - }, - false - ) - .toSafeFuture() - } - - override fun decodeAsync(blocks: List): SafeFuture> { - return SafeFuture.collectAll(blocks.map { decodeAsync(it) }.stream()) - } -} diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRlpMainnetEncoder.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRlpMainnetEncoder.kt new file mode 100644 index 000000000..998e25b39 --- /dev/null +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRlpMainnetEncoder.kt @@ -0,0 +1,50 @@ +package linea.rlp + +import io.vertx.core.Vertx +import net.consensys.linea.async.toSafeFuture +import org.hyperledger.besu.ethereum.core.Block +import tech.pegasys.teku.infrastructure.async.SafeFuture +import java.util.concurrent.Callable + +object BesuMainnetBlockRlpEncoder : BesuBlockRlpEncoder { + override fun encode(block: Block): ByteArray = RLP.encodeBlock(block) +} + +object BesuMainnetBlockRlpDecoder : BesuBlockRlpDecoder { + override fun decode(block: ByteArray): Block = RLP.decodeBlockWithMainnetFunctions(block) +} + +class BesuRlpMainnetEncoderAsyncVertxImpl( + val vertx: Vertx, + val encoder: BesuBlockRlpEncoder = BesuMainnetBlockRlpEncoder +) : BesuBlockRlpEncoderAsync { + override fun encodeAsync(block: Block): SafeFuture { + return vertx.executeBlocking( + Callable { + encoder.encode(block) + }, + false + ) + .toSafeFuture() + } +} + +/** + * We can decode with Mainnet full functionality or + * with custom decoder for blob decompressed transactions without signature and blocks without header + * used for state reconstruction + */ +class BesuRlpDecoderAsyncVertxImpl( + val vertx: Vertx, + val decoder: BesuBlockRlpDecoder +) : BesuBlockRlpDecoderAsync { + override fun decodeAsync(block: ByteArray): SafeFuture { + return vertx.executeBlocking( + Callable { + decoder.decode(block) + }, + false + ) + .toSafeFuture() + } +} diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BlockRlpEncoderInterfaces.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BlockRlpEncoderInterfaces.kt new file mode 100644 index 000000000..e3270fa82 --- /dev/null +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BlockRlpEncoderInterfaces.kt @@ -0,0 +1,30 @@ +package linea.rlp + +import tech.pegasys.teku.infrastructure.async.SafeFuture + +interface BlockRlpEncoder { + fun encode(block: T): ByteArray + fun encode(blocks: List): List = blocks.map { encode(it) } +} + +interface BlockRlpDecoder { + fun decode(block: ByteArray): T + fun decode(blocks: List): List = blocks.map { decode(it) } +} + +interface BlockRlpEncoderAsync { + fun encodeAsync(block: T): SafeFuture + fun encodeAsync(blocks: List): SafeFuture> = + SafeFuture.collectAll(blocks.map { encodeAsync(it) }.stream()) +} + +interface BlockRlpDecoderAsync { + fun decodeAsync(block: ByteArray): SafeFuture + fun decodeAsync(blocks: List): SafeFuture> = + SafeFuture.collectAll(blocks.map { decodeAsync(it) }.stream()) +} + +interface BesuBlockRlpEncoder : BlockRlpEncoder +interface BesuBlockRlpEncoderAsync : BlockRlpEncoderAsync +interface BesuBlockRlpDecoder : BlockRlpDecoder +interface BesuBlockRlpDecoderAsync : BlockRlpDecoderAsync From 3632b12201fa331cafc094f1cf30e8dda7d6ef38 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:19:16 +0000 Subject: [PATCH 09/31] coordinator: improver block encoder/decoder interfaces --- .../src/main/kotlin/linea/rlp/RLP.kt | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/RLP.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/RLP.kt index 1922b6a89..ac9692833 100644 --- a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/RLP.kt +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/RLP.kt @@ -1,27 +1,17 @@ package linea.rlp -import linea.domain.toBesu -import linea.domain.toDomain import org.apache.tuweni.bytes.Bytes import org.hyperledger.besu.ethereum.core.Block import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput import org.hyperledger.besu.ethereum.rlp.RLP -object RLP : RLPBlockEncoder { - override fun encode(block: linea.domain.Block): ByteArray { - return encodeBlock(block.toBesu()) - } - - override fun decode(block: ByteArray): linea.domain.Block { - return decodeBlock(block).toDomain() - } - +object RLP { fun encodeBlock(besuBlock: org.hyperledger.besu.ethereum.core.Block): ByteArray { return besuBlock.toRlp().toArray() } - fun decodeBlock(block: ByteArray): org.hyperledger.besu.ethereum.core.Block { + fun decodeBlockWithMainnetFunctions(block: ByteArray): org.hyperledger.besu.ethereum.core.Block { return Block.readFrom( RLP.input(Bytes.wrap(block)), MainnetBlockHeaderFunctions() From 41f40b03638f9058db08ddced2cfa9d692bfa19e Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:19:28 +0000 Subject: [PATCH 10/31] coordinator: improver block encoder/decoder interfaces --- .../main/kotlin/linea/rlp/RLPBlockEncoder.kt | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/RLPBlockEncoder.kt diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/RLPBlockEncoder.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/RLPBlockEncoder.kt deleted file mode 100644 index c6a0506c9..000000000 --- a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/RLPBlockEncoder.kt +++ /dev/null @@ -1,18 +0,0 @@ -package linea.rlp - -import linea.domain.Block -import tech.pegasys.teku.infrastructure.async.SafeFuture - -interface RLPBlockEncoder { - fun encode(block: Block): ByteArray - fun encode(blocks: List): List = blocks.map { encode(it) } - fun decode(block: ByteArray): Block - fun decode(blocks: List): List = blocks.map { decode(it) } -} - -interface RLPBlockEncoderAsync { - fun encodeAsync(block: Block): SafeFuture - fun encodeAsync(blocks: List): SafeFuture> - fun decodeAsync(block: ByteArray): SafeFuture - fun decodeAsync(blocks: List): SafeFuture> -} From b00801788f15e9e295dc2c0a4c190cb889b2f8f1 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Tue, 10 Dec 2024 21:55:18 +0000 Subject: [PATCH 11/31] coordinator: fix web3j to domain mapper --- .../main/kotlin/linea/domain/Transaction.kt | 16 +++++++----- .../web3j/EthGetBlockToLineaBlockMappers.kt | 25 +++++++++++++------ .../EthGetBlockToLineaBlockMapperTest.kt | 16 ++++++------ 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/Transaction.kt b/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/Transaction.kt index 51e15c21c..1f5c2f4c2 100644 --- a/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/Transaction.kt +++ b/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/Transaction.kt @@ -21,6 +21,10 @@ enum class TransactionType(private val typeValue: Int) { return serializedType.compareTo(b!!) } + fun supports1559FeeMarket(): Boolean { + return !TransactionType.LEGACY_FEE_MARKET_TRANSACTION_TYPES.contains(this) + } + companion object { private val ACCESS_LIST_SUPPORTED_TRANSACTION_TYPES: Set = EnumSet.of(ACCESS_LIST, EIP1559, BLOB, DELEGATE_CODE) @@ -47,8 +51,8 @@ enum class TransactionType(private val typeValue: Int) { } data class Transaction( + val type: TransactionType, val nonce: ULong, - val gasPrice: ULong?, val gasLimit: ULong, val to: ByteArray?, // Nullable for contract creation transactions val value: BigInteger, @@ -57,10 +61,10 @@ data class Transaction( val s: BigInteger, val v: ULong, val yParity: ULong?, - val type: TransactionType, val chainId: ULong? = null, // Optional field for EIP-155 transactions - val maxPriorityFeePerGas: ULong? = null, // null for non EIP-1559 transactions + val gasPrice: ULong?, // null for EIP-1559 transactions val maxFeePerGas: ULong? = null, // null for EIP-1559 transactions + val maxPriorityFeePerGas: ULong? = null, // null for non EIP-1559 transactions val accessList: List? // null non for EIP-2930 transactions ) { companion object { @@ -116,8 +120,8 @@ data class Transaction( override fun toString(): String { return "Transaction(" + + "type=$type, " + "nonce=$nonce, " + - "gasPrice=$gasPrice, " + "gasLimit=$gasLimit, " + "to=${to?.encodeHex()}, " + "value=$value, " + @@ -126,10 +130,10 @@ data class Transaction( "s=$s, " + "v=$v, " + "yParity=$yParity, " + - "type=$type, " + "chainId=$chainId, " + - "maxPriorityFeePerGas=$maxPriorityFeePerGas, " + + "gasPrice=$gasPrice, " + "maxFeePerGas=$maxFeePerGas, " + + "maxPriorityFeePerGas=$maxPriorityFeePerGas, " + "accessList=$accessList)" } } diff --git a/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/EthGetBlockToLineaBlockMappers.kt b/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/EthGetBlockToLineaBlockMappers.kt index 01987c87c..d9ef8bebc 100644 --- a/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/EthGetBlockToLineaBlockMappers.kt +++ b/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/EthGetBlockToLineaBlockMappers.kt @@ -5,6 +5,7 @@ import linea.domain.Block import linea.domain.Transaction import linea.domain.TransactionType import net.consensys.decodeHex +import net.consensys.toBigIntegerFromHex import net.consensys.toIntFromHex import net.consensys.toULong import net.consensys.toULongFromHex @@ -46,10 +47,17 @@ fun mapToDomain(web3jBlock: EthBlock.Block): Block { } fun EthBlock.TransactionObject.toDomain(): Transaction { - val maxFeePerGas = this.maxFeePerGas?.toULong() - // Web3J throws an exception if maxPriorityFeePerGas null, instead of a check like in maxFeePerGas - // we need to check if maxFeePerGas is null to avoid the exception - val maxPriorityFeePerGas = if (maxFeePerGas != null) this.maxPriorityFeePerGas?.toULong() else null + val txType = mapType(this.type) + var gasPrice: ULong? = null + var maxFeePerGas: ULong? = null + var maxPriorityFeePerGas: ULong? = null + + if (txType.supports1559FeeMarket()) { + maxFeePerGas = this.maxFeePerGas?.toULong() + maxPriorityFeePerGas = this.maxPriorityFeePerGas?.toULong() + } else { + gasPrice = this.gasPrice.toULong() + } val accessList = this.accessList?.map { accessListEntry -> AccessListEntry( accessListEntry.address.decodeHex(), @@ -57,23 +65,24 @@ fun EthBlock.TransactionObject.toDomain(): Transaction { ) } - return Transaction( + val domainTx = Transaction( nonce = this.nonce.toULong(), - gasPrice = this.gasPrice.toULong(), gasLimit = this.gas.toULong(), to = this.to?.decodeHex(), value = this.value, input = this.input.decodeHex(), - r = this.r.removePrefix("0x").toBigInteger(16), - s = this.s.removePrefix("0x").toBigInteger(16), + r = this.r.toBigIntegerFromHex(), + s = this.s.toBigIntegerFromHex(), v = this.v.toULong(), yParity = this.getyParity()?.toULongFromHex(), type = mapType(this.type), // Optional field for EIP-2718 typed transactions chainId = this.chainId?.toULong(), // Optional field for EIP-155 transactions + gasPrice = gasPrice, // Optional field for EIP-1559 transactions maxFeePerGas = maxFeePerGas, // Optional field for EIP-1559 transactions maxPriorityFeePerGas = maxPriorityFeePerGas, // Optional field for EIP-1559 transactions, accessList = accessList ) + return domainTx } fun mapType(type: String?): TransactionType { diff --git a/jvm-libs/linea/web3j-extensions/src/test/kotlin/linea/web3j/EthGetBlockToLineaBlockMapperTest.kt b/jvm-libs/linea/web3j-extensions/src/test/kotlin/linea/web3j/EthGetBlockToLineaBlockMapperTest.kt index d071db16f..ecc4ef346 100644 --- a/jvm-libs/linea/web3j-extensions/src/test/kotlin/linea/web3j/EthGetBlockToLineaBlockMapperTest.kt +++ b/jvm-libs/linea/web3j-extensions/src/test/kotlin/linea/web3j/EthGetBlockToLineaBlockMapperTest.kt @@ -283,7 +283,7 @@ class EthGetBlockToLineaBlockMapperTest { "v": "0x1", "r": "0xeb4f70991ea4f14d23efb32591da3621d551406fd32bdfdd78bb677dec13160a", "s": "0x783aaa89f73ef7535924da8fd5f12e15cae1a0811c4c4746d1c23abff1eacddf", - "maxFeePerGas": "0x1017df87", + "maxFeePerGas": "0x1017dff7", "maxPriorityFeePerGas": "0x1017df87" } """.trimIndent() @@ -293,7 +293,9 @@ class EthGetBlockToLineaBlockMapperTest { assertThat(txDomain).isEqualTo( Transaction( nonce = 0UL, - gasPrice = 0x1017df87UL, + // when type is EIP1559 gasPrice is null, + // eth_getBlock returns effectiveGasPrice but we will place as null here + gasPrice = null, gasLimit = 0x5208UL, to = "0xe4392c8ecc46b304c83cdb5edaf742899b1bda93".decodeHex(), value = 0x2386f26fc10000UL.toBigInteger(), @@ -304,7 +306,7 @@ class EthGetBlockToLineaBlockMapperTest { yParity = 1UL, type = TransactionType.EIP1559, chainId = 0x539UL, - maxFeePerGas = 0x1017df87UL, + maxFeePerGas = 0x1017dff7UL, maxPriorityFeePerGas = 0x1017df87UL, accessList = emptyList() ) @@ -313,7 +315,7 @@ class EthGetBlockToLineaBlockMapperTest { txDomain.toBesu().also { txBesu -> assertThat(txBesu.type).isEqualTo(org.hyperledger.besu.datatypes.TransactionType.EIP1559) assertThat(txBesu.nonce).isEqualTo(0L) - assertThat(txBesu.gasPrice.getOrNull()).isEqualTo(Wei.of(0x1017df87L)) + assertThat(txBesu.gasPrice.getOrNull()).isNull() assertThat(txBesu.gasLimit).isEqualTo(0x5208L) assertThat(txBesu.to.getOrNull()).isEqualTo(Address.fromHexString("0xe4392c8ecc46b304c83cdb5edaf742899b1bda93")) assertThat(txBesu.value).isEqualTo(Wei.of(0x2386f26fc10000L)) @@ -326,7 +328,7 @@ class EthGetBlockToLineaBlockMapperTest { ) assertThat(txBesu.signature.recId).isEqualTo(1) assertThat(txBesu.chainId.getOrNull()).isEqualTo(0x539L) - assertThat(txBesu.maxFeePerGas.getOrNull()).isEqualTo(Wei.of(0x1017df87L)) + assertThat(txBesu.maxFeePerGas.getOrNull()).isEqualTo(Wei.of(0x1017dff7L)) assertThat(txBesu.maxPriorityFeePerGas.getOrNull()).isEqualTo(Wei.of(0x1017df87L)) assertThat(txBesu.accessList.getOrNull()).isEmpty() } @@ -368,7 +370,6 @@ class EthGetBlockToLineaBlockMapperTest { assertThat(txDomain).isEqualTo( Transaction( nonce = 1UL, - gasPrice = 0x7UL, gasLimit = 0x83a3dUL, to = null, value = 0UL.toBigInteger(), @@ -379,6 +380,7 @@ class EthGetBlockToLineaBlockMapperTest { yParity = 1UL, type = TransactionType.EIP1559, chainId = 0x539UL, + gasPrice = null, maxFeePerGas = 0xeUL, maxPriorityFeePerGas = 0UL, accessList = emptyList() @@ -388,7 +390,7 @@ class EthGetBlockToLineaBlockMapperTest { txDomain.toBesu().let { txBesu -> assertThat(txBesu.type).isEqualTo(org.hyperledger.besu.datatypes.TransactionType.EIP1559) assertThat(txBesu.nonce).isEqualTo(1L) - assertThat(txBesu.gasPrice.getOrNull()).isEqualTo(Wei.of(0x7L)) + assertThat(txBesu.gasPrice.getOrNull()).isNull() assertThat(txBesu.gasLimit).isEqualTo(0x83a3dL) assertThat(txBesu.to.getOrNull()).isNull() assertThat(txBesu.value).isEqualTo(Wei.ZERO) From 2ec3ab13c2d17a57e3b0936d4fb9720af8423aaf Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Tue, 10 Dec 2024 21:56:45 +0000 Subject: [PATCH 12/31] coordinator: increase maxExpectedCompressionRatio=20 --- .../main/kotlin/linea/rlp/BesuRlpMainnetEncoder.kt | 14 ++++++++++++-- .../linea/blob/GoNativeBlobDecompressor.kt | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRlpMainnetEncoder.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRlpMainnetEncoder.kt index 998e25b39..94603150f 100644 --- a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRlpMainnetEncoder.kt +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRlpMainnetEncoder.kt @@ -35,9 +35,19 @@ class BesuRlpMainnetEncoderAsyncVertxImpl( * used for state reconstruction */ class BesuRlpDecoderAsyncVertxImpl( - val vertx: Vertx, - val decoder: BesuBlockRlpDecoder + private val vertx: Vertx, + private val decoder: BesuBlockRlpDecoder ) : BesuBlockRlpDecoderAsync { + companion object { + fun mainnetDecoder(vertx: Vertx): BesuBlockRlpDecoderAsync { + return BesuRlpDecoderAsyncVertxImpl(vertx, BesuMainnetBlockRlpDecoder) + } + + fun blobDecoder(vertx: Vertx): BesuBlockRlpDecoderAsync { + return BesuRlpDecoderAsyncVertxImpl(vertx, BesuRlpBlobDecoder) + } + } + override fun decodeAsync(block: ByteArray): SafeFuture { return vertx.executeBlocking( Callable { diff --git a/jvm-libs/linea/blob-decompressor/src/main/kotlin/net/consensys/linea/blob/GoNativeBlobDecompressor.kt b/jvm-libs/linea/blob-decompressor/src/main/kotlin/net/consensys/linea/blob/GoNativeBlobDecompressor.kt index fa78dca2c..7b01cf0e8 100644 --- a/jvm-libs/linea/blob-decompressor/src/main/kotlin/net/consensys/linea/blob/GoNativeBlobDecompressor.kt +++ b/jvm-libs/linea/blob-decompressor/src/main/kotlin/net/consensys/linea/blob/GoNativeBlobDecompressor.kt @@ -13,7 +13,7 @@ interface BlobDecompressor { internal class Adapter( private val delegate: GoNativeBlobDecompressorJnaBinding, - private val maxExpectedCompressionRatio: Int = 10, + private val maxExpectedCompressionRatio: Int = 20, dictionaries: List ) : BlobDecompressor { init { From 805893706ec1d83bbe3f757d715b0461727acf9c Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Thu, 12 Dec 2024 10:20:43 +0000 Subject: [PATCH 13/31] coordinator: adds tool for backtesting compression/decompression --- .../kotlin/linea/rlp/BesuRlpBlobDecoder.kt | 43 +++++ .../rlp/NoSignatureTransactionDecoder.kt | 141 ++++++++++++++++ transaction-decoder-tool/build.gradle | 5 + .../linea/TransactionEncodingToolMain.kt | 30 ---- .../linea/test/BlockEncodingValidator.kt | 157 ++++++++++++++++++ .../test/kotlin/linea/test/BlocksFetcher.kt | 77 +++++++++ .../linea/test/FetchAndValidationRunner.kt | 67 ++++++++ .../src/test/kotlin/linea/test/MainRunner.kt | 58 +++++++ 8 files changed, 548 insertions(+), 30 deletions(-) create mode 100644 jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRlpBlobDecoder.kt create mode 100644 jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/NoSignatureTransactionDecoder.kt delete mode 100644 transaction-decoder-tool/src/main/kotlin/net/consensys/linea/TransactionEncodingToolMain.kt create mode 100644 transaction-decoder-tool/src/test/kotlin/linea/test/BlockEncodingValidator.kt create mode 100644 transaction-decoder-tool/src/test/kotlin/linea/test/BlocksFetcher.kt create mode 100644 transaction-decoder-tool/src/test/kotlin/linea/test/FetchAndValidationRunner.kt create mode 100644 transaction-decoder-tool/src/test/kotlin/linea/test/MainRunner.kt diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRlpBlobDecoder.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRlpBlobDecoder.kt new file mode 100644 index 000000000..47d14aafc --- /dev/null +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRlpBlobDecoder.kt @@ -0,0 +1,43 @@ +package linea.rlp + +import org.apache.tuweni.bytes.Bytes +import org.hyperledger.besu.ethereum.core.Block +import org.hyperledger.besu.ethereum.core.BlockBody +import org.hyperledger.besu.ethereum.core.BlockHeader +import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions +import org.hyperledger.besu.ethereum.core.Transaction +import org.hyperledger.besu.ethereum.core.Withdrawal +import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions +import org.hyperledger.besu.ethereum.rlp.RLPInput + +object BesuRlpBlobDecoder : BesuBlockRlpDecoder { + val transactionDecoder: NoSignatureTransactionDecoder = NoSignatureTransactionDecoder() + val hashFunction: BlockHeaderFunctions = MainnetBlockHeaderFunctions() + + override fun decode(block: ByteArray): Block { + return decode(org.hyperledger.besu.ethereum.rlp.RLP.input(Bytes.wrap(block)), hashFunction) + } + + fun decode(rlpInput: RLPInput, hashFunction: BlockHeaderFunctions): Block { + rlpInput.enterList() + + // Read the header + val header: BlockHeader = BlockHeader.readFrom(rlpInput, hashFunction) + + // Use NoSignatureTransactionDecoder to decode transactions + val transactions: List = rlpInput.readList(transactionDecoder::decode) + + // Read the ommers + val ommers: List = rlpInput.readList { rlp: RLPInput -> + BlockHeader.readFrom(rlp, hashFunction) + } + + // Read the withdrawals + if (!rlpInput.isEndOfCurrentList) { + rlpInput.readList(Withdrawal::readFrom) + } + + rlpInput.leaveList() + return Block(header, BlockBody(transactions, ommers)) + } +} diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/NoSignatureTransactionDecoder.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/NoSignatureTransactionDecoder.kt new file mode 100644 index 000000000..f04d418f3 --- /dev/null +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/NoSignatureTransactionDecoder.kt @@ -0,0 +1,141 @@ +package linea.rlp + +import org.apache.tuweni.bytes.Bytes +import org.hyperledger.besu.crypto.SECPSignature +import org.hyperledger.besu.datatypes.AccessListEntry +import org.hyperledger.besu.datatypes.Address +import org.hyperledger.besu.datatypes.TransactionType +import org.hyperledger.besu.datatypes.Wei +import org.hyperledger.besu.ethereum.core.Transaction +import org.hyperledger.besu.ethereum.rlp.RLP +import org.hyperledger.besu.ethereum.rlp.RLPInput +import java.math.BigInteger + +class NoSignatureTransactionDecoder { + fun decode(input: RLPInput): Transaction { + if (!input.nextIsList()) { + val typedTransactionBytes = input.readBytes() + val transactionInput = RLP.input(typedTransactionBytes.slice(1)) + val transactionType = typedTransactionBytes[0] + if (transactionType.toInt() == 0x01) { + return decodeAccessList(transactionInput) + } + if (transactionType.toInt() == 0x02) { + return decode1559(transactionInput) + } + throw IllegalArgumentException("Unsupported transaction type") + } else { // Frontier transaction + return decodeFrontier(input) + } + } + + private fun decodeAccessList(transactionInput: RLPInput): Transaction { + val builder = Transaction.builder() + + transactionInput.enterList() + builder + .type(TransactionType.ACCESS_LIST) + .chainId(BigInteger.valueOf(transactionInput.readLongScalar())) + .nonce(transactionInput.readLongScalar()) + .gasPrice(Wei.of(transactionInput.readUInt256Scalar())) + .gasLimit(transactionInput.readLongScalar()) + .to( + transactionInput + .readBytes { addressBytes: Bytes -> + if (addressBytes.isEmpty) null else Address.wrap(addressBytes) + } + ) + .value(Wei.of(transactionInput.readUInt256Scalar())) + .payload(transactionInput.readBytes()) + .accessList( + transactionInput.readList { accessListEntryRLPInput: RLPInput -> + accessListEntryRLPInput.enterList() + val accessListEntry = + AccessListEntry( + Address.wrap(accessListEntryRLPInput.readBytes()), + accessListEntryRLPInput.readList { obj: RLPInput -> obj.readBytes32() } + ) + accessListEntryRLPInput.leaveList() + accessListEntry + } + ) + transactionInput.readUnsignedByteScalar() + builder.sender(Address.extract(transactionInput.readUInt256Scalar())) + transactionInput.readUInt256Scalar() + transactionInput.leaveList() + return builder.signature(SECPSignature(BigInteger.ZERO, BigInteger.ZERO, 0.toByte())).build() + } + + private fun decode1559(transactionInput: RLPInput): Transaction { + val builder = Transaction.builder() + transactionInput.enterList() + val chainId = transactionInput.readBigIntegerScalar() + builder + .type(TransactionType.EIP1559) + .chainId(chainId) + .nonce(transactionInput.readLongScalar()) + .maxPriorityFeePerGas(Wei.of(transactionInput.readUInt256Scalar())) + .maxFeePerGas(Wei.of(transactionInput.readUInt256Scalar())) + .gasLimit(transactionInput.readLongScalar()) + .to( + transactionInput.readBytes { v: Bytes -> + if (v.isEmpty) { + null + } else { + Address.wrap( + v + ) + } + } + ) + .value(Wei.of(transactionInput.readUInt256Scalar())) + .payload(transactionInput.readBytes()) + .accessList( + transactionInput.readList { accessListEntryRLPInput: RLPInput -> + accessListEntryRLPInput.enterList() + val accessListEntry = + AccessListEntry( + Address.wrap(accessListEntryRLPInput.readBytes()), + accessListEntryRLPInput.readList { obj: RLPInput -> obj.readBytes32() } + ) + accessListEntryRLPInput.leaveList() + accessListEntry + } + ) + transactionInput.readUnsignedByteScalar() + builder.sender(Address.extract(transactionInput.readUInt256Scalar())) + transactionInput.readUInt256Scalar() + transactionInput.leaveList() + return builder.signature(SECPSignature(BigInteger.ZERO, BigInteger.ZERO, 0.toByte())).build() + } + + private fun decodeFrontier(input: RLPInput): Transaction { + val builder = Transaction.builder() + input.enterList() + builder + .type(TransactionType.FRONTIER) + .nonce(input.readLongScalar()) + .gasPrice(Wei.of(input.readUInt256Scalar())) + .gasLimit(input.readLongScalar()) + .to( + input.readBytes { v: Bytes -> + if (v.isEmpty) { + null + } else { + Address.wrap( + v + ) + } + } + ) + .value(Wei.of(input.readUInt256Scalar())) + .payload(input.readBytes()) + + input.readBigIntegerScalar() + builder.sender(Address.extract(input.readUInt256Scalar())) + input.readUInt256Scalar() + val signature = SECPSignature(BigInteger.ZERO, BigInteger.ZERO, 0.toByte()) + input.leaveList() + return builder.signature(signature).build() + } +} diff --git a/transaction-decoder-tool/build.gradle b/transaction-decoder-tool/build.gradle index 6346723aa..c2e435342 100644 --- a/transaction-decoder-tool/build.gradle +++ b/transaction-decoder-tool/build.gradle @@ -3,6 +3,11 @@ plugins { } dependencies { + implementation project(':jvm-libs:generic:extensions:futures') implementation project(':jvm-libs:linea:core:domain-models') + implementation project(':jvm-libs:linea:core:long-running-service') implementation project(':jvm-libs:linea:web3j-extensions') + implementation project(':jvm-libs:linea:blob-compressor') + implementation project(':jvm-libs:linea:blob-decompressor') + implementation project(':jvm-libs:linea:besu-rlp-and-mappers') } diff --git a/transaction-decoder-tool/src/main/kotlin/net/consensys/linea/TransactionEncodingToolMain.kt b/transaction-decoder-tool/src/main/kotlin/net/consensys/linea/TransactionEncodingToolMain.kt deleted file mode 100644 index ae3a3de47..000000000 --- a/transaction-decoder-tool/src/main/kotlin/net/consensys/linea/TransactionEncodingToolMain.kt +++ /dev/null @@ -1,30 +0,0 @@ -package net.consensys.linea - -import build.linea.web3j.domain.toWeb3j -import linea.web3j.toDomain -import net.consensys.linea.BlockParameter.Companion.toBlockParameter -import org.apache.logging.log4j.LogManager -import org.web3j.protocol.Web3j -import org.web3j.protocol.http.HttpService -import org.web3j.utils.Async - -class TransactionEncodingToolMain { - - companion object { - private val log = LogManager.getLogger(TransactionEncodingToolMain::class) - - @JvmStatic - fun main(args: Array) { - startApp() - } - - private fun startApp() { - val web3j: Web3j = Web3j.build( - HttpService("https://linea-sepolia.infura.io/v3/"), - 1000, - Async.defaultExecutorService() - ) - web3j.ethGetBlockByNumber(924973UL.toBlockParameter().toWeb3j(), true).sendAsync().get().block.toDomain() - } - } -} diff --git a/transaction-decoder-tool/src/test/kotlin/linea/test/BlockEncodingValidator.kt b/transaction-decoder-tool/src/test/kotlin/linea/test/BlockEncodingValidator.kt new file mode 100644 index 000000000..4b20a4d91 --- /dev/null +++ b/transaction-decoder-tool/src/test/kotlin/linea/test/BlockEncodingValidator.kt @@ -0,0 +1,157 @@ +package linea.test + +import io.vertx.core.Vertx +import linea.blob.GoBackedBlobCompressor +import linea.domain.Block +import linea.domain.toBesu +import linea.rlp.BesuRlpDecoderAsyncVertxImpl +import linea.rlp.BesuRlpMainnetEncoderAsyncVertxImpl +import linea.rlp.RLP +import net.consensys.linea.CommonDomainFunctions +import net.consensys.linea.blob.BlobCompressorVersion +import net.consensys.linea.blob.BlobDecompressorVersion +import net.consensys.linea.blob.GoNativeBlobDecompressorFactory +import net.consensys.zkevm.PeriodicPollingService +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.fail +import tech.pegasys.teku.infrastructure.async.SafeFuture +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.atomic.AtomicReference +import kotlin.jvm.optionals.getOrNull +import kotlin.time.Duration.Companion.seconds + +class BlockEncodingValidator( + val vertx: Vertx, + val compressorVersion: BlobCompressorVersion = BlobCompressorVersion.V1_0_1, + val decompressorVersion: BlobDecompressorVersion = BlobDecompressorVersion.V1_1_0, + val blobSizeLimitBytes: UInt = 1024u * 1024U, // 1MB, much larger than a real blob, but just for testing + val log: Logger = LogManager.getLogger(BlockEncodingValidator::class.java) +) : PeriodicPollingService(vertx, pollingIntervalMs = 1.seconds.inWholeMilliseconds, log = log) { + + val compressor = GoBackedBlobCompressor.getInstance(compressorVersion, blobSizeLimitBytes) + val decompressor = GoNativeBlobDecompressorFactory.getInstance(decompressorVersion) + val rlpEncoder = BesuRlpMainnetEncoderAsyncVertxImpl(vertx) + val rlpMainnetDecoder = BesuRlpDecoderAsyncVertxImpl.mainnetDecoder(vertx) + val rlpBlobDecoder = BesuRlpDecoderAsyncVertxImpl.blobDecoder(vertx) + val queueOfBlocksToValidate = ConcurrentLinkedQueue() + var highestValidatedBlockNumber = AtomicReference(ULong.MIN_VALUE) + + override fun action(): SafeFuture<*> { + return validateCycle() + } + + fun validateRlpEncodingDecoding(blocks: List): SafeFuture { + val besuBlocks = blocks.map { it.toBesu() } + return rlpEncoder.encodeAsync(besuBlocks) + .thenCompose { encodedBlocks -> + rlpMainnetDecoder.decodeAsync(encodedBlocks) + } + .thenApply { decodedBlocks -> + val unMatchingBlocks = besuBlocks.zip(decodedBlocks).filter { (expected, actual) -> expected != actual } + if (unMatchingBlocks.isEmpty()) { + log.info( + "all blocks encoding/decoding match: blocks={}", + CommonDomainFunctions.blockIntervalString(blocks.first().number, blocks.last().number) + ) + } else { + unMatchingBlocks.forEach { (expected, actual) -> + log.error( + "block encoding/decoding mismatch: block={} \nexpected={} \nactual={}", + expected.header.number, + expected, + actual + ) + } + } + } + } + + fun validateCompression(blocks: List): SafeFuture { + queueOfBlocksToValidate.addAll(blocks) + return SafeFuture.completedFuture(Unit) + } + + fun validateCycle(): SafeFuture { + val blocks = queueOfBlocksToValidate.pull(300) + if (blocks.isEmpty()) { + return SafeFuture.completedFuture(Unit) + } + log.info( + "compression validation blocks={} started", + CommonDomainFunctions.blockIntervalString(blocks.first().number, blocks.last().number) + ) + val besuBlocks = blocks.map { it.toBesu() } + return rlpEncoder.encodeAsync(besuBlocks) + .thenCompose { encodedBlocks -> + encodedBlocks.forEach { compressor.appendBlock(it) } + val compressedData = compressor.getCompressedData() + compressor.reset() + val decompressedData = decompressor.decompress(compressedData) + val decompressedBlocksList = RLP.decodeList(decompressedData) + rlpBlobDecoder.decodeAsync(decompressedBlocksList) + }.thenApply { decompressedBlocks -> + decompressedBlocks.zip(besuBlocks).forEach { (decompressed, original) -> + runCatching { + assertBlock(decompressed, original) + }.getOrElse { + log.error("Decompressed block={} does not match", original.header.number, it) + } + } + } + .thenPeek { + highestValidatedBlockNumber.set(highestValidatedBlockNumber.get().coerceAtLeast(blocks.last().number)) + log.info( + "compression validation blocks={} finished", + CommonDomainFunctions.blockIntervalString(blocks.first().number, blocks.last().number) + ) + } + } +} + +fun ConcurrentLinkedQueue.pull(elementsLimit: Int): List { + val elements = mutableListOf() + var element = poll() + while (element != null && elements.size < elementsLimit) { + elements.add(element) + element = poll() + } + return elements +} + +fun assertBlock( + decompressedBlock: org.hyperledger.besu.ethereum.core.Block, + originalBlock: org.hyperledger.besu.ethereum.core.Block +) { + // on decompression, the hash is placed as parentHash because besu recomputes the hash + assertThat(decompressedBlock.header.timestamp).isEqualTo(originalBlock.header.timestamp) + assertThat(decompressedBlock.header.parentHash).isEqualTo(originalBlock.header.hash) + + decompressedBlock.body.transactions.forEachIndexed { index, decompressedTx -> + val originalTx = originalBlock.body.transactions[index] + runCatching { + assertThat(decompressedTx.type).isEqualTo(originalTx.type) + assertThat(decompressedTx.sender).isEqualTo(originalTx.sender) + assertThat(decompressedTx.nonce).isEqualTo(originalTx.nonce) + assertThat(decompressedTx.gasLimit).isEqualTo(originalTx.gasLimit) + if (originalTx.type.supports1559FeeMarket()) { + assertThat(decompressedTx.maxFeePerGas).isEqualTo(originalTx.maxFeePerGas) + assertThat(decompressedTx.maxPriorityFeePerGas).isEqualTo(originalTx.maxPriorityFeePerGas) + } else { + assertThat(decompressedTx.gasPrice).isEqualTo(originalTx.gasPrice) + } + // FIXME: tmp work around until decompressor is fixed + originalTx.to.getOrNull()?.let { assertThat(decompressedTx.to.getOrNull()).isEqualTo(it) } + assertThat(decompressedTx.value).isEqualTo(originalTx.value) + assertThat(decompressedTx.data).isEqualTo(originalTx.data) + assertThat(decompressedTx.accessList).isEqualTo(originalTx.accessList) + }.getOrElse { th -> + fail( + "Transaction does not match: block=${originalBlock.header.number} " + + "txIndex=$index error=${th.message} origTxRlp=${originalTx.encoded()}", + th + ) + } + } +} diff --git a/transaction-decoder-tool/src/test/kotlin/linea/test/BlocksFetcher.kt b/transaction-decoder-tool/src/test/kotlin/linea/test/BlocksFetcher.kt new file mode 100644 index 000000000..f8f836917 --- /dev/null +++ b/transaction-decoder-tool/src/test/kotlin/linea/test/BlocksFetcher.kt @@ -0,0 +1,77 @@ +package linea.test + +import build.linea.web3j.domain.toWeb3j +import io.vertx.core.Vertx +import linea.domain.Block +import linea.web3j.toDomain +import net.consensys.linea.BlockParameter.Companion.toBlockParameter +import net.consensys.linea.async.AsyncRetryer +import net.consensys.linea.async.toSafeFuture +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.web3j.protocol.Web3j +import tech.pegasys.teku.infrastructure.async.SafeFuture +import java.util.concurrent.atomic.AtomicLong +import kotlin.time.Duration.Companion.milliseconds + +class BlocksFetcher( + val web3j: Web3j, + val vertx: Vertx = Vertx.vertx(), + val pollingChuckSize: UInt = 100U, + val log: Logger = LogManager.getLogger(BlocksFetcher::class.java) +) { + fun fetchBlocks( + startBlockNumber: ULong, + endBlockNumber: ULong + ): SafeFuture> { + return (startBlockNumber..endBlockNumber).toList() + .map { blockNumber -> + web3j.ethGetBlockByNumber(blockNumber.toBlockParameter().toWeb3j(), true) + .sendAsync() + .toSafeFuture() + .thenApply { + if (it.hasError()) { + log.error("Error fetching block={} errorMessage={}", blockNumber, it.error.message) + } + runCatching { + it.block.toDomain() + } + .getOrElse { + log.error("Error parsing block=$blockNumber", it) + null + } + } + } + .let { SafeFuture.collectAll(it.stream()) } + .thenApply { blocks: List -> + blocks.filterNotNull().sortedBy { it.number } + } + } + + fun consumeBlocks( + startBlockNumber: ULong, + chunkSize: UInt = pollingChuckSize, + endBlockNumber: ULong? = null, + consumer: (List) -> SafeFuture<*> + ): SafeFuture<*> { + val lastBlockFetched = AtomicLong(startBlockNumber.toLong() - 1) + return AsyncRetryer.retry( + vertx, + backoffDelay = 1000.milliseconds, + stopRetriesPredicate = { + endBlockNumber?.let { lastBlockFetched.get().toULong() >= it } ?: false + }, + stopRetriesOnErrorPredicate = { + it is Exception + } + ) { + val start = (lastBlockFetched.get() + 1).toULong() + val end = (start + chunkSize).coerceAtMost(endBlockNumber ?: ULong.MAX_VALUE) + fetchBlocks(start, end) + .thenCompose { blocks -> + lastBlockFetched.set(blocks.last().number.toLong()) + consumer(blocks) + } + } + } +} diff --git a/transaction-decoder-tool/src/test/kotlin/linea/test/FetchAndValidationRunner.kt b/transaction-decoder-tool/src/test/kotlin/linea/test/FetchAndValidationRunner.kt new file mode 100644 index 000000000..0991e31d6 --- /dev/null +++ b/transaction-decoder-tool/src/test/kotlin/linea/test/FetchAndValidationRunner.kt @@ -0,0 +1,67 @@ +package linea.test + +import io.vertx.core.Vertx +import linea.web3j.createWeb3jHttpClient +import net.consensys.linea.CommonDomainFunctions +import org.apache.logging.log4j.Level +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.web3j.protocol.Web3j +import tech.pegasys.teku.infrastructure.async.SafeFuture +import java.util.concurrent.atomic.AtomicReference + +class FetchAndValidationRunner( + val vertx: Vertx = Vertx.vertx(), + val rpcUrl: String, + val log: Logger = LogManager.getLogger(FetchAndValidationRunner::class.java) +) { + val web3j: Web3j = createWeb3jHttpClient( + rpcUrl = rpcUrl, +// executorService = vertx.nettyEventLoopGroup(), + log = LogManager.getLogger("test.client.web3j"), + requestResponseLogLevel = Level.DEBUG, + failuresLogLevel = Level.ERROR + ) + val validator = BlockEncodingValidator(vertx = vertx, log = log).also { it.start() } + val blocksFetcher = BlocksFetcher(web3j, log = log) + val targetEndBlockNumber = AtomicReference() + + fun awaitValidationFinishes(): SafeFuture { + val result = SafeFuture() + vertx.setPeriodic(2000) { timerId -> + if (targetEndBlockNumber.get() != null && + validator.highestValidatedBlockNumber.get() >= targetEndBlockNumber.get()!! + ) { + vertx.cancelTimer(timerId) + validator.stop() + web3j.shutdown() + result.complete(Unit) + } + } + return result + } + + fun fetchAndValidateBlocks( + startBlockNumber: ULong, + endBlockNumber: ULong? = null, + chuckSize: UInt = 100U, + rlpEncodingDecodingOnly: Boolean = false + ): SafeFuture<*> { + targetEndBlockNumber.set(endBlockNumber) + return blocksFetcher.consumeBlocks( + startBlockNumber = startBlockNumber, + endBlockNumber = endBlockNumber, + chunkSize = chuckSize + ) { blocks -> + log.info( + "got blocks: {}", + CommonDomainFunctions.blockIntervalString(blocks.first().number, blocks.last().number) + ) + if (rlpEncodingDecodingOnly) { + validator.validateRlpEncodingDecoding(blocks) + } else { + validator.validateCompression(blocks) + } + } + } +} diff --git a/transaction-decoder-tool/src/test/kotlin/linea/test/MainRunner.kt b/transaction-decoder-tool/src/test/kotlin/linea/test/MainRunner.kt new file mode 100644 index 000000000..049cae7de --- /dev/null +++ b/transaction-decoder-tool/src/test/kotlin/linea/test/MainRunner.kt @@ -0,0 +1,58 @@ +package linea.test + +import io.vertx.core.Vertx +import net.consensys.linea.async.get +import org.apache.logging.log4j.Level +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.core.config.Configurator +import java.util.concurrent.TimeUnit + +fun configureLoggers(loggerConfigs: List>) { + loggerConfigs.forEach { (loggerName, level) -> + Configurator.setLevel(loggerName, level) + } +} + +fun main() { + val lineaSepoliaUrl = "https://linea-sepolia.infura.io/v3/${System.getenv("INFURA_PROJECT_ID")}" +// val lineaMainnetUrl = "https://linea-mainnet.infura.io/v3/${System.getenv("INFURA_PROJECT_ID")}" + val vertx = Vertx.vertx() + vertx.exceptionHandler { error -> + println("Unhandled exception: message=${error.message}") + LogManager.getLogger("vertx").error("Unhandled exception: message={}", error.message, error) + } + val fetcherAndValidate = + FetchAndValidationRunner( + rpcUrl = lineaSepoliaUrl, + vertx = vertx, + log = LogManager.getLogger("test.validator") + ) + configureLoggers( + listOf( + "test.client.web3j" to Level.INFO, + "test.validator" to Level.INFO + ) + ) + +// val startBlockNumber = 924973UL +// val startBlockNumber = 924976UL +// val startBlockNumber = 929527UL + val startBlockNumber = 100_034UL // encoding/decoding does not match +// val startBlockNumber = 100_000UL + runCatching { + fetcherAndValidate.fetchAndValidateBlocks( + startBlockNumber = startBlockNumber, +// endBlockNumber = startBlockNumber + 5000u, + endBlockNumber = startBlockNumber + 0u, + chuckSize = 500U, + rlpEncodingDecodingOnly = false + ).get(2, TimeUnit.MINUTES) + }.onFailure { error -> + fetcherAndValidate.log.error("Error fetching and validating blocks", error) + } + fetcherAndValidate.awaitValidationFinishes().get() + println("waited validation finishes") +// fetcherAndValidate.validator.stop() + vertx.close().get() + println("closed vertx") +} From 2da1710782921942a6e27ed89a94fe0ae62172bd Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Thu, 12 Dec 2024 14:38:35 +0000 Subject: [PATCH 14/31] coordinator: fixes on Block decoder from L1 Blobs --- .../kotlin/linea/rlp/BesuRlpBlobDecoder.kt | 17 +++++++++-- .../linea/test/BlockEncodingValidator.kt | 28 ++++++++++++++++--- .../test/kotlin/linea/test/BlocksFetcher.kt | 2 +- .../src/test/kotlin/linea/test/MainRunner.kt | 25 +++++++++-------- 4 files changed, 54 insertions(+), 18 deletions(-) diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRlpBlobDecoder.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRlpBlobDecoder.kt index 47d14aafc..a9d2e9dcf 100644 --- a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRlpBlobDecoder.kt +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BesuRlpBlobDecoder.kt @@ -1,20 +1,33 @@ package linea.rlp +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger import org.apache.tuweni.bytes.Bytes +import org.hyperledger.besu.datatypes.Hash import org.hyperledger.besu.ethereum.core.Block import org.hyperledger.besu.ethereum.core.BlockBody import org.hyperledger.besu.ethereum.core.BlockHeader import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions +import org.hyperledger.besu.ethereum.core.ParsedExtraData import org.hyperledger.besu.ethereum.core.Transaction import org.hyperledger.besu.ethereum.core.Withdrawal -import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions import org.hyperledger.besu.ethereum.rlp.RLPInput object BesuRlpBlobDecoder : BesuBlockRlpDecoder { + val log: Logger = LogManager.getLogger(BesuRlpBlobDecoder::class.java) val transactionDecoder: NoSignatureTransactionDecoder = NoSignatureTransactionDecoder() - val hashFunction: BlockHeaderFunctions = MainnetBlockHeaderFunctions() + + // 1.Decompressor places Block's hash in parentHash + // Because we are reusing Geth/Besu rlp encoding that recalculate the hashes. + // so here we override the hash function to use the parentHash as the hash + // 2. we don't compresse extraData, so just returning null + val hashFunction: BlockHeaderFunctions = object : BlockHeaderFunctions { + override fun hash(blockHeader: BlockHeader): Hash = blockHeader.parentHash + override fun parseExtraData(blockHeader: BlockHeader): ParsedExtraData? = null + } override fun decode(block: ByteArray): Block { + log.trace("Decoding block from RLP blob: rawRlpSize={}", block.size) return decode(org.hyperledger.besu.ethereum.rlp.RLP.input(Bytes.wrap(block)), hashFunction) } diff --git a/transaction-decoder-tool/src/test/kotlin/linea/test/BlockEncodingValidator.kt b/transaction-decoder-tool/src/test/kotlin/linea/test/BlockEncodingValidator.kt index 4b20a4d91..cbde61085 100644 --- a/transaction-decoder-tool/src/test/kotlin/linea/test/BlockEncodingValidator.kt +++ b/transaction-decoder-tool/src/test/kotlin/linea/test/BlockEncodingValidator.kt @@ -92,11 +92,20 @@ class BlockEncodingValidator( val decompressedBlocksList = RLP.decodeList(decompressedData) rlpBlobDecoder.decodeAsync(decompressedBlocksList) }.thenApply { decompressedBlocks -> + assertThat(decompressedBlocks.size).isEqualTo(besuBlocks.size) + .withFailMessage( + "decompressedBlocks.size=${decompressedBlocks.size} != originalBlocks.size=${besuBlocks.size}" + ) decompressedBlocks.zip(besuBlocks).forEach { (decompressed, original) -> runCatching { assertBlock(decompressed, original) }.getOrElse { - log.error("Decompressed block={} does not match", original.header.number, it) + log.error( + "Decompressed block={} does not match: error={}", + original.header.number, + it.message, + it + ) } } } @@ -122,14 +131,25 @@ fun ConcurrentLinkedQueue.pull(elementsLimit: Int): List { fun assertBlock( decompressedBlock: org.hyperledger.besu.ethereum.core.Block, - originalBlock: org.hyperledger.besu.ethereum.core.Block + originalBlock: org.hyperledger.besu.ethereum.core.Block, + log: Logger = LogManager.getLogger("test.assert.Block") ) { // on decompression, the hash is placed as parentHash because besu recomputes the hash + // but custom decoder overrides hash calculation to use parentHash assertThat(decompressedBlock.header.timestamp).isEqualTo(originalBlock.header.timestamp) - assertThat(decompressedBlock.header.parentHash).isEqualTo(originalBlock.header.hash) + assertThat(decompressedBlock.header.hash).isEqualTo(originalBlock.header.hash) decompressedBlock.body.transactions.forEachIndexed { index, decompressedTx -> val originalTx = originalBlock.body.transactions[index] + log.trace( + "block={} txIndex={} \n originalTx={} \n decodedTx={} \n originalTxRlp={}", + originalBlock.header.number, + index, + originalTx, + decompressedTx, + + originalTx.encoded() + ) runCatching { assertThat(decompressedTx.type).isEqualTo(originalTx.type) assertThat(decompressedTx.sender).isEqualTo(originalTx.sender) @@ -144,8 +164,8 @@ fun assertBlock( // FIXME: tmp work around until decompressor is fixed originalTx.to.getOrNull()?.let { assertThat(decompressedTx.to.getOrNull()).isEqualTo(it) } assertThat(decompressedTx.value).isEqualTo(originalTx.value) - assertThat(decompressedTx.data).isEqualTo(originalTx.data) assertThat(decompressedTx.accessList).isEqualTo(originalTx.accessList) + assertThat(decompressedTx.payload).isEqualTo(originalTx.payload) }.getOrElse { th -> fail( "Transaction does not match: block=${originalBlock.header.number} " + diff --git a/transaction-decoder-tool/src/test/kotlin/linea/test/BlocksFetcher.kt b/transaction-decoder-tool/src/test/kotlin/linea/test/BlocksFetcher.kt index f8f836917..acb88c156 100644 --- a/transaction-decoder-tool/src/test/kotlin/linea/test/BlocksFetcher.kt +++ b/transaction-decoder-tool/src/test/kotlin/linea/test/BlocksFetcher.kt @@ -66,7 +66,7 @@ class BlocksFetcher( } ) { val start = (lastBlockFetched.get() + 1).toULong() - val end = (start + chunkSize).coerceAtMost(endBlockNumber ?: ULong.MAX_VALUE) + val end = (start + chunkSize - 1U).coerceAtMost(endBlockNumber ?: ULong.MAX_VALUE) fetchBlocks(start, end) .thenCompose { blocks -> lastBlockFetched.set(blocks.last().number.toLong()) diff --git a/transaction-decoder-tool/src/test/kotlin/linea/test/MainRunner.kt b/transaction-decoder-tool/src/test/kotlin/linea/test/MainRunner.kt index 049cae7de..a2f2aa47a 100644 --- a/transaction-decoder-tool/src/test/kotlin/linea/test/MainRunner.kt +++ b/transaction-decoder-tool/src/test/kotlin/linea/test/MainRunner.kt @@ -14,8 +14,10 @@ fun configureLoggers(loggerConfigs: List>) { } fun main() { - val lineaSepoliaUrl = "https://linea-sepolia.infura.io/v3/${System.getenv("INFURA_PROJECT_ID")}" -// val lineaMainnetUrl = "https://linea-mainnet.infura.io/v3/${System.getenv("INFURA_PROJECT_ID")}" + val rplUrl = run { + "https://linea-sepolia.infura.io/v3/${System.getenv("INFURA_PROJECT_ID")}" +// "https://linea-mainnet.infura.io/v3/${System.getenv("INFURA_PROJECT_ID")}" + } val vertx = Vertx.vertx() vertx.exceptionHandler { error -> println("Unhandled exception: message=${error.message}") @@ -23,28 +25,29 @@ fun main() { } val fetcherAndValidate = FetchAndValidationRunner( - rpcUrl = lineaSepoliaUrl, + rpcUrl = rplUrl, vertx = vertx, log = LogManager.getLogger("test.validator") ) configureLoggers( listOf( + "linea.rlp" to Level.INFO, "test.client.web3j" to Level.INFO, "test.validator" to Level.INFO ) ) -// val startBlockNumber = 924973UL -// val startBlockNumber = 924976UL -// val startBlockNumber = 929527UL - val startBlockNumber = 100_034UL // encoding/decoding does not match -// val startBlockNumber = 100_000UL + // Sepolia Blocks + val startBlockNumber = 924_973UL +// val startBlockNumber = 5_099_599UL + // Mainnet Blocks +// val startBlockNumber = 10_000_308UL runCatching { fetcherAndValidate.fetchAndValidateBlocks( startBlockNumber = startBlockNumber, -// endBlockNumber = startBlockNumber + 5000u, - endBlockNumber = startBlockNumber + 0u, - chuckSize = 500U, + endBlockNumber = startBlockNumber + 1_000_000U, +// endBlockNumber = startBlockNumber + 0u, + chuckSize = 1_000U, rlpEncodingDecodingOnly = false ).get(2, TimeUnit.MINUTES) }.onFailure { error -> From 75d11bb97f9f8f70c41bc249ec45c94df9557909 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Fri, 13 Dec 2024 20:51:50 +0000 Subject: [PATCH 15/31] coordinator: adds TestingJsonRpcServer --- jvm-libs/generic/json-rpc/build.gradle | 1 + .../linea/jsonrpc/JsonRpcMessageProcessor.kt | 16 +- .../linea/jsonrpc/JsonRpcResponse.kt | 5 +- .../jsonrpc/httpserver/HttpJsonRpcServer.kt | 15 +- .../linea/jsonrpc/TestingJsonRpcServerTest.kt | 123 ++++++++++++ .../jsonrpc/JsonRpcMessageProcessorTest.kt | 1 + .../linea/jsonrpc/TestingJsonRpcServer.kt | 175 ++++++++++++++++++ 7 files changed, 327 insertions(+), 9 deletions(-) create mode 100644 jvm-libs/generic/json-rpc/src/test/kotlin/linea/jsonrpc/TestingJsonRpcServerTest.kt create mode 100644 jvm-libs/generic/json-rpc/src/testFixtures/kotlin/linea/jsonrpc/TestingJsonRpcServer.kt diff --git a/jvm-libs/generic/json-rpc/build.gradle b/jvm-libs/generic/json-rpc/build.gradle index e2e4c4337..f52a677f3 100644 --- a/jvm-libs/generic/json-rpc/build.gradle +++ b/jvm-libs/generic/json-rpc/build.gradle @@ -1,6 +1,7 @@ plugins { id 'net.consensys.zkevm.kotlin-library-conventions' id 'java-library' + id 'java-test-fixtures' } description = "JSON RPC 2.0 utilities" diff --git a/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/JsonRpcMessageProcessor.kt b/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/JsonRpcMessageProcessor.kt index ebe6c8109..563402861 100644 --- a/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/JsonRpcMessageProcessor.kt +++ b/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/JsonRpcMessageProcessor.kt @@ -1,5 +1,7 @@ package net.consensys.linea.jsonrpc +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import com.github.michaelbull.result.Err import com.github.michaelbull.result.Ok @@ -51,12 +53,14 @@ private data class RequestContext( class JsonRpcMessageProcessor( private val requestsHandler: JsonRpcRequestHandler, private val meterRegistry: MeterRegistry, - private val requestParser: JsonRpcRequestParser = Companion::parseRequest + private val requestParser: JsonRpcRequestParser = Companion::parseRequest, + private val log: Logger = LogManager.getLogger(JsonRpcMessageProcessor::class.java), + private val responseObjectMapper: ObjectMapper = ObjectMapper().registerKotlinModule() ) : JsonRpcMessageHandler { init { DatabindCodec.mapper().registerKotlinModule() } - private val log: Logger = LogManager.getLogger(this.javaClass) + private val counterBuilder = Counter.builder("jsonrpc.counter") override fun invoke(user: User?, messageJsonStr: String): Future = handleMessage(user, messageJsonStr) @@ -174,7 +178,13 @@ class JsonRpcMessageProcessor( return SimpleTimerCapture(meterRegistry, "jsonrpc.serialization.response") .setDescription("Time of json response serialization") .setTag("method", requestContext.method) - .captureTime { Json.encode(requestContext.result.merge()) } + .captureTime { + val result = requestContext.result.map { successResponse -> + val resultJsonNode = responseObjectMapper.valueToTree(successResponse.result) + successResponse.copy(result = resultJsonNode) + } + Json.encode(result.merge()) + } } private fun handleRequest( diff --git a/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/JsonRpcResponse.kt b/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/JsonRpcResponse.kt index 6fbebe302..b244bf500 100644 --- a/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/JsonRpcResponse.kt +++ b/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/JsonRpcResponse.kt @@ -32,6 +32,7 @@ data class JsonRpcSuccessResponse( val result: Any? ) : JsonRpcResponse(jsonrpc, id) { constructor(id: Any, result: Any?) : this("2.0", id, result) + constructor(request: JsonRpcRequest, result: Any?) : this(request.jsonrpc, id = request.id, result) } @JsonPropertyOrder("jsonrpc", "id", "error") @@ -112,4 +113,6 @@ class JsonRpcErrorResponseException( val rpcErrorCode: Int, val rpcErrorMessage: String, val rpcErrorData: Any? = null -) : RuntimeException("code=$rpcErrorCode message=$rpcErrorMessage errorData=$rpcErrorData") +) : RuntimeException("code=$rpcErrorCode message=$rpcErrorMessage errorData=$rpcErrorData") { + fun asJsonRpcError(): JsonRpcError = JsonRpcError(rpcErrorCode, rpcErrorMessage, rpcErrorData) +} diff --git a/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/httpserver/HttpJsonRpcServer.kt b/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/httpserver/HttpJsonRpcServer.kt index 503b51adc..40ec62416 100644 --- a/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/httpserver/HttpJsonRpcServer.kt +++ b/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/httpserver/HttpJsonRpcServer.kt @@ -14,11 +14,11 @@ import org.apache.logging.log4j.Logger class HttpJsonRpcServer( private val port: UInt, private val path: String, - private val requestHandler: Handler + private val requestHandler: Handler, + val serverName: String = "" ) : AbstractVerticle() { private val log: Logger = LogManager.getLogger(this.javaClass) private lateinit var httpServer: HttpServer - val bindedPort: Int get() = if (this::httpServer.isInitialized) { httpServer.actualPort() @@ -28,15 +28,19 @@ class HttpJsonRpcServer( override fun start(startPromise: Promise) { val options = HttpServerOptions().setPort(port.toInt()).setReusePort(true) - log.debug("Creating Http server on port {}", port) + log.debug("creating {} Http server on port {}", port) httpServer = vertx.createHttpServer(options) httpServer.requestHandler(buildRouter()) httpServer.listen { res: AsyncResult -> if (res.succeeded()) { - log.info("Http server started and listening on port {}", res.result().actualPort()) + log.info( + "{} http server started and listening on port {}", + serverName, + res.result().actualPort() + ) startPromise.complete() } else { - log.error("Creating Http server: {}", res.cause()) + log.error("error creating {} http server: {}", serverName, res.cause()) startPromise.fail(res.cause()) } } @@ -50,5 +54,6 @@ class HttpJsonRpcServer( override fun stop(endFuture: Promise) { httpServer.close(endFuture) + super.stop(endFuture) } } diff --git a/jvm-libs/generic/json-rpc/src/test/kotlin/linea/jsonrpc/TestingJsonRpcServerTest.kt b/jvm-libs/generic/json-rpc/src/test/kotlin/linea/jsonrpc/TestingJsonRpcServerTest.kt new file mode 100644 index 000000000..033e15502 --- /dev/null +++ b/jvm-libs/generic/json-rpc/src/test/kotlin/linea/jsonrpc/TestingJsonRpcServerTest.kt @@ -0,0 +1,123 @@ +package linea.jsonrpc + +import com.github.michaelbull.result.Err +import com.github.michaelbull.result.Ok +import io.micrometer.core.instrument.simple.SimpleMeterRegistry +import io.vertx.junit5.VertxExtension +import net.consensys.linea.async.get +import net.consensys.linea.jsonrpc.JsonRpcErrorResponse +import net.consensys.linea.jsonrpc.JsonRpcErrorResponseException +import net.consensys.linea.jsonrpc.JsonRpcSuccessResponse +import net.consensys.linea.jsonrpc.client.JsonRpcV2Client +import net.consensys.linea.jsonrpc.client.RequestRetryConfig +import net.consensys.linea.jsonrpc.client.VertxHttpJsonRpcClientFactory +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import java.net.URI +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.minutes + +@ExtendWith(VertxExtension::class) +class TestingJsonRpcServerTest { + private lateinit var jsonRpcServer: TestingJsonRpcServer + private lateinit var client: JsonRpcV2Client + + @BeforeEach + fun beforeEach(vertx: io.vertx.core.Vertx) { + jsonRpcServer = TestingJsonRpcServer( + vertx = vertx, + recordRequestsResponses = true + ) + val rpcClientFactory = VertxHttpJsonRpcClientFactory( + vertx = vertx, + meterRegistry = SimpleMeterRegistry() + ) + client = rpcClientFactory.createJsonRpcV2Client( + endpoints = listOf(URI.create("http://localhost:${jsonRpcServer.bindedPort}")), + retryConfig = RequestRetryConfig( + maxRetries = 10u, + backoffDelay = 10.milliseconds, + timeout = 2.minutes + ), + shallRetryRequestsClientBasePredicate = { + false + } // disable retry + ) + } + + @Test + fun `when no method handler is defined returns method not found`() { + assertThatThrownBy { + client.makeRequest( + method = "not_existing_method", + params = mapOf("k1" to "v1", "k2" to 100), + resultMapper = { it }, + shallRetryRequestPredicate = { false } + ).get() + }.hasCauseInstanceOf(JsonRpcErrorResponseException::class.java) + .hasMessageContaining("Method not found") + + // check recorded request + jsonRpcServer.recordedRequests().also { + assertThat(it).hasSize(1) + val (request, responseFuture) = it[0] + assertThat(request.method).isEqualTo("not_existing_method") + assertThat(request.params).isEqualTo(mapOf("k1" to "v1", "k2" to 100)) + assertThat(responseFuture.get()).isEqualTo( + Err(JsonRpcErrorResponse.methodNotFound(request.id, data = "not_existing_method")) + ) + } + } + + @Test + fun `when handlers are provided shall forward to correct one`() { + jsonRpcServer.handle("add") { request -> + @Suppress("UNCHECKED_CAST") + val params = request.params as List + params.sumOf { it } + } + jsonRpcServer.handle("addUser") { request -> + @Suppress("UNCHECKED_CAST") + val params = request.params as Map + "user=${params["name"]} email=${params["email"]}" + } + jsonRpcServer.handle("multiply") { _ -> "not expected" } + + assertThat( + client.makeRequest( + method = "add", + params = listOf(1, 2, 3), + resultMapper = { it } + ).get() + ) + .isEqualTo(6) + + assertThat( + client.makeRequest( + method = "addUser", + params = mapOf("name" to "John", "email" to "john@email.com"), + resultMapper = { it } + ).get() + ) + .isEqualTo("user=John email=john@email.com") + + // check recorded request + jsonRpcServer.recordedRequests().also { + assertThat(it).hasSize(2) + it[0].also { (request, responseFuture) -> + assertThat(request.method).isEqualTo("add") + assertThat(request.params).isEqualTo(listOf(1, 2, 3)) + assertThat(responseFuture.get()).isEqualTo(Ok(JsonRpcSuccessResponse(id = request.id, result = 6))) + } + it[1].also { (request, responseFuture) -> + assertThat(request.method).isEqualTo("addUser") + assertThat(request.params).isEqualTo(mapOf("name" to "John", "email" to "john@email.com")) + assertThat(responseFuture.get()) + .isEqualTo(Ok(JsonRpcSuccessResponse(id = request.id, result = "user=John email=john@email.com"))) + } + } + } +} diff --git a/jvm-libs/generic/json-rpc/src/test/kotlin/net/consensys/linea/jsonrpc/JsonRpcMessageProcessorTest.kt b/jvm-libs/generic/json-rpc/src/test/kotlin/net/consensys/linea/jsonrpc/JsonRpcMessageProcessorTest.kt index 4bec02f53..e1be9320a 100644 --- a/jvm-libs/generic/json-rpc/src/test/kotlin/net/consensys/linea/jsonrpc/JsonRpcMessageProcessorTest.kt +++ b/jvm-libs/generic/json-rpc/src/test/kotlin/net/consensys/linea/jsonrpc/JsonRpcMessageProcessorTest.kt @@ -10,6 +10,7 @@ import io.vertx.core.json.JsonObject import io.vertx.ext.auth.User import io.vertx.junit5.VertxExtension import io.vertx.junit5.VertxTestContext +import linea.jsonrpc2.JsonRpcMessageProcessor import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test diff --git a/jvm-libs/generic/json-rpc/src/testFixtures/kotlin/linea/jsonrpc/TestingJsonRpcServer.kt b/jvm-libs/generic/json-rpc/src/testFixtures/kotlin/linea/jsonrpc/TestingJsonRpcServer.kt new file mode 100644 index 000000000..3dc29502c --- /dev/null +++ b/jvm-libs/generic/json-rpc/src/testFixtures/kotlin/linea/jsonrpc/TestingJsonRpcServer.kt @@ -0,0 +1,175 @@ +package linea.jsonrpc + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.github.michaelbull.result.Err +import com.github.michaelbull.result.Ok +import com.github.michaelbull.result.Result +import io.micrometer.core.instrument.simple.SimpleMeterRegistry +import io.vertx.core.DeploymentOptions +import io.vertx.core.Future +import io.vertx.core.Promise +import io.vertx.core.Vertx +import io.vertx.core.json.JsonObject +import io.vertx.ext.auth.User +import net.consensys.linea.async.get +import net.consensys.linea.jsonrpc.HttpRequestHandler +import net.consensys.linea.jsonrpc.JsonRpcErrorResponse +import net.consensys.linea.jsonrpc.JsonRpcErrorResponseException +import net.consensys.linea.jsonrpc.JsonRpcMessageProcessor +import net.consensys.linea.jsonrpc.JsonRpcRequest +import net.consensys.linea.jsonrpc.JsonRpcSuccessResponse +import net.consensys.linea.jsonrpc.httpserver.HttpJsonRpcServer +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import tech.pegasys.teku.infrastructure.async.SafeFuture +import java.util.concurrent.ConcurrentHashMap +import kotlin.time.Duration +import kotlin.time.Duration.Companion.milliseconds + +open class TestingJsonRpcServer( + port: Int = 0, + val apiPath: String = "/", + val recordRequestsResponses: Boolean = false, + val serverName: String = "FakeJsonRpcServer", + loggerName: String = serverName, + val vertx: Vertx = Vertx.vertx(), + val responseObjectMapper: ObjectMapper = jacksonObjectMapper(), + responsesArtificialDelay: Duration? = null +) { + val log: Logger = LogManager.getLogger(loggerName) + private var httpServer: HttpJsonRpcServer = createHttpServer(port) + val bindedPort: Int + get() = httpServer.bindedPort + private var verticleId: String? = null + private val handlers: MutableMap Any?> = ConcurrentHashMap() + private var requests: MutableList< + Pair>> + > = mutableListOf() + + var responsesArtificialDelay: Duration? = responsesArtificialDelay + set(value) { + require(value == null || value > 0.milliseconds) { "artificialDelay=$value must be greater than 0ms" } + field = value + } + + private fun createHttpServer(port: Int?): HttpJsonRpcServer { + return HttpJsonRpcServer( + port = port?.toUInt() ?: 0u, + path = apiPath, + requestHandler = HttpRequestHandler( + JsonRpcMessageProcessor( + requestsHandler = this::handleRequest, + meterRegistry = SimpleMeterRegistry(), + log = log, + responseObjectMapper = responseObjectMapper + ) + ), + serverName = serverName + ) + } + + init { + vertx + .deployVerticle(httpServer, DeploymentOptions().setInstances(1)) + .onSuccess { verticleId: String -> this.verticleId = verticleId } + .get() + } + + fun stopHttpServer(): Future { + return vertx.undeploy(verticleId).map { } + } + + fun resumeHttpServer(): Future { + // reuse the same port + httpServer = createHttpServer(bindedPort) + return vertx + .deployVerticle(httpServer, DeploymentOptions().setInstances(1)) + .onSuccess { verticleId: String -> + log.info("Http server resumed at port {}", httpServer.bindedPort) + this.verticleId = verticleId + } + .onFailure { th -> + log.error("Error resuming http server", th) + } + .map { } + } + + @Suppress("UNUSED_PARAMETER") + private fun handleRequest( + user: User?, + jsonRpcRequest: JsonRpcRequest, + requestJson: JsonObject + ): Future> { + // need this otherwise kotlin compiler/IDE struggle to infer the type + val result: Future> = ( + handlers[jsonRpcRequest.method] + ?.let { handler -> + try { + val result = handler(jsonRpcRequest) + Future.succeededFuture( + Ok( + JsonRpcSuccessResponse( + request = jsonRpcRequest, + result = result + ) + ) + ) + } catch (e: JsonRpcErrorResponseException) { + Future.succeededFuture(Err(JsonRpcErrorResponse(jsonRpcRequest.id, e.asJsonRpcError()))) + } catch (e: Exception) { + Future.succeededFuture(Err(JsonRpcErrorResponse.internalError(jsonRpcRequest.id, data = e.message))) + } + } + ?: Future.succeededFuture(Err(JsonRpcErrorResponse.methodNotFound(jsonRpcRequest.id, jsonRpcRequest.method))) + ) + + return result + .let { future -> + responsesArtificialDelay?.let { future.delayed(it) } ?: future + } + .also { + if (recordRequestsResponses) { + requests.add(jsonRpcRequest to it) + } + } + } + + /** + * Handler shall return response result or throw [JsonRpcErrorResponseException] if error + */ + fun handle( + method: String, + methodHandler: (jsonRpcRequest: JsonRpcRequest) -> Any? + ) { + handlers[method] = methodHandler + } + + fun recordedRequests(): List>>> { + return requests.toList() + } + + fun cleanRecordedRequests() { + requests.clear() + } + + fun callCountByMethod(method: String): Int { + return requests.count { it.first.method == method } + } + + private fun Future.delayed(delay: Duration): Future { + val promise = Promise.promise() + vertx.setTimer(delay.inWholeMilliseconds) { + this.onComplete(promise) + } + return promise.future() + } + + private fun SafeFuture.delayed(delay: Duration): SafeFuture { + val promise = SafeFuture() + vertx.setTimer(delay.inWholeMilliseconds) { + this.thenAccept(promise::complete).exceptionally { promise.completeExceptionally(it); null } + } + return promise + } +} From 00f9381e330562f09ead1f2a4e7b0b2eaf4cef93 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Fri, 13 Dec 2024 20:52:30 +0000 Subject: [PATCH 16/31] coordinator: adds testes for BlockCreationMonitor --- coordinator/app/build.gradle | 3 + .../zkevm/coordinator/app/L1DependentApp.kt | 2 +- .../blockcreation/BlockCreationMonitor.kt | 41 +- .../blockcreation/BlockCreationMonitorTest.kt | 372 ++++++++++++++++++ .../main/kotlin/linea/log4j/Configuration.kt | 14 + .../kotlin/linea/domain/BlockFactory.kt | 60 +++ jvm-libs/linea/web3j-extensions/build.gradle | 1 + .../main/kotlin/linea/web3j/Web3JFactory.kt | 2 +- .../consensys/linea/web3j/ExtendedWeb3J.kt | 27 +- 9 files changed, 494 insertions(+), 28 deletions(-) create mode 100644 coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitorTest.kt create mode 100644 jvm-libs/generic/logging/src/main/kotlin/linea/log4j/Configuration.kt diff --git a/coordinator/app/build.gradle b/coordinator/app/build.gradle index 89e778f6f..241fec9fe 100644 --- a/coordinator/app/build.gradle +++ b/coordinator/app/build.gradle @@ -64,6 +64,9 @@ dependencies { implementation "com.fasterxml.jackson.module:jackson-module-kotlin:${libs.versions.jackson.get()}" implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${libs.versions.jackson.get()}") testImplementation "org.apache.logging.log4j:log4j-slf4j2-impl:${libs.versions.log4j.get()}" + testImplementation project(':jvm-libs:generic:serialization:jackson') + testImplementation testFixtures(project(':jvm-libs:linea:core:domain-models')) + testImplementation testFixtures(project(':jvm-libs:generic:json-rpc')) testImplementation project(':coordinator:ethereum:test-utils') testImplementation "io.vertx:vertx-junit5" } diff --git a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt index c88594551..9986f7d07 100644 --- a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt +++ b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt @@ -904,7 +904,7 @@ class L1DependentApp( log.info("Resuming conflation from block={} inclusive", lastProcessedBlockNumber + 1UL) val blockCreationMonitor = BlockCreationMonitor( vertx = vertx, - web3j = l2ExtendedWeb3j.web3jClient, + web3j = l2ExtendedWeb3j, startingBlockNumberExclusive = lastProcessedBlockNumber.toLong(), blockCreationListener = block2BatchCoordinator, lastProvenBlockNumberProviderAsync = lastProvenBlockNumberProvider, diff --git a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitor.kt b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitor.kt index 7df27335e..27b658193 100644 --- a/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitor.kt +++ b/coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitor.kt @@ -2,19 +2,15 @@ package net.consensys.zkevm.coordinator.blockcreation import io.vertx.core.Vertx import linea.domain.Block -import linea.web3j.toDomain -import net.consensys.decodeHex import net.consensys.encodeHex +import net.consensys.linea.BlockParameter.Companion.toBlockParameter import net.consensys.linea.async.AsyncRetryer -import net.consensys.linea.async.toSafeFuture +import net.consensys.linea.web3j.ExtendedWeb3J import net.consensys.zkevm.PeriodicPollingService import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreated import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreationListener import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger -import org.web3j.protocol.Web3j -import org.web3j.protocol.core.DefaultBlockParameter -import org.web3j.protocol.core.methods.response.EthBlock import tech.pegasys.teku.infrastructure.async.SafeFuture import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicLong @@ -24,7 +20,7 @@ import kotlin.time.Duration.Companion.days class BlockCreationMonitor( private val vertx: Vertx, - private val web3j: Web3j, + private val web3j: ExtendedWeb3J, private val startingBlockNumberExclusive: Long, private val blockCreationListener: BlockCreationListener, private val lastProvenBlockNumberProviderAsync: LastProvenBlockNumberProviderAsync, @@ -40,7 +36,7 @@ class BlockCreationMonitor( val blocksToFinalization: Long, val blocksFetchLimit: Long, val startingBlockWaitTimeout: Duration = 14.days, - val lastL2BlockNumberToProcessInclusive: ULong? + val lastL2BlockNumberToProcessInclusive: ULong? = null ) private val _nexBlockNumberToFetch: AtomicLong = AtomicLong(startingBlockNumberExclusive + 1) @@ -75,8 +71,8 @@ class BlockCreationMonitor( vertx, backoffDelay = config.pollingInterval, timeout = config.startingBlockWaitTimeout, - stopRetriesPredicate = { block: EthBlock -> - if (block.block == null) { + stopRetriesPredicate = { block: Block? -> + if (block == null) { log.warn( "Block {} not found yet. Retrying in {}", startingBlockNumberExclusive, @@ -85,15 +81,12 @@ class BlockCreationMonitor( false } else { log.info("Block {} found. Resuming block monitor", startingBlockNumberExclusive) - expectedParentBlockHash.set(block.block.hash.decodeHex()) + expectedParentBlockHash.set(block.hash) true } } ) { - web3j - .ethGetBlockByNumber(DefaultBlockParameter.valueOf(startingBlockNumberExclusive.toBigInteger()), false) - .sendAsync() - .toSafeFuture() + web3j.ethGetBlock(startingBlockNumberExclusive.toBlockParameter()) } } @@ -101,7 +94,7 @@ class BlockCreationMonitor( } override fun action(): SafeFuture<*> { - log.trace("tick start") + log.trace("tick start: nexBlockNumberToFetch={}", nexBlockNumberToFetch) return lastProvenBlockNumberProviderAsync.getLastProvenBlockNumber() .thenCompose { lastProvenBlockNumber -> if (!nextBlockNumberWithinLimit(lastProvenBlockNumber)) { @@ -118,8 +111,8 @@ class BlockCreationMonitor( nexBlockNumberToFetch.toULong() > config.lastL2BlockNumberToProcessInclusive ) { log.warn( - "Stopping Conflation, Blob and Aggregation at lastL2BlockNumberInclusiveToProcess - 1. " + - "All blocks unto and including lastL2BlockNumberInclusiveToProcess={} have been processed. " + + "stopping conflation at lastL2BlockNumberInclusiveToProcess - 1. " + + "All blocks upto and including lastL2BlockNumberInclusiveToProcess={} have been processed. " + "nextBlockNumberToFetch={}", config.lastL2BlockNumberToProcessInclusive, nexBlockNumberToFetch @@ -165,6 +158,7 @@ class BlockCreationMonitor( } private fun notifyListener(payload: Block): SafeFuture { + log.trace("notifying blockCreationListener: block={}", payload.number) return blockCreationListener .acceptBlock(BlockCreated(payload)) .thenApply { @@ -186,19 +180,16 @@ class BlockCreationMonitor( private fun getNetNextSafeBlock(): SafeFuture { return web3j .ethBlockNumber() - .sendAsync() - .toSafeFuture() - .thenApply { it.blockNumber } .thenCompose { latestBlockNumber -> // Check if is safe to fetch nextWaitingBlockNumber if (latestBlockNumber.toLong() >= _nexBlockNumberToFetch.get() + config.blocksToFinalization ) { val blockNumber = _nexBlockNumberToFetch.get() - web3j.ethGetBlockByNumber(DefaultBlockParameter.valueOf(blockNumber.toBigInteger()), true) - .sendAsync() - .toSafeFuture() - .thenApply { it.block.toDomain() } + web3j.ethGetBlock(blockNumber.toBlockParameter()) + .thenPeek { block -> + log.trace("requestedBock={} responselock={}", blockNumber, block?.number) + } .whenException { log.warn( "eth_getBlockByNumber({}) failed: errorMessage={}", diff --git a/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitorTest.kt b/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitorTest.kt new file mode 100644 index 000000000..4a53d8e90 --- /dev/null +++ b/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitorTest.kt @@ -0,0 +1,372 @@ +package net.consensys.zkevm.coordinator.blockcreation + +import build.linea.s11n.jackson.ethApiObjectMapper +import io.vertx.core.Vertx +import io.vertx.junit5.VertxExtension +import linea.domain.Block +import linea.domain.createBlock +import linea.domain.toEthGetBlockResponse +import linea.jsonrpc.TestingJsonRpcServer +import linea.log4j.configureLoggers +import linea.web3j.createWeb3jHttpClient +import net.consensys.ByteArrayExt +import net.consensys.linea.async.get +import net.consensys.linea.web3j.ExtendedWeb3J +import net.consensys.linea.web3j.ExtendedWeb3JImpl +import net.consensys.toHexString +import net.consensys.toULongFromHex +import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreated +import net.consensys.zkevm.ethereum.coordination.blockcreation.BlockCreationListener +import org.apache.logging.log4j.Level +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.assertj.core.api.Assertions.assertThat +import org.awaitility.Awaitility.await +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.kotlin.mock +import tech.pegasys.teku.infrastructure.async.SafeFuture +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds +import kotlin.time.toJavaDuration + +@ExtendWith(VertxExtension::class) +class BlockCreationMonitorTest { + private lateinit var log: Logger + private lateinit var web3jClient: ExtendedWeb3J + private lateinit var blockCreationListener: BlockCreationListenerDouble + private var config: BlockCreationMonitor.Config = + BlockCreationMonitor.Config( + pollingInterval = 100.milliseconds, + blocksToFinalization = 2L, + blocksFetchLimit = 500, + lastL2BlockNumberToProcessInclusive = null + ) + private lateinit var vertx: Vertx + private val executor = Executors.newSingleThreadScheduledExecutor() + private lateinit var lastProvenBlockNumberProvider: LastProvenBlockNumberProviderDouble + private lateinit var monitor: BlockCreationMonitor + + private lateinit var fakeL2RpcNode: TestingJsonRpcServer + + private class BlockCreationListenerDouble() : BlockCreationListener { + val blocksReceived: MutableList = CopyOnWriteArrayList() + + override fun acceptBlock(blockEvent: BlockCreated): SafeFuture { + blocksReceived.add(blockEvent.block) + return SafeFuture.completedFuture(Unit) + } + } + + private class LastProvenBlockNumberProviderDouble( + initialValue: ULong + ) : LastProvenBlockNumberProviderAsync { + var lastProvenBlock: AtomicLong = AtomicLong(initialValue.toLong()) + override fun getLastProvenBlockNumber(): SafeFuture { + return SafeFuture.completedFuture(lastProvenBlock.get()) + } + } + + fun createBlockCreationMonitor( + startingBlockNumberExclusive: Long = 99, + blockCreationListener: BlockCreationListener = this.blockCreationListener, + config: BlockCreationMonitor.Config = this.config + ): BlockCreationMonitor { + return BlockCreationMonitor( + this.vertx, + web3jClient, + startingBlockNumberExclusive = startingBlockNumberExclusive, + blockCreationListener, + lastProvenBlockNumberProvider, + config + ) + } + + @BeforeEach + fun beforeEach(vertx: Vertx) { + configureLoggers(Level.INFO, "test.client.l2.web3j" to Level.TRACE) + this.vertx = vertx + log = mock() + + fakeL2RpcNode = TestingJsonRpcServer( + vertx = vertx, + recordRequestsResponses = true, + responseObjectMapper = ethApiObjectMapper + ) + blockCreationListener = BlockCreationListenerDouble() + web3jClient = ExtendedWeb3JImpl( + createWeb3jHttpClient( + rpcUrl = "http://localhost:${fakeL2RpcNode.bindedPort}", + log = LogManager.getLogger("test.client.l2.web3j") + ) + ) + lastProvenBlockNumberProvider = LastProvenBlockNumberProviderDouble(99u) + } + + @AfterEach + fun afterEach(vertx: Vertx) { + monitor.stop() + vertx.close().get() + } + + fun createBlocks( + startBlockNumber: ULong, + numberOfBlocks: Int, + startBlockHash: ByteArray = ByteArrayExt.random32(), + startBlockParentHash: ByteArray = ByteArrayExt.random32() + ): List { + var blockHash = startBlockHash + var parentHash = startBlockParentHash + return (0..numberOfBlocks).map { i -> + createBlock( + number = startBlockNumber + i.toULong(), + hash = blockHash, + parentHash = parentHash + ).also { + blockHash = ByteArrayExt.random32() + parentHash = it.hash + } + } + } + + private fun setupFakeExecutionLayerWithBlocks(blocks: List) { + fakeL2RpcNode.handle("eth_getBlockByNumber") { request -> + val blockNumber = ((request.params as List)[0] as String).toULongFromHex() + blocks.find { it.number == blockNumber }?.toEthGetBlockResponse() + } + + fakeL2RpcNode.handle("eth_blockNumber") { request -> + blocks.last().number.toHexString() + } + } + + @Test + fun `should stop fetching blocks after lastBlockNumberInclusiveToProcess`() { + monitor = createBlockCreationMonitor( + startingBlockNumberExclusive = 99, + config = config.copy(lastL2BlockNumberToProcessInclusive = 103u) + ) + + setupFakeExecutionLayerWithBlocks(createBlocks(startBlockNumber = 99u, numberOfBlocks = 20)) + + monitor.start() + await() + .atMost(20.seconds.toJavaDuration()) + .untilAsserted { + assertThat(blockCreationListener.blocksReceived).isNotEmpty + assertThat(blockCreationListener.blocksReceived.last().number).isGreaterThanOrEqualTo(103u) + } + + // Wait for a while to make sure no more blocks are fetched + await().atLeast(config.pollingInterval.times(3).toJavaDuration()) + + assertThat(blockCreationListener.blocksReceived.last().number).isEqualTo(103UL) + } + + @Test + fun `should notify lister only after block is considered final on L2`() { + monitor = createBlockCreationMonitor( + startingBlockNumberExclusive = 99, + config = config.copy(blocksToFinalization = 2, blocksFetchLimit = 500) + ) + + setupFakeExecutionLayerWithBlocks(createBlocks(startBlockNumber = 99u, numberOfBlocks = 200)) + fakeL2RpcNode.handle("eth_blockNumber") { request -> 105UL.toHexString() } + // latest eligible conflation is: 105 - 2 = 103, inclusive + + monitor.start() + await() + .atMost(20.seconds.toJavaDuration()) + .untilAsserted { + assertThat(blockCreationListener.blocksReceived).isNotEmpty + assertThat(blockCreationListener.blocksReceived.last().number).isGreaterThanOrEqualTo(103u) + } + // Wait for a while to make sure no more blocks are fetched + await().atLeast(config.pollingInterval.times(3).toJavaDuration()) + // assert that no more block were sent to the listener + assertThat(blockCreationListener.blocksReceived.last().number).isEqualTo(103UL) + + // move chain head forward + fakeL2RpcNode.handle("eth_blockNumber") { request -> 120UL.toHexString() } + + // assert it resumes conflation + await() + .atMost(20.seconds.toJavaDuration()) + .untilAsserted { + assertThat(blockCreationListener.blocksReceived.last().number).isEqualTo(118UL) + } + } + + @Test + fun `shall retry notify the listener when it throws and keeps block order`() { + val fakeBuggyLister = object : BlockCreationListener { + var errorCount = 0 + override fun acceptBlock(blockEvent: BlockCreated): SafeFuture { + return if (blockEvent.block.number == 105UL && errorCount < 3) { + errorCount++ + throw RuntimeException("Error on block 105") + } else { + blockCreationListener.acceptBlock(blockEvent) + } + } + } + + monitor = createBlockCreationMonitor( + startingBlockNumberExclusive = 99, + blockCreationListener = fakeBuggyLister, + config = config.copy(blocksToFinalization = 2, lastL2BlockNumberToProcessInclusive = 112u) + ) + + setupFakeExecutionLayerWithBlocks(createBlocks(startBlockNumber = 99u, numberOfBlocks = 20)) + + monitor.start() + await() + .atMost(20.seconds.toJavaDuration()) + .untilAsserted { + assertThat(blockCreationListener.blocksReceived).isNotEmpty + assertThat(blockCreationListener.blocksReceived.last().number).isGreaterThanOrEqualTo(110u) + } + + // assert it got block only once and in order + assertThat(blockCreationListener.blocksReceived.map { it.number }).containsExactly( + 100UL, 101UL, 102UL, 103UL, 104UL, 105UL, 106UL, 107UL, 108UL, 109UL, 110UL + ) + } + + @Test + fun `should be resilient to connection failures`() { + monitor = createBlockCreationMonitor( + startingBlockNumberExclusive = 99 + ) + + setupFakeExecutionLayerWithBlocks(createBlocks(startBlockNumber = 99u, numberOfBlocks = 200)) + + monitor.start() + await() + .atMost(20.seconds.toJavaDuration()) + .untilAsserted { + assertThat(blockCreationListener.blocksReceived).isNotEmpty + assertThat(blockCreationListener.blocksReceived.last().number).isGreaterThanOrEqualTo(103u) + } + fakeL2RpcNode.stopHttpServer() + val lastBlockReceived = blockCreationListener.blocksReceived.last().number + + // Wait for a while to make sure no more blocks are fetched + await().atLeast(config.pollingInterval.times(2).toJavaDuration()) + fakeL2RpcNode.resumeHttpServer() + await() + .atMost(20.seconds.toJavaDuration()) + .untilAsserted { + assertThat(blockCreationListener.blocksReceived).isNotEmpty + assertThat(blockCreationListener.blocksReceived.last().number).isGreaterThan(lastBlockReceived) + } + } + + @Test + fun `should stop when reorg is detected above blocksToFinalization limit - manual intervention necessary`() { + monitor = createBlockCreationMonitor( + startingBlockNumberExclusive = 99 + ) + + // simulate reorg by changing parent hash of block 105 + val blocks = createBlocks(startBlockNumber = 99u, numberOfBlocks = 20).map { block: Block -> + if (block.number == 105UL) { + block.copy(parentHash = ByteArrayExt.random32()) + } else { + block + } + } + + setupFakeExecutionLayerWithBlocks(blocks) + + monitor.start() + await() + .atMost(20.seconds.toJavaDuration()) + .untilAsserted { + assertThat(blockCreationListener.blocksReceived).isNotEmpty + assertThat(blockCreationListener.blocksReceived.last().number).isGreaterThanOrEqualTo(104UL) + } + + // Wait for a while to make sure no more blocks are fetched + await().atLeast(config.pollingInterval.times(3).toJavaDuration()) + + assertThat(blockCreationListener.blocksReceived.last().number).isEqualTo(104UL) + } + + @Test + fun `should poll in order when response takes longer that polling interval`() { + monitor = createBlockCreationMonitor( + startingBlockNumberExclusive = 99, + config = config.copy(pollingInterval = 100.milliseconds) + ) + + val blocks = createBlocks(startBlockNumber = 99u, numberOfBlocks = 20) + setupFakeExecutionLayerWithBlocks(blocks) + fakeL2RpcNode.responsesArtificialDelay = 600.milliseconds + + monitor.start() + await() + .atMost(20.seconds.toJavaDuration()) + .untilAsserted { + assertThat(blockCreationListener.blocksReceived).isNotEmpty + assertThat(blockCreationListener.blocksReceived.map { it.number }).containsExactly( + 100UL, + 101UL, + 102UL, + 103UL, + 104UL, + 105UL + ) + } + } + + @Test + fun `start allow 2nd call when already started`() { + monitor = createBlockCreationMonitor( + startingBlockNumberExclusive = 99 + ) + setupFakeExecutionLayerWithBlocks(createBlocks(startBlockNumber = 99u, numberOfBlocks = 5)) + monitor.start().get() + monitor.start().get() + } + + @Test + fun `should stop fetching blocks when gap is greater than fetch limit and resume upon catchup`() { + monitor = createBlockCreationMonitor( + startingBlockNumberExclusive = 99, + config = config.copy(blocksToFinalization = 0, blocksFetchLimit = 5) + ) + + setupFakeExecutionLayerWithBlocks(createBlocks(startBlockNumber = 99u, numberOfBlocks = 30)) + lastProvenBlockNumberProvider.lastProvenBlock.set(105) + + monitor.start() + await() + .atMost(20.seconds.toJavaDuration()) + .untilAsserted { + assertThat(blockCreationListener.blocksReceived).isNotEmpty + assertThat(blockCreationListener.blocksReceived.last().number).isGreaterThanOrEqualTo(110UL) + } + + // Wait for a while to make sure no more blocks are fetched + await().atLeast(config.pollingInterval.times(3).toJavaDuration()) + + // it shall remain at 110 + assertThat(blockCreationListener.blocksReceived.last().number).isEqualTo(110UL) + + // simulate prover catchup + lastProvenBlockNumberProvider.lastProvenBlock.set(120) + + // assert it resumes conflation + await() + .atMost(20.seconds.toJavaDuration()) + .untilAsserted { + assertThat(blockCreationListener.blocksReceived.last().number).isGreaterThanOrEqualTo(125UL) + } + } +} diff --git a/jvm-libs/generic/logging/src/main/kotlin/linea/log4j/Configuration.kt b/jvm-libs/generic/logging/src/main/kotlin/linea/log4j/Configuration.kt new file mode 100644 index 000000000..200ef4487 --- /dev/null +++ b/jvm-libs/generic/logging/src/main/kotlin/linea/log4j/Configuration.kt @@ -0,0 +1,14 @@ +package linea.log4j + +import org.apache.logging.log4j.Level +import org.apache.logging.log4j.core.config.Configurator + +fun configureLoggers( + rootLevel: Level = Level.INFO, + vararg loggerConfigs: Pair +) { + Configurator.setRootLevel(rootLevel) + loggerConfigs.forEach { (loggerName, level) -> + Configurator.setLevel(loggerName, level) + } +} diff --git a/jvm-libs/linea/core/domain-models/src/testFixtures/kotlin/linea/domain/BlockFactory.kt b/jvm-libs/linea/core/domain-models/src/testFixtures/kotlin/linea/domain/BlockFactory.kt index 1526a7cda..f2bd9c686 100644 --- a/jvm-libs/linea/core/domain-models/src/testFixtures/kotlin/linea/domain/BlockFactory.kt +++ b/jvm-libs/linea/core/domain-models/src/testFixtures/kotlin/linea/domain/BlockFactory.kt @@ -45,3 +45,63 @@ fun createBlock( ommers = emptyList() ) } + +/** + * This is very similar to Block class, + * but creating DTO to avoid coupling with domain model, + * some fields are not present in domain model, e.g uncles + * + * This is meant to help creating fake JSON-RPC server + */ +class EthGetBlockResponseDTO( + val number: ULong, + val hash: ByteArray, + val parentHash: ByteArray, + val miner: ByteArray, + val stateRoot: ByteArray, + val transactionsRoot: ByteArray, + val receiptsRoot: ByteArray, + val logsBloom: ByteArray, + val difficulty: ULong, + val gasLimit: ULong, + val gasUsed: ULong, + val timestamp: ULong, + val extraData: ByteArray, + val mixHash: ByteArray, + val nonce: ULong, + val baseFeePerGas: ULong?, + val sha3Uncles: ByteArray, // ommersHash + val size: ULong, + val totalDifficulty: ULong, + val transactions: List, + val uncles: List = emptyList() +) + +fun Block?.toEthGetBlockResponse( + size: ULong = 10UL * 1024UL, + totalDifficulty: ULong = this?.difficulty ?: 0UL +): EthGetBlockResponseDTO? { + if (this == null) return null + return EthGetBlockResponseDTO( + number = this.number, + hash = this.hash, + parentHash = this.parentHash, + miner = this.miner, + stateRoot = this.stateRoot, + transactionsRoot = this.transactionsRoot, + receiptsRoot = this.receiptsRoot, + logsBloom = this.logsBloom, + difficulty = this.difficulty, + gasLimit = this.gasLimit, + gasUsed = this.gasUsed, + timestamp = this.timestamp, + extraData = this.extraData, + mixHash = this.mixHash, + nonce = this.nonce, + baseFeePerGas = this.baseFeePerGas, + sha3Uncles = this.ommersHash, + size = size, + totalDifficulty = totalDifficulty, + transactions = emptyList() + ) +} diff --git a/jvm-libs/linea/web3j-extensions/build.gradle b/jvm-libs/linea/web3j-extensions/build.gradle index e3e2405d0..16d1caf69 100644 --- a/jvm-libs/linea/web3j-extensions/build.gradle +++ b/jvm-libs/linea/web3j-extensions/build.gradle @@ -12,6 +12,7 @@ dependencies { api project(':jvm-libs:generic:logging') api project(':jvm-libs:linea:besu-libs') implementation project(":jvm-libs:generic:extensions:kotlin") + implementation project(":jvm-libs:generic:extensions:futures") implementation "tech.pegasys.teku.internal:bytes:${libs.versions.teku.get()}" implementation "tech.pegasys.teku.internal:jackson:${libs.versions.teku.get()}" implementation "tech.pegasys.teku.internal:unsigned:${libs.versions.teku.get()}" diff --git a/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/Web3JFactory.kt b/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/Web3JFactory.kt index 6ac151866..ef2ef3a5e 100644 --- a/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/Web3JFactory.kt +++ b/jvm-libs/linea/web3j-extensions/src/main/kotlin/linea/web3j/Web3JFactory.kt @@ -12,7 +12,7 @@ import kotlin.time.Duration.Companion.milliseconds fun createWeb3jHttpClient( rpcUrl: String, - log: Logger, + log: Logger = org.apache.logging.log4j.LogManager.getLogger(Web3j::class.java), pollingInterval: Duration = 500.milliseconds, executorService: ScheduledExecutorService = Async.defaultExecutorService(), requestResponseLogLevel: Level = Level.TRACE, diff --git a/jvm-libs/linea/web3j-extensions/src/main/kotlin/net/consensys/linea/web3j/ExtendedWeb3J.kt b/jvm-libs/linea/web3j-extensions/src/main/kotlin/net/consensys/linea/web3j/ExtendedWeb3J.kt index 94ebab2e9..c2fc8c171 100644 --- a/jvm-libs/linea/web3j-extensions/src/main/kotlin/net/consensys/linea/web3j/ExtendedWeb3J.kt +++ b/jvm-libs/linea/web3j-extensions/src/main/kotlin/net/consensys/linea/web3j/ExtendedWeb3J.kt @@ -1,5 +1,10 @@ package net.consensys.linea.web3j +import build.linea.web3j.domain.toWeb3j +import linea.domain.Block +import linea.web3j.toDomain +import net.consensys.linea.BlockParameter +import net.consensys.linea.async.toSafeFuture import org.web3j.protocol.Web3j import org.web3j.protocol.core.DefaultBlockParameter import org.web3j.protocol.core.Response @@ -13,13 +18,14 @@ import java.math.BigInteger interface ExtendedWeb3J { val web3jClient: Web3j fun ethBlockNumber(): SafeFuture + fun ethGetBlock(blockParameter: BlockParameter): SafeFuture fun ethGetBlockTimestampByNumber(blockNumber: Long): SafeFuture } class ExtendedWeb3JImpl(override val web3jClient: Web3j) : ExtendedWeb3J { private fun buildException(error: Response.Error): Exception = - Exception("${error.code}: ${error.message}") + RuntimeException("${error.code}: ${error.message}") override fun ethBlockNumber(): SafeFuture { return SafeFuture.of(web3jClient.ethBlockNumber().sendAsync()).thenCompose { response -> @@ -31,6 +37,25 @@ class ExtendedWeb3JImpl(override val web3jClient: Web3j) : ExtendedWeb3J { } } + override fun ethGetBlock(blockParameter: BlockParameter): SafeFuture { + return web3jClient + .ethGetBlockByNumber( + blockParameter.toWeb3j(), + true + ) + .sendAsync() + .toSafeFuture() + .thenCompose { response -> + if (response.hasError()) { + SafeFuture.failedFuture(buildException(response.error)) + } else { + response.block?.let { + SafeFuture.completedFuture(response.block.toDomain()) + } ?: SafeFuture.failedFuture(RuntimeException("Block $blockParameter not found!")) + } + } + } + override fun ethGetBlockTimestampByNumber( blockNumber: Long ): SafeFuture { From 7dc0c553d2da2c18dfcc8b1a62df4e42bbb521f8 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Fri, 13 Dec 2024 22:24:24 +0000 Subject: [PATCH 17/31] coordinator: small code cosmetic and unit test fixes --- .../blockcreation/BlockCreationMonitorTest.kt | 6 +++--- gradle/libs.versions.toml | 1 - .../test/kotlin/net/consensys/StringExtensionsTest.kt | 2 -- .../consensys/linea/jsonrpc/JsonRpcMessageProcessor.kt | 9 ++++++--- .../kotlin/linea/jsonrpc/TestingJsonRpcServer.kt | 2 +- .../main/kotlin/linea/domain/MapperBesuToLineaDomain.kt | 8 ++++---- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitorTest.kt b/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitorTest.kt index 4a53d8e90..9f07d2460 100644 --- a/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitorTest.kt +++ b/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitorTest.kt @@ -140,7 +140,7 @@ class BlockCreationMonitorTest { blocks.find { it.number == blockNumber }?.toEthGetBlockResponse() } - fakeL2RpcNode.handle("eth_blockNumber") { request -> + fakeL2RpcNode.handle("eth_blockNumber") { _ -> blocks.last().number.toHexString() } } @@ -176,7 +176,7 @@ class BlockCreationMonitorTest { ) setupFakeExecutionLayerWithBlocks(createBlocks(startBlockNumber = 99u, numberOfBlocks = 200)) - fakeL2RpcNode.handle("eth_blockNumber") { request -> 105UL.toHexString() } + fakeL2RpcNode.handle("eth_blockNumber") { _ -> 105UL.toHexString() } // latest eligible conflation is: 105 - 2 = 103, inclusive monitor.start() @@ -192,7 +192,7 @@ class BlockCreationMonitorTest { assertThat(blockCreationListener.blocksReceived.last().number).isEqualTo(103UL) // move chain head forward - fakeL2RpcNode.handle("eth_blockNumber") { request -> 120UL.toHexString() } + fakeL2RpcNode.handle("eth_blockNumber") { _ -> 120UL.toHexString() } // assert it resumes conflation await() diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4102e4cc2..ede6c9b3d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -17,7 +17,6 @@ junit = "5.10.1" kotlinxDatetime = "0.6.1" ktlint = "0.47.0" log4j = "2.20.0" -slf4j = "1.7.30" micrometer = "1.8.4" netty = "4.1.92.Final" picoli = "4.7.1" diff --git a/jvm-libs/generic/extensions/kotlin/src/test/kotlin/net/consensys/StringExtensionsTest.kt b/jvm-libs/generic/extensions/kotlin/src/test/kotlin/net/consensys/StringExtensionsTest.kt index 14854e3e6..d69e71b5d 100644 --- a/jvm-libs/generic/extensions/kotlin/src/test/kotlin/net/consensys/StringExtensionsTest.kt +++ b/jvm-libs/generic/extensions/kotlin/src/test/kotlin/net/consensys/StringExtensionsTest.kt @@ -28,7 +28,6 @@ class StringExtensionsTest { assertThat("this string won't match".containsAny(stringList, ignoreCase = true)).isFalse() } - @OptIn(ExperimentalStdlibApi::class) @Test fun `String#toIntFromHex`() { assertThat("0x00".toIntFromHex()).isEqualTo(0) @@ -37,7 +36,6 @@ class StringExtensionsTest { assertThat("0x7FFFFFFF".toIntFromHex()).isEqualTo(Int.MAX_VALUE) } - @OptIn(ExperimentalStdlibApi::class) @Test fun `String#toLongFromHex`() { assertThat("0x00".toLongFromHex()).isEqualTo(0L) diff --git a/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/JsonRpcMessageProcessor.kt b/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/JsonRpcMessageProcessor.kt index 563402861..2b3c65b7b 100644 --- a/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/JsonRpcMessageProcessor.kt +++ b/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/JsonRpcMessageProcessor.kt @@ -2,6 +2,7 @@ package net.consensys.linea.jsonrpc import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule import com.github.michaelbull.result.Err import com.github.michaelbull.result.Ok @@ -23,6 +24,7 @@ import io.vertx.core.json.Json import io.vertx.core.json.JsonArray import io.vertx.core.json.JsonObject import io.vertx.core.json.jackson.DatabindCodec +import io.vertx.core.json.jackson.VertxModule import io.vertx.ext.auth.User import net.consensys.linea.metrics.micrometer.DynamicTagTimerCapture import net.consensys.linea.metrics.micrometer.SimpleTimerCapture @@ -55,7 +57,8 @@ class JsonRpcMessageProcessor( private val meterRegistry: MeterRegistry, private val requestParser: JsonRpcRequestParser = Companion::parseRequest, private val log: Logger = LogManager.getLogger(JsonRpcMessageProcessor::class.java), - private val responseObjectMapper: ObjectMapper = ObjectMapper().registerKotlinModule() + private val responseResultObjectMapper: ObjectMapper = jacksonObjectMapper().registerModules(VertxModule()), + private val rpcEnvelopeObjectMapper: ObjectMapper = jacksonObjectMapper() ) : JsonRpcMessageHandler { init { DatabindCodec.mapper().registerKotlinModule() @@ -180,10 +183,10 @@ class JsonRpcMessageProcessor( .setTag("method", requestContext.method) .captureTime { val result = requestContext.result.map { successResponse -> - val resultJsonNode = responseObjectMapper.valueToTree(successResponse.result) + val resultJsonNode = responseResultObjectMapper.valueToTree(successResponse.result) successResponse.copy(result = resultJsonNode) } - Json.encode(result.merge()) + rpcEnvelopeObjectMapper.writeValueAsString(result.merge()) } } diff --git a/jvm-libs/generic/json-rpc/src/testFixtures/kotlin/linea/jsonrpc/TestingJsonRpcServer.kt b/jvm-libs/generic/json-rpc/src/testFixtures/kotlin/linea/jsonrpc/TestingJsonRpcServer.kt index 3dc29502c..b1a834213 100644 --- a/jvm-libs/generic/json-rpc/src/testFixtures/kotlin/linea/jsonrpc/TestingJsonRpcServer.kt +++ b/jvm-libs/generic/json-rpc/src/testFixtures/kotlin/linea/jsonrpc/TestingJsonRpcServer.kt @@ -62,7 +62,7 @@ open class TestingJsonRpcServer( requestsHandler = this::handleRequest, meterRegistry = SimpleMeterRegistry(), log = log, - responseObjectMapper = responseObjectMapper + responseResultObjectMapper = responseObjectMapper ) ), serverName = serverName diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperBesuToLineaDomain.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperBesuToLineaDomain.kt index fd81a61a8..1ca3416e0 100644 --- a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperBesuToLineaDomain.kt +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/domain/MapperBesuToLineaDomain.kt @@ -41,7 +41,7 @@ object MapperBesuToLineaDomain { nonce = transaction.nonce.toULong(), gasPrice = transaction.getGasPrice().getOrNull()?.toBigInteger()?.toULong(), gasLimit = transaction.gasLimit.toULong(), - to = transaction.to?.getOrNull()?.toArray(), + to = transaction.to.getOrNull()?.toArray(), value = transaction.value.toBigInteger(), input = transaction.payload.toArray(), r = transaction.signature.getR(), @@ -49,9 +49,9 @@ object MapperBesuToLineaDomain { v = transaction.getV().toULong(), yParity = transaction.yParity?.toULong(), type = transaction.type.toDomain(), - chainId = transaction.chainId?.getOrNull()?.toULong(), - maxFeePerGas = transaction.maxFeePerGas?.getOrNull()?.toBigInteger()?.toULong(), - maxPriorityFeePerGas = transaction.maxPriorityFeePerGas?.getOrNull()?.toBigInteger()?.toULong(), + chainId = transaction.chainId.getOrNull()?.toULong(), + maxFeePerGas = transaction.maxFeePerGas.getOrNull()?.toBigInteger()?.toULong(), + maxPriorityFeePerGas = transaction.maxPriorityFeePerGas.getOrNull()?.toBigInteger()?.toULong(), accessList = transaction.accessList.getOrNull()?.map { accessListEntry -> AccessListEntry( accessListEntry.address.toArray(), From 05d20ddba85dde27508c14a0dac6347d1cd7997e Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Fri, 13 Dec 2024 22:24:43 +0000 Subject: [PATCH 18/31] coordinator: small code cosmetic and unit test fixes --- build.gradle | 6 ------ 1 file changed, 6 deletions(-) diff --git a/build.gradle b/build.gradle index 28dfc60d5..46307076e 100644 --- a/build.gradle +++ b/build.gradle @@ -23,12 +23,6 @@ allprojects { apply plugin: 'java' // do not add kotlin plugin here, it will add unnecessary Kotlin runtime dependencies apply plugin: 'jacoco' - configurations.all { - resolutionStrategy { - force "org.web3j:core:${libs.versions.web3j.get()}" - } - } - tasks.withType(KotlinCompile).configureEach { compileAll.dependsOn it compilerOptions { From c4fb114160a41f3dff2f357e88bd747559d12dea Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Sat, 14 Dec 2024 12:45:01 +0000 Subject: [PATCH 19/31] coordinator: fix import --- .../net/consensys/linea/jsonrpc/JsonRpcMessageProcessorTest.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/jvm-libs/generic/json-rpc/src/test/kotlin/net/consensys/linea/jsonrpc/JsonRpcMessageProcessorTest.kt b/jvm-libs/generic/json-rpc/src/test/kotlin/net/consensys/linea/jsonrpc/JsonRpcMessageProcessorTest.kt index e1be9320a..4bec02f53 100644 --- a/jvm-libs/generic/json-rpc/src/test/kotlin/net/consensys/linea/jsonrpc/JsonRpcMessageProcessorTest.kt +++ b/jvm-libs/generic/json-rpc/src/test/kotlin/net/consensys/linea/jsonrpc/JsonRpcMessageProcessorTest.kt @@ -10,7 +10,6 @@ import io.vertx.core.json.JsonObject import io.vertx.ext.auth.User import io.vertx.junit5.VertxExtension import io.vertx.junit5.VertxTestContext -import linea.jsonrpc2.JsonRpcMessageProcessor import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test From 4416403e23ce2bc4decbbea84cd046c0945b015a Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Sat, 14 Dec 2024 14:47:06 +0000 Subject: [PATCH 20/31] coordinator: update decompressor version with transaction.to fixed --- .../src/main/kotlin/linea/blob/BlobCompressor.kt | 4 ++++ jvm-libs/linea/blob-decompressor/build.gradle | 2 +- .../kotlin/linea/test/BlockEncodingValidator.kt | 16 +++++++++------- .../src/test/kotlin/linea/test/MainRunner.kt | 4 ++-- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/jvm-libs/linea/blob-compressor/src/main/kotlin/linea/blob/BlobCompressor.kt b/jvm-libs/linea/blob-compressor/src/main/kotlin/linea/blob/BlobCompressor.kt index 92c114ec7..dc272c896 100644 --- a/jvm-libs/linea/blob-compressor/src/main/kotlin/linea/blob/BlobCompressor.kt +++ b/jvm-libs/linea/blob-compressor/src/main/kotlin/linea/blob/BlobCompressor.kt @@ -74,6 +74,10 @@ class GoBackedBlobCompressor private constructor( return goNativeBlobCompressor.CanWrite(blockRLPEncoded, blockRLPEncoded.size) } + fun inflightBlobSize(): Int { + return goNativeBlobCompressor.Len() + } + override fun appendBlock(blockRLPEncoded: ByteArray): BlobCompressor.AppendResult { val compressionSizeBefore = goNativeBlobCompressor.Len() val appended = goNativeBlobCompressor.Write(blockRLPEncoded, blockRLPEncoded.size) diff --git a/jvm-libs/linea/blob-decompressor/build.gradle b/jvm-libs/linea/blob-decompressor/build.gradle index 722ad32a6..2fbaee83b 100644 --- a/jvm-libs/linea/blob-decompressor/build.gradle +++ b/jvm-libs/linea/blob-decompressor/build.gradle @@ -36,7 +36,7 @@ def libsZipDownloadOutputDir = project.parent.layout.buildDirectory.asFile.get() task downloadNativeLibs { doLast { - fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-v1.1.0-test8/linea-blob-libs-v1.1.0-test8.zip", "blob_decompressor", libsZipDownloadOutputDir) + fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-v1.1.0-test9/linea-blob-libs-v1.1.0-test9.zip", "blob_decompressor", libsZipDownloadOutputDir) } } diff --git a/transaction-decoder-tool/src/test/kotlin/linea/test/BlockEncodingValidator.kt b/transaction-decoder-tool/src/test/kotlin/linea/test/BlockEncodingValidator.kt index cbde61085..fd9fb2d29 100644 --- a/transaction-decoder-tool/src/test/kotlin/linea/test/BlockEncodingValidator.kt +++ b/transaction-decoder-tool/src/test/kotlin/linea/test/BlockEncodingValidator.kt @@ -19,16 +19,17 @@ import org.assertj.core.api.Assertions.fail import tech.pegasys.teku.infrastructure.async.SafeFuture import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.atomic.AtomicReference -import kotlin.jvm.optionals.getOrNull -import kotlin.time.Duration.Companion.seconds +import kotlin.time.Duration.Companion.milliseconds +// 100MB, much larger than a real blob, but just for testing to allow faster testing by compressing more blocks +val BLOB_COMPRESSOR_SIZE: UInt = 100u * 1024u * 1024U class BlockEncodingValidator( val vertx: Vertx, val compressorVersion: BlobCompressorVersion = BlobCompressorVersion.V1_0_1, val decompressorVersion: BlobDecompressorVersion = BlobDecompressorVersion.V1_1_0, - val blobSizeLimitBytes: UInt = 1024u * 1024U, // 1MB, much larger than a real blob, but just for testing + val blobSizeLimitBytes: UInt = BLOB_COMPRESSOR_SIZE, val log: Logger = LogManager.getLogger(BlockEncodingValidator::class.java) -) : PeriodicPollingService(vertx, pollingIntervalMs = 1.seconds.inWholeMilliseconds, log = log) { +) : PeriodicPollingService(vertx, pollingIntervalMs = 1.milliseconds.inWholeMilliseconds, log = log) { val compressor = GoBackedBlobCompressor.getInstance(compressorVersion, blobSizeLimitBytes) val decompressor = GoNativeBlobDecompressorFactory.getInstance(decompressorVersion) @@ -83,6 +84,7 @@ class BlockEncodingValidator( CommonDomainFunctions.blockIntervalString(blocks.first().number, blocks.last().number) ) val besuBlocks = blocks.map { it.toBesu() } + val originalBlockInterval = CommonDomainFunctions.blockIntervalString(blocks.first().number, blocks.last().number) return rlpEncoder.encodeAsync(besuBlocks) .thenCompose { encodedBlocks -> encodedBlocks.forEach { compressor.appendBlock(it) } @@ -94,7 +96,8 @@ class BlockEncodingValidator( }.thenApply { decompressedBlocks -> assertThat(decompressedBlocks.size).isEqualTo(besuBlocks.size) .withFailMessage( - "decompressedBlocks.size=${decompressedBlocks.size} != originalBlocks.size=${besuBlocks.size}" + // this can happen if not all blocks fit into compressor limit + "originalBlocks=$originalBlockInterval decompressedBlocks.size=${decompressedBlocks.size} != " ) decompressedBlocks.zip(besuBlocks).forEach { (decompressed, original) -> runCatching { @@ -161,8 +164,7 @@ fun assertBlock( } else { assertThat(decompressedTx.gasPrice).isEqualTo(originalTx.gasPrice) } - // FIXME: tmp work around until decompressor is fixed - originalTx.to.getOrNull()?.let { assertThat(decompressedTx.to.getOrNull()).isEqualTo(it) } + assertThat(decompressedTx.to).isEqualTo(originalTx.to) assertThat(decompressedTx.value).isEqualTo(originalTx.value) assertThat(decompressedTx.accessList).isEqualTo(originalTx.accessList) assertThat(decompressedTx.payload).isEqualTo(originalTx.payload) diff --git a/transaction-decoder-tool/src/test/kotlin/linea/test/MainRunner.kt b/transaction-decoder-tool/src/test/kotlin/linea/test/MainRunner.kt index a2f2aa47a..6b7673a78 100644 --- a/transaction-decoder-tool/src/test/kotlin/linea/test/MainRunner.kt +++ b/transaction-decoder-tool/src/test/kotlin/linea/test/MainRunner.kt @@ -38,14 +38,14 @@ fun main() { ) // Sepolia Blocks - val startBlockNumber = 924_973UL + val startBlockNumber = 930_973UL // val startBlockNumber = 5_099_599UL // Mainnet Blocks // val startBlockNumber = 10_000_308UL runCatching { fetcherAndValidate.fetchAndValidateBlocks( startBlockNumber = startBlockNumber, - endBlockNumber = startBlockNumber + 1_000_000U, + endBlockNumber = startBlockNumber + 100_000U, // endBlockNumber = startBlockNumber + 0u, chuckSize = 1_000U, rlpEncodingDecodingOnly = false From 1719d79e288496d7c7d3b6e53dc099afcc7ec340 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:57:55 +0000 Subject: [PATCH 21/31] coordinator: fix block parameter --- .../src/main/kotlin/net/consensys/linea/BlockParameter.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvm-libs/linea/core/domain-models/src/main/kotlin/net/consensys/linea/BlockParameter.kt b/jvm-libs/linea/core/domain-models/src/main/kotlin/net/consensys/linea/BlockParameter.kt index 899558f8b..8671d9461 100644 --- a/jvm-libs/linea/core/domain-models/src/main/kotlin/net/consensys/linea/BlockParameter.kt +++ b/jvm-libs/linea/core/domain-models/src/main/kotlin/net/consensys/linea/BlockParameter.kt @@ -7,7 +7,7 @@ sealed interface BlockParameter { companion object { fun fromNumber(blockNumber: Number): BlockNumber { - require(blockNumber.toLong() > 0) { "block number must be greater than 0, value=$blockNumber" } + require(blockNumber.toLong() >= 0) { "block number must be greater or equal than 0, value=$blockNumber" } return BlockNumber(blockNumber.toLong().toULong()) } From 6079daa7402d6c9add41c5b3dde9d96fefa180e1 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:30:47 +0000 Subject: [PATCH 22/31] coordinator: update runners build --- .github/workflows/coordinator-build-and-publish.yml | 2 +- .github/workflows/traces-api-facade-build-and-publish.yml | 2 +- .../workflows/transaction-exclusion-api-build-and-publish.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coordinator-build-and-publish.yml b/.github/workflows/coordinator-build-and-publish.yml index 6b96c8945..a19522d91 100644 --- a/.github/workflows/coordinator-build-and-publish.yml +++ b/.github/workflows/coordinator-build-and-publish.yml @@ -51,7 +51,7 @@ concurrency: jobs: build-and-publish: - runs-on: [self-hosted, ubuntu-20.04, X64, small] + runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med name: Coordinator build env: COMMIT_TAG: ${{ inputs.commit_tag }} diff --git a/.github/workflows/traces-api-facade-build-and-publish.yml b/.github/workflows/traces-api-facade-build-and-publish.yml index b93f1904b..47985aab2 100644 --- a/.github/workflows/traces-api-facade-build-and-publish.yml +++ b/.github/workflows/traces-api-facade-build-and-publish.yml @@ -51,7 +51,7 @@ concurrency: jobs: build-and-publish: - runs-on: [self-hosted, ubuntu-20.04, X64, small] + runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med name: Traces api facade build env: COMMIT_TAG: ${{ inputs.commit_tag }} diff --git a/.github/workflows/transaction-exclusion-api-build-and-publish.yml b/.github/workflows/transaction-exclusion-api-build-and-publish.yml index eaa159e25..fb09686b1 100644 --- a/.github/workflows/transaction-exclusion-api-build-and-publish.yml +++ b/.github/workflows/transaction-exclusion-api-build-and-publish.yml @@ -51,7 +51,7 @@ concurrency: jobs: build-and-publish: - runs-on: [self-hosted, ubuntu-20.04, X64, small] + runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med name: Transaction exclusion api build env: COMMIT_TAG: ${{ inputs.commit_tag }} From 099082fd04caf7d99ded6b7c471a1ddb43060e6b Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:54:30 +0000 Subject: [PATCH 23/31] coordinator: update runners build --- .github/workflows/reuse-run-e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml index 985cbd6a1..aa7c6bd71 100644 --- a/.github/workflows/reuse-run-e2e-tests.yml +++ b/.github/workflows/reuse-run-e2e-tests.yml @@ -76,7 +76,7 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} outputs: tests_outcome: ${{ steps.run_e2e_tests.outcome }} - runs-on: [self-hosted, ubuntu-20.04, X64, large] + runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-large steps: - name: Setup upterm session if: ${{ inputs.e2e-tests-with-ssh }} From 583c71ee6b1d17a2556a753a3d5e86d403cee5c1 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:15:32 +0000 Subject: [PATCH 24/31] coordinator: try fix GHA runners configs --- .github/workflows/coordinator-testing.yml | 2 +- .github/workflows/reuse-run-e2e-tests.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml index fd0b7322f..fdcd59704 100644 --- a/.github/workflows/coordinator-testing.yml +++ b/.github/workflows/coordinator-testing.yml @@ -26,7 +26,7 @@ jobs: GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN_RELEASE_ACCESS }} DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - runs-on: [self-hosted, ubuntu-22.04, X64, medium] + runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-large name: Coordinator tests steps: - name: Checkout diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml index aa7c6bd71..eb4b827f5 100644 --- a/.github/workflows/reuse-run-e2e-tests.yml +++ b/.github/workflows/reuse-run-e2e-tests.yml @@ -116,6 +116,8 @@ jobs: make pull-images-external-to-monorepo - name: Download local docker image artifacts uses: actions/download-artifact@v4 + with: + pattern: linea-* - name: Load Docker images run: | gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-coordinator/linea-coordinator-docker-image.tar.gz | docker load && From b1506622b57bca98d22145548826eaae4658613a Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:32:45 +0000 Subject: [PATCH 25/31] coordinator: try fix GHA runners configs --- .github/workflows/coordinator-build-and-publish.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coordinator-build-and-publish.yml b/.github/workflows/coordinator-build-and-publish.yml index a19522d91..66286299b 100644 --- a/.github/workflows/coordinator-build-and-publish.yml +++ b/.github/workflows/coordinator-build-and-publish.yml @@ -68,12 +68,16 @@ jobs: echo "TAGS=${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }},${{ env.IMAGE_NAME }}:${{ env.DEVELOP_TAG }}" >> $GITHUB_ENV - name: Checkout uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b #v4.5.0 with: distribution: temurin java-version: 21 - name: Setup Gradle - uses: gradle/actions/setup-gradle@v4 + # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. + # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md + uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1 + with: + gradle-version: 8.10.2 - name: Build dist run: | ./gradlew coordinator:app:installDist --no-daemon From d8049674d1fe5171aa151ed1a87234c552fabaec Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:44:14 +0000 Subject: [PATCH 26/31] coordinator: try fix GHA runners configs --- .github/workflows/reuse-run-e2e-tests.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml index eb4b827f5..44cd6e1da 100644 --- a/.github/workflows/reuse-run-e2e-tests.yml +++ b/.github/workflows/reuse-run-e2e-tests.yml @@ -120,11 +120,12 @@ jobs: pattern: linea-* - name: Load Docker images run: | - gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-coordinator/linea-coordinator-docker-image.tar.gz | docker load && - gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-postman/linea-postman-docker-image.tar.gz | docker load && - gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-prover/linea-prover-docker-image.tar.gz | docker load && - gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-traces-api-facade/linea-traces-api-facade-docker-image.tar.gz | docker load && - gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-transaction-exclusion-api/linea-transaction-exclusion-api-docker-image.tar.gz | docker load + pwd && ls -la && echo "GITHUB_WORKSPACE=$GITHUB_WORKSPACE" && + gunzip -c $GITHUB_WORKSPACE/linea-coordinator/linea-coordinator-docker-image.tar.gz | docker load && + gunzip -c $GITHUB_WORKSPACE/linea-postman/linea-postman-docker-image.tar.gz | docker load && + gunzip -c $GITHUB_WORKSPACE/linea-prover/linea-prover-docker-image.tar.gz | docker load && + gunzip -c $GITHUB_WORKSPACE/linea-traces-api-facade/linea-traces-api-facade-docker-image.tar.gz | docker load && + gunzip -c $GITHUB_WORKSPACE/linea-transaction-exclusion-api/linea-transaction-exclusion-api-docker-image.tar.gz | docker load shell: bash - name: Spin up fresh environment with geth tracing with retry if: ${{ inputs.tracing-engine == 'geth' }} From b248358584289d2dbe8c7c3581675465937dcfe4 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:54:13 +0000 Subject: [PATCH 27/31] coordinator: try fix GHA runners configs - revert coordinator build --- .github/workflows/coordinator-build-and-publish.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/coordinator-build-and-publish.yml b/.github/workflows/coordinator-build-and-publish.yml index 66286299b..6b96c8945 100644 --- a/.github/workflows/coordinator-build-and-publish.yml +++ b/.github/workflows/coordinator-build-and-publish.yml @@ -51,7 +51,7 @@ concurrency: jobs: build-and-publish: - runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med + runs-on: [self-hosted, ubuntu-20.04, X64, small] name: Coordinator build env: COMMIT_TAG: ${{ inputs.commit_tag }} @@ -68,16 +68,12 @@ jobs: echo "TAGS=${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }},${{ env.IMAGE_NAME }}:${{ env.DEVELOP_TAG }}" >> $GITHUB_ENV - name: Checkout uses: actions/checkout@v4 - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b #v4.5.0 + - uses: actions/setup-java@v4 with: distribution: temurin java-version: 21 - name: Setup Gradle - # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. - # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md - uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1 - with: - gradle-version: 8.10.2 + uses: gradle/actions/setup-gradle@v4 - name: Build dist run: | ./gradlew coordinator:app:installDist --no-daemon From 423ce68bb4baeef72e4744ef039efa63529c8349 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:38:14 +0000 Subject: [PATCH 28/31] coordinator: try fix GHA runners configs - revert coordinator build --- .github/workflows/coordinator-testing.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml index fdcd59704..edaf2e582 100644 --- a/.github/workflows/coordinator-testing.yml +++ b/.github/workflows/coordinator-testing.yml @@ -31,12 +31,14 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b #v4.5.0 with: distribution: temurin java-version: 21 - name: Setup Gradle - uses: gradle/actions/setup-gradle@v4 + # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. + # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md + uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1 - name: Restore cached images id: restore-cached-images uses: actions/cache/restore@v4.0.2 From 8eb3588a10b62a935fb498182cf277c374929bdb Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:43:29 +0000 Subject: [PATCH 29/31] coordinator: try fix GHA runners configs - revert coordinator build --- .github/workflows/coordinator-build-and-publish.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coordinator-build-and-publish.yml b/.github/workflows/coordinator-build-and-publish.yml index 6b96c8945..310c6f508 100644 --- a/.github/workflows/coordinator-build-and-publish.yml +++ b/.github/workflows/coordinator-build-and-publish.yml @@ -51,7 +51,7 @@ concurrency: jobs: build-and-publish: - runs-on: [self-hosted, ubuntu-20.04, X64, small] + runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med name: Coordinator build env: COMMIT_TAG: ${{ inputs.commit_tag }} @@ -68,12 +68,14 @@ jobs: echo "TAGS=${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }},${{ env.IMAGE_NAME }}:${{ env.DEVELOP_TAG }}" >> $GITHUB_ENV - name: Checkout uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b #v4.5.0 with: distribution: temurin java-version: 21 - name: Setup Gradle - uses: gradle/actions/setup-gradle@v4 + # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. + # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md + uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1 - name: Build dist run: | ./gradlew coordinator:app:installDist --no-daemon From 64c1543e1bd1f07dd5ae941902fc0c728231d3b5 Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Mon, 16 Dec 2024 16:21:07 +0000 Subject: [PATCH 30/31] coordinator: fix typos and small code cosmetics --- .../blockcreation/BlockCreationMonitorTest.kt | 2 +- .../jsonrpc/httpserver/HttpJsonRpcServer.kt | 2 +- .../linea/jsonrpc/TestingJsonRpcServerTest.kt | 2 +- .../linea/jsonrpc/TestingJsonRpcServer.kt | 8 ++-- .../linea/rlp/BlockRlpEncoderInterfaces.kt | 38 +++++-------------- .../kotlin/linea/domain/BinaryEncoding.kt | 25 ++++++++++++ .../linea/test/BlockEncodingValidator.kt | 3 +- .../test/kotlin/linea/test/BlocksFetcher.kt | 2 +- .../linea/transactionexclusion/app/api/Api.kt | 2 +- 9 files changed, 46 insertions(+), 38 deletions(-) create mode 100644 jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/BinaryEncoding.kt diff --git a/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitorTest.kt b/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitorTest.kt index 9f07d2460..0007f2a5f 100644 --- a/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitorTest.kt +++ b/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/blockcreation/BlockCreationMonitorTest.kt @@ -101,7 +101,7 @@ class BlockCreationMonitorTest { blockCreationListener = BlockCreationListenerDouble() web3jClient = ExtendedWeb3JImpl( createWeb3jHttpClient( - rpcUrl = "http://localhost:${fakeL2RpcNode.bindedPort}", + rpcUrl = "http://localhost:${fakeL2RpcNode.boundPort}", log = LogManager.getLogger("test.client.l2.web3j") ) ) diff --git a/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/httpserver/HttpJsonRpcServer.kt b/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/httpserver/HttpJsonRpcServer.kt index 40ec62416..20dc92eef 100644 --- a/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/httpserver/HttpJsonRpcServer.kt +++ b/jvm-libs/generic/json-rpc/src/main/kotlin/net/consensys/linea/jsonrpc/httpserver/HttpJsonRpcServer.kt @@ -19,7 +19,7 @@ class HttpJsonRpcServer( ) : AbstractVerticle() { private val log: Logger = LogManager.getLogger(this.javaClass) private lateinit var httpServer: HttpServer - val bindedPort: Int + val boundPort: Int get() = if (this::httpServer.isInitialized) { httpServer.actualPort() } else { diff --git a/jvm-libs/generic/json-rpc/src/test/kotlin/linea/jsonrpc/TestingJsonRpcServerTest.kt b/jvm-libs/generic/json-rpc/src/test/kotlin/linea/jsonrpc/TestingJsonRpcServerTest.kt index 033e15502..efe0c552c 100644 --- a/jvm-libs/generic/json-rpc/src/test/kotlin/linea/jsonrpc/TestingJsonRpcServerTest.kt +++ b/jvm-libs/generic/json-rpc/src/test/kotlin/linea/jsonrpc/TestingJsonRpcServerTest.kt @@ -36,7 +36,7 @@ class TestingJsonRpcServerTest { meterRegistry = SimpleMeterRegistry() ) client = rpcClientFactory.createJsonRpcV2Client( - endpoints = listOf(URI.create("http://localhost:${jsonRpcServer.bindedPort}")), + endpoints = listOf(URI.create("http://localhost:${jsonRpcServer.boundPort}")), retryConfig = RequestRetryConfig( maxRetries = 10u, backoffDelay = 10.milliseconds, diff --git a/jvm-libs/generic/json-rpc/src/testFixtures/kotlin/linea/jsonrpc/TestingJsonRpcServer.kt b/jvm-libs/generic/json-rpc/src/testFixtures/kotlin/linea/jsonrpc/TestingJsonRpcServer.kt index b1a834213..64921a6b7 100644 --- a/jvm-libs/generic/json-rpc/src/testFixtures/kotlin/linea/jsonrpc/TestingJsonRpcServer.kt +++ b/jvm-libs/generic/json-rpc/src/testFixtures/kotlin/linea/jsonrpc/TestingJsonRpcServer.kt @@ -39,8 +39,8 @@ open class TestingJsonRpcServer( ) { val log: Logger = LogManager.getLogger(loggerName) private var httpServer: HttpJsonRpcServer = createHttpServer(port) - val bindedPort: Int - get() = httpServer.bindedPort + val boundPort: Int + get() = httpServer.boundPort private var verticleId: String? = null private val handlers: MutableMap Any?> = ConcurrentHashMap() private var requests: MutableList< @@ -82,11 +82,11 @@ open class TestingJsonRpcServer( fun resumeHttpServer(): Future { // reuse the same port - httpServer = createHttpServer(bindedPort) + httpServer = createHttpServer(boundPort) return vertx .deployVerticle(httpServer, DeploymentOptions().setInstances(1)) .onSuccess { verticleId: String -> - log.info("Http server resumed at port {}", httpServer.bindedPort) + log.info("Http server resumed at port {}", httpServer.boundPort) this.verticleId = verticleId } .onFailure { th -> diff --git a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BlockRlpEncoderInterfaces.kt b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BlockRlpEncoderInterfaces.kt index e3270fa82..84a865a05 100644 --- a/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BlockRlpEncoderInterfaces.kt +++ b/jvm-libs/linea/besu-rlp-and-mappers/src/main/kotlin/linea/rlp/BlockRlpEncoderInterfaces.kt @@ -1,30 +1,12 @@ package linea.rlp -import tech.pegasys.teku.infrastructure.async.SafeFuture - -interface BlockRlpEncoder { - fun encode(block: T): ByteArray - fun encode(blocks: List): List = blocks.map { encode(it) } -} - -interface BlockRlpDecoder { - fun decode(block: ByteArray): T - fun decode(blocks: List): List = blocks.map { decode(it) } -} - -interface BlockRlpEncoderAsync { - fun encodeAsync(block: T): SafeFuture - fun encodeAsync(blocks: List): SafeFuture> = - SafeFuture.collectAll(blocks.map { encodeAsync(it) }.stream()) -} - -interface BlockRlpDecoderAsync { - fun decodeAsync(block: ByteArray): SafeFuture - fun decodeAsync(blocks: List): SafeFuture> = - SafeFuture.collectAll(blocks.map { decodeAsync(it) }.stream()) -} - -interface BesuBlockRlpEncoder : BlockRlpEncoder -interface BesuBlockRlpEncoderAsync : BlockRlpEncoderAsync -interface BesuBlockRlpDecoder : BlockRlpDecoder -interface BesuBlockRlpDecoderAsync : BlockRlpDecoderAsync +import linea.domain.BinaryDecoder +import linea.domain.BinaryDecoderAsync +import linea.domain.BinaryEncoder +import linea.domain.BinaryEncoderAsync +import org.hyperledger.besu.ethereum.core.Block + +interface BesuBlockRlpEncoder : BinaryEncoder +interface BesuBlockRlpEncoderAsync : BinaryEncoderAsync +interface BesuBlockRlpDecoder : BinaryDecoder +interface BesuBlockRlpDecoderAsync : BinaryDecoderAsync diff --git a/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/BinaryEncoding.kt b/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/BinaryEncoding.kt new file mode 100644 index 000000000..4abe14cd4 --- /dev/null +++ b/jvm-libs/linea/core/domain-models/src/main/kotlin/linea/domain/BinaryEncoding.kt @@ -0,0 +1,25 @@ +package linea.domain + +import tech.pegasys.teku.infrastructure.async.SafeFuture + +interface BinaryEncoder { + fun encode(block: T): ByteArray + fun encode(blocks: List): List = blocks.map { encode(it) } +} + +interface BinaryDecoder { + fun decode(block: ByteArray): T + fun decode(blocks: List): List = blocks.map { decode(it) } +} + +interface BinaryEncoderAsync { + fun encodeAsync(block: T): SafeFuture + fun encodeAsync(blocks: List): SafeFuture> = + SafeFuture.collectAll(blocks.map { encodeAsync(it) }.stream()) +} + +interface BinaryDecoderAsync { + fun decodeAsync(block: ByteArray): SafeFuture + fun decodeAsync(blocks: List): SafeFuture> = + SafeFuture.collectAll(blocks.map { decodeAsync(it) }.stream()) +} diff --git a/transaction-decoder-tool/src/test/kotlin/linea/test/BlockEncodingValidator.kt b/transaction-decoder-tool/src/test/kotlin/linea/test/BlockEncodingValidator.kt index fd9fb2d29..c45edb0e2 100644 --- a/transaction-decoder-tool/src/test/kotlin/linea/test/BlockEncodingValidator.kt +++ b/transaction-decoder-tool/src/test/kotlin/linea/test/BlockEncodingValidator.kt @@ -23,6 +23,7 @@ import kotlin.time.Duration.Companion.milliseconds // 100MB, much larger than a real blob, but just for testing to allow faster testing by compressing more blocks val BLOB_COMPRESSOR_SIZE: UInt = 100u * 1024u * 1024U + class BlockEncodingValidator( val vertx: Vertx, val compressorVersion: BlobCompressorVersion = BlobCompressorVersion.V1_0_1, @@ -116,7 +117,7 @@ class BlockEncodingValidator( highestValidatedBlockNumber.set(highestValidatedBlockNumber.get().coerceAtLeast(blocks.last().number)) log.info( "compression validation blocks={} finished", - CommonDomainFunctions.blockIntervalString(blocks.first().number, blocks.last().number) + originalBlockInterval ) } } diff --git a/transaction-decoder-tool/src/test/kotlin/linea/test/BlocksFetcher.kt b/transaction-decoder-tool/src/test/kotlin/linea/test/BlocksFetcher.kt index acb88c156..3165d6b8f 100644 --- a/transaction-decoder-tool/src/test/kotlin/linea/test/BlocksFetcher.kt +++ b/transaction-decoder-tool/src/test/kotlin/linea/test/BlocksFetcher.kt @@ -50,8 +50,8 @@ class BlocksFetcher( fun consumeBlocks( startBlockNumber: ULong, - chunkSize: UInt = pollingChuckSize, endBlockNumber: ULong? = null, + chunkSize: UInt = pollingChuckSize, consumer: (List) -> SafeFuture<*> ): SafeFuture<*> { val lastBlockFetched = AtomicLong(startBlockNumber.toLong() - 1) diff --git a/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/Api.kt b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/Api.kt index 78150bfde..ebb82931e 100644 --- a/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/Api.kt +++ b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/Api.kt @@ -78,7 +78,7 @@ class Api( ) .compose { verticleId: String -> jsonRpcServerId = verticleId - serverPort = httpServer!!.bindedPort + serverPort = httpServer!!.boundPort vertx.deployVerticle(observabilityServer).onSuccess { monitorVerticleId -> this.observabilityServerId = monitorVerticleId } From 91030c8acdccfbd86001d121a634569a487ad03b Mon Sep 17 00:00:00 2001 From: Pedro Novais <1478752+jpnovais@users.noreply.github.com> Date: Mon, 16 Dec 2024 16:32:53 +0000 Subject: [PATCH 31/31] coordinator: fix typo --- .../src/test/kotlin/linea/test/MainRunner.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transaction-decoder-tool/src/test/kotlin/linea/test/MainRunner.kt b/transaction-decoder-tool/src/test/kotlin/linea/test/MainRunner.kt index 6b7673a78..d98e55044 100644 --- a/transaction-decoder-tool/src/test/kotlin/linea/test/MainRunner.kt +++ b/transaction-decoder-tool/src/test/kotlin/linea/test/MainRunner.kt @@ -14,7 +14,7 @@ fun configureLoggers(loggerConfigs: List>) { } fun main() { - val rplUrl = run { + val rpcUrl = run { "https://linea-sepolia.infura.io/v3/${System.getenv("INFURA_PROJECT_ID")}" // "https://linea-mainnet.infura.io/v3/${System.getenv("INFURA_PROJECT_ID")}" } @@ -25,7 +25,7 @@ fun main() { } val fetcherAndValidate = FetchAndValidationRunner( - rpcUrl = rplUrl, + rpcUrl = rpcUrl, vertx = vertx, log = LogManager.getLogger("test.validator") )