Coordinator: replace teku ExecutionPlayloadV1 by Domain Block class - fix block encoding

.github/workflows/coordinator-build-and-publish.yml

@@ -51,7 +51,7 @@ concurrency:
 
 jobs:
   build-and-publish:
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
     name: Coordinator build
     env:
       COMMIT_TAG: ${{ inputs.commit_tag }}
@@ -68,12 +68,14 @@ jobs:
           echo "TAGS=${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }},${{ env.IMAGE_NAME }}:${{ env.DEVELOP_TAG }}" >> $GITHUB_ENV
       - name: Checkout
         uses: actions/checkout@v4
-      - uses: actions/setup-java@v4
+      - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b #v4.5.0
         with:
           distribution: temurin
           java-version: 21
       - name: Setup Gradle
-        uses: gradle/actions/setup-gradle@v4
+        # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
+        # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
+        uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
       - name: Build dist
         run: |
           ./gradlew coordinator:app:installDist --no-daemon

.github/workflows/coordinator-testing.yml

@@ -26,17 +26,19 @@ jobs:
       GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN_RELEASE_ACCESS }}
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-    runs-on: [self-hosted, ubuntu-22.04, X64, medium]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-large
     name: Coordinator tests
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - uses: actions/setup-java@v4
+      - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b #v4.5.0
         with:
           distribution: temurin
           java-version: 21
       - name: Setup Gradle
-        uses: gradle/actions/setup-gradle@v4
+        # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
+        # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
+        uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
       - name: Restore cached images
         id: restore-cached-images
         uses: actions/cache/[email protected]

.github/workflows/maven-release.yml

@@ -13,7 +13,7 @@ on:
 
 jobs:
   release:
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

.github/workflows/reuse-run-e2e-tests.yml

@@ -76,7 +76,7 @@ jobs:
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
     outputs:
       tests_outcome: ${{ steps.run_e2e_tests.outcome }}
-    runs-on: [self-hosted, ubuntu-20.04, X64, large]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-large
     steps:
       - name: Setup upterm session
         if: ${{ inputs.e2e-tests-with-ssh }}
@@ -116,13 +116,16 @@ jobs:
             make pull-images-external-to-monorepo
       - name: Download local docker image artifacts
         uses: actions/download-artifact@v4
+        with:
+          pattern: linea-*
       - name: Load Docker images
         run: |
-          gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-coordinator/linea-coordinator-docker-image.tar.gz | docker load &&
-          gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-postman/linea-postman-docker-image.tar.gz | docker load &&
-          gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-prover/linea-prover-docker-image.tar.gz | docker load &&
-          gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-traces-api-facade/linea-traces-api-facade-docker-image.tar.gz | docker load &&
-          gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-transaction-exclusion-api/linea-transaction-exclusion-api-docker-image.tar.gz | docker load
+          pwd && ls -la && echo "GITHUB_WORKSPACE=$GITHUB_WORKSPACE" &&
+          gunzip -c $GITHUB_WORKSPACE/linea-coordinator/linea-coordinator-docker-image.tar.gz | docker load &&
+          gunzip -c $GITHUB_WORKSPACE/linea-postman/linea-postman-docker-image.tar.gz | docker load &&
+          gunzip -c $GITHUB_WORKSPACE/linea-prover/linea-prover-docker-image.tar.gz | docker load &&
+          gunzip -c $GITHUB_WORKSPACE/linea-traces-api-facade/linea-traces-api-facade-docker-image.tar.gz | docker load &&
+          gunzip -c $GITHUB_WORKSPACE/linea-transaction-exclusion-api/linea-transaction-exclusion-api-docker-image.tar.gz | docker load
         shell: bash
       - name: Spin up fresh environment with geth tracing with retry
         if: ${{ inputs.tracing-engine == 'geth' }}

.github/workflows/traces-api-facade-build-and-publish.yml

@@ -51,7 +51,7 @@ concurrency:
 
 jobs:
   build-and-publish:
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
     name: Traces api facade build
     env:
       COMMIT_TAG: ${{ inputs.commit_tag }}

.github/workflows/transaction-exclusion-api-build-and-publish.yml

@@ -51,7 +51,7 @@ concurrency:
 
 jobs:
   build-and-publish:
-    runs-on: [self-hosted, ubuntu-20.04, X64, small]
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
     name: Transaction exclusion api build
     env:
       COMMIT_TAG: ${{ inputs.commit_tag }}

coordinator/app/build.gradle

@@ -41,8 +41,6 @@ dependencies {
   implementation project(':coordinator:persistence:batch')
   implementation project(':coordinator:persistence:feehistory')
   implementation project(':coordinator:persistence:db-common')
-  implementation project(":jvm-libs:linea:teku-execution-client")
-  implementation "tech.pegasys.teku.internal:bytes:${libs.versions.teku.get()}"
 
   implementation project(':coordinator:ethereum:gas-pricing:static-cap')
   implementation project(':coordinator:ethereum:gas-pricing:dynamic-cap')
@@ -66,8 +64,10 @@ dependencies {
   implementation "com.fasterxml.jackson.module:jackson-module-kotlin:${libs.versions.jackson.get()}"
   implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${libs.versions.jackson.get()}")
   testImplementation "org.apache.logging.log4j:log4j-slf4j2-impl:${libs.versions.log4j.get()}"
+  testImplementation project(':jvm-libs:generic:serialization:jackson')
+  testImplementation testFixtures(project(':jvm-libs:linea:core:domain-models'))
+  testImplementation testFixtures(project(':jvm-libs:generic:json-rpc'))
   testImplementation project(':coordinator:ethereum:test-utils')
-  testImplementation project(':jvm-libs:linea:testing:teku-helper')
   testImplementation "io.vertx:vertx-junit5"
 }
 

coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/CoordinatorApp.kt

@@ -1,9 +1,7 @@
 package net.consensys.zkevm.coordinator.app
 
-import com.fasterxml.jackson.databind.module.SimpleModule
 import io.micrometer.core.instrument.MeterRegistry
 import io.vertx.core.Vertx
-import io.vertx.core.json.jackson.DatabindCodec
 import io.vertx.micrometer.backends.BackendRegistries
 import io.vertx.sqlclient.SqlClient
 import net.consensys.linea.async.toSafeFuture
@@ -32,11 +30,9 @@ import net.consensys.zkevm.persistence.db.PersistenceRetryer
 import org.apache.logging.log4j.Level
 import org.apache.logging.log4j.LogManager
 import org.apache.logging.log4j.Logger
-import org.apache.tuweni.bytes.Bytes
 import org.web3j.protocol.Web3j
 import org.web3j.protocol.http.HttpService
 import org.web3j.utils.Async
-import tech.pegasys.teku.ethereum.executionclient.serialization.BytesSerializer
 import tech.pegasys.teku.infrastructure.async.SafeFuture
 import kotlin.time.toKotlinDuration
 
@@ -48,12 +44,6 @@ class CoordinatorApp(private val configs: CoordinatorConfig) {
     log.debug("Vertx full configs: {}", vertxConfig)
     log.info("App configs: {}", configs)
 
-    // TODO: adapt JsonMessageProcessor to use custom ObjectMapper
-    // this is just dark magic.
-    val module = SimpleModule()
-    module.addSerializer(Bytes::class.java, BytesSerializer())
-    DatabindCodec.mapper().registerModule(module)
-    // .enable(SerializationFeature.INDENT_OUTPUT)
     Vertx.vertx(vertxConfig)
   }
   private val meterRegistry: MeterRegistry = BackendRegistries.getDefaultNow()

coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt

@@ -6,6 +6,7 @@ import build.linea.contract.l1.LineaRollupSmartContractClientReadOnly
 import build.linea.contract.l1.Web3JLineaRollupSmartContractClientReadOnly
 import io.vertx.core.Vertx
 import kotlinx.datetime.Clock
+import linea.encoding.BlockRLPEncoder
 import net.consensys.linea.BlockNumberAndHash
 import net.consensys.linea.blob.ShnarfCalculatorVersion
 import net.consensys.linea.contract.LineaRollupAsyncFriendly
@@ -56,7 +57,6 @@ import net.consensys.zkevm.coordinator.clients.smartcontract.LineaRollupSmartCon
 import net.consensys.zkevm.domain.BlobSubmittedEvent
 import net.consensys.zkevm.domain.BlocksConflation
 import net.consensys.zkevm.domain.FinalizationSubmittedEvent
-import net.consensys.zkevm.encoding.ExecutionPayloadV1RLPEncoderByBesuImplementation
 import net.consensys.zkevm.ethereum.coordination.EventDispatcher
 import net.consensys.zkevm.ethereum.coordination.HighestConflationTracker
 import net.consensys.zkevm.ethereum.coordination.HighestProvenBatchTracker
@@ -323,7 +323,7 @@ class L1DependentApp(
         lastBlockNumber = lastProcessedBlockNumber,
         clock = Clock.System,
         latestBlockProvider = GethCliqueSafeBlockProvider(
-          l2ExtendedWeb3j,
+          l2ExtendedWeb3j.web3jClient,
           GethCliqueSafeBlockProvider.Config(configs.l2.blocksToFinalization.toLong())
         )
       )
@@ -608,7 +608,7 @@ class L1DependentApp(
         deadlineCheckInterval = configs.proofAggregation.deadlineCheckInterval.toKotlinDuration(),
         aggregationDeadline = configs.proofAggregation.aggregationDeadline.toKotlinDuration(),
         latestBlockProvider = GethCliqueSafeBlockProvider(
-          l2ExtendedWeb3j,
+          l2ExtendedWeb3j.web3jClient,
           GethCliqueSafeBlockProvider.Config(configs.l2.blocksToFinalization.toLong())
         ),
         maxProofsPerAggregation = configs.proofAggregation.aggregationProofsLimit.toUInt(),
@@ -868,7 +868,7 @@ class L1DependentApp(
       conflationService = conflationService,
       tracesCountersClient = tracesCountersClient,
       vertx = vertx,
-      payloadEncoder = ExecutionPayloadV1RLPEncoderByBesuImplementation
+      encoder = BlockRLPEncoder
     )
   }
 
@@ -904,7 +904,7 @@ class L1DependentApp(
     log.info("Resuming conflation from block={} inclusive", lastProcessedBlockNumber + 1UL)
     val blockCreationMonitor = BlockCreationMonitor(
       vertx = vertx,
-      extendedWeb3j = l2ExtendedWeb3j,
+      web3j = l2ExtendedWeb3j,
       startingBlockNumberExclusive = lastProcessedBlockNumber.toLong(),
       blockCreationListener = block2BatchCoordinator,
       lastProvenBlockNumberProviderAsync = lastProvenBlockNumberProvider,