Merge pull request #1796 from ergoplatform/v4.0.39

Candidate for 4.0.39

README.md

@@ -82,7 +82,7 @@ To run specific Ergo version `<VERSION>` as a service with custom config `/path/
         -e MAX_HEAP=3G \
         ergoplatform/ergo:<VERSION> --<networkId> -c /etc/myergo.conf
 
-Available versions can be found on [Ergo Docker image page](https://hub.docker.com/r/ergoplatform/ergo/tags), for example, `v4.0.38`.
+Available versions can be found on [Ergo Docker image page](https://hub.docker.com/r/ergoplatform/ergo/tags), for example, `v4.0.39`.
 
 This will connect to the Ergo mainnet or testnet following your configuration passed in `myergo.conf` and network flag `--<networkId>`. Every default config value would be overwritten with corresponding value in `myergo.conf`. `MAX_HEAP` variable can be used to control how much memory can the node consume.
 

src/main/resources/api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.2"
 
 info:
-  version: "4.0.38"
+  version: "4.0.39"
   title: Ergo Node API
   description: API docs for Ergo Node. Models are shared between all Ergo products
   contact:

src/main/resources/application.conf

@@ -396,7 +396,7 @@ scorex {
     nodeName = "ergo-node"
 
     # Network protocol version to be sent in handshakes
-    appVersion = 4.0.38
+    appVersion = 4.0.39
 
     # Network agent name. May contain information about client code
     # stack, starting from core code-base up to the end graphical interface.

src/main/resources/mainnet.conf

@@ -65,7 +65,7 @@ scorex {
   network {
     magicBytes = [1, 0, 2, 4]
     bindAddress = "0.0.0.0:9030"
-    nodeName = "ergo-mainnet-4.0.38"
+    nodeName = "ergo-mainnet-4.0.39"
     nodeName = ${?NODENAME}
     knownPeers = [
       "213.239.193.208:9030",

src/main/scala/org/ergoplatform/http/api/ErgoPeersApiRoute.scala

@@ -1,6 +1,6 @@
 package org.ergoplatform.http.api
 
-import java.net.{InetAddress, InetSocketAddress, URL}
+import java.net.{InetAddress, InetSocketAddress}
 import akka.actor.{ActorRef, ActorRefFactory}
 import akka.http.scaladsl.server.Route
 import akka.util.Timeout
@@ -44,7 +44,7 @@ class ErgoPeersApiRoute(peerManager: ActorRef,
   def allPeers: Route = (path("all") & get) {
     val result = askActor[Map[InetSocketAddress, PeerInfo]](peerManager, GetAllPeers).map {
       _.map { case (address, peerInfo) =>
-        PeerInfoResponse.fromAddressAndInfo(address, settings.publicUrl, peerInfo)
+        PeerInfoResponse.fromAddressAndInfo(address, peerInfo)
       }
     }
     ApiResponse(result)
@@ -121,13 +121,13 @@ object ErgoPeersApiRoute {
                               restApiUrl: Option[String])
 
   object PeerInfoResponse {
-    def fromAddressAndInfo(address: InetSocketAddress, restApiUrl: Option[URL], peerInfo: PeerInfo): PeerInfoResponse = PeerInfoResponse(
+    def fromAddressAndInfo(address: InetSocketAddress, peerInfo: PeerInfo): PeerInfoResponse = PeerInfoResponse(
       address.toString,
       0,
       peerInfo.lastHandshake,
       peerInfo.peerSpec.nodeName,
       peerInfo.connectionType.map(_.toString),
-      restApiUrl.map(_.toString)
+      peerInfo.peerSpec.publicUrlOpt.map(_.toString)
     )
 
     @SuppressWarnings(Array("org.wartremover.warts.PublicInference"))

src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala

@@ -409,8 +409,20 @@ object CandidateGenerator extends ScorexLogging {
       nextHeight >= 4096
     }
 
+    // we automatically vote for 5.0 soft-fork in the mainnet if 120 = 0 vote not provided in settings
+    val forkOrdered = if (ergoSettings.networkType.isMainNet && protocolVersion == 2) {
+      ergoSettings.votingTargets.softForkOption.getOrElse(1) == 1
+    } else {
+      ergoSettings.votingTargets.softForkOption.getOrElse(0) == 1
+    }
+
+    //todo: remove after 5.0 soft-fork activation
+    log.debug(s"betterVersion: $betterVersion, forkVotingAllowed: $forkVotingAllowed, " +
+              s"forkOrdered: $forkOrdered, nextHeightCondition: $nextHeightCondition")
+
     betterVersion &&
       forkVotingAllowed &&
+      forkOrdered &&
       nextHeightCondition
   }
 

src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala

@@ -134,7 +134,7 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input],
     val (isCostValid, scriptCost: Long) =
       costTry match {
         case Failure(t) =>
-          log.warn(s"Tx verification failed: ${t.getMessage}")
+          log.warn(s"Tx verification failed: ${t.getMessage}", t)
           log.warn(s"Tx $id verification context: " +
             s"${JsonCodecsWrapper.ergoLikeContextEncoder.apply(ctx)} " +
             s"input context: $inputContext " +

src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala

@@ -629,6 +629,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
         // (so having UTXO set, and the chain is synced
         if (!settings.nodeSettings.stateType.requireProofs &&
           hr.isHeadersChainSynced &&
+          hr.headersHeight >= syncTracker.maxHeight().getOrElse(0) &&
           hr.fullBlockHeight == hr.headersHeight) {
           val unknownMods =
             invData.ids.filter(mid => deliveryTracker.status(mid, modifierTypeId, Seq(mp)) == ModifiersStatus.Unknown)
@@ -658,7 +659,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
     */
   protected def requestMoreModifiers(historyReader: ErgoHistory): Unit = {
     if (historyReader.isHeadersChainSynced) {
-      // our requested list is is half empty - request more missed modifiers
+      // our requested list is half empty - request more missed modifiers
       self ! CheckModifiersToDownload
     } else {
       // headers chain is not synced yet, but our requested list is half empty - ask for more headers
@@ -791,7 +792,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
       syncTracker.clearStatus(connectedPeer)
   }
 
-  protected def getLocalSyncInfo(historyReader: ErgoHistory): Receive = {
+  protected def sendLocalSyncInfo(historyReader: ErgoHistory): Receive = {
     case SendLocalSyncInfo =>
       sendSync(historyReader)
   }
@@ -896,7 +897,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
   def initialized(hr: ErgoHistory, mp: ErgoMemPool, blockAppliedTxsCache: FixedSizeApproximateCacheQueue): PartialFunction[Any, Unit] = {
     processDataFromPeer(msgHandlers(hr, mp, blockAppliedTxsCache)) orElse
       onDownloadRequest(hr) orElse
-      getLocalSyncInfo(hr) orElse
+      sendLocalSyncInfo(hr) orElse
       viewHolderEvents(hr, mp, blockAppliedTxsCache) orElse
       peerManagerEvents orElse
       checkDelivery orElse {

src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala

@@ -333,7 +333,9 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti
           applyFromCacheLoop()
           val cleared = modifiersCache.cleanOverfull()
 
-          context.system.eventStream.publish(ModifiersRemovedFromCache(cleared))
+          if (cleared.nonEmpty) {
+            context.system.eventStream.publish(ModifiersRemovedFromCache(cleared))
+          }
           log.debug(s"Cache size after: ${modifiersCache.size}")
       }
   }

src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala

@@ -9,11 +9,12 @@ import org.ergoplatform.settings.{ErgoSettings, MonetarySettings, NodeConfigurat
 import scorex.core.transaction.state.TransactionValidation
 import scorex.util.{ModifierId, ScorexLogging, bytesToId}
 import OrderedTxPool.weighted
+import org.ergoplatform.nodeView.mempool.ErgoMemPool.SortingOption
 import spire.syntax.all.cfor
 
 import scala.annotation.tailrec
 import scala.collection.mutable
-import scala.util.Try
+import scala.util.{Failure, Random, Success, Try}
 
 
 /**
@@ -23,14 +24,22 @@ import scala.util.Try
   *                 used for implementing all transaction-related methods
   * @param stats    - Mempool statistics, that allows to track
   *                 information about mempool's state and transactions in it.
+  * @param sortingOption - this input sets how transactions are sorted in the pool, by fee-per-byte or fee-per-cycle
   */
-class ErgoMemPool private[mempool](pool: OrderedTxPool,
-                                   private[mempool] val stats : MemPoolStatistics)(implicit settings: ErgoSettings)
+class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool,
+                                   private[mempool] val stats: MemPoolStatistics,
+                                   private[mempool] val sortingOption: SortingOption)
+                                  (implicit settings: ErgoSettings)
   extends ErgoMemPoolReader with ScorexLogging {
 
   import ErgoMemPool._
   import EmissionRules.CoinsInOneErgo
 
+  /**
+    * When there's no reason to re-check transactions immediately, we assign fake fee factor to them
+    */
+  private val FakeFeeFactor = 1000
+
   private val nodeSettings: NodeConfigurationSettings = settings.nodeSettings
   private implicit val monetarySettings: MonetarySettings = settings.chainSettings.monetary
 
@@ -70,8 +79,9 @@ class ErgoMemPool private[mempool](pool: OrderedTxPool,
   override def getAllPrioritized: Seq[ErgoTransaction] = pool.orderedTransactions.values.toSeq
 
   /**
-    * Method to put a transaction into the memory pool. Validation of tha transactions against
+    * Method to put a transaction into the memory pool. Validation of the transactions against
     * the state is done in NodeVieHolder. This put() method can check whether a transaction is valid
+    *
     * @param tx
     * @return Success(updatedPool), if transaction successfully added to the pool, Failure(_) otherwise
     */
@@ -82,19 +92,19 @@ class ErgoMemPool private[mempool](pool: OrderedTxPool,
   }
 
   def putWithoutCheck(txs: Iterable[ErgoTransaction]): ErgoMemPool = {
-    val updatedPool = txs.toSeq.distinct.foldLeft(pool) { case (acc, tx) => acc.put(tx) }
-    new ErgoMemPool(updatedPool, stats)
+    val updatedPool = txs.toSeq.distinct.foldLeft(pool) { case (acc, tx) => acc.put(tx, FakeFeeFactor) }
+    new ErgoMemPool(updatedPool, stats, sortingOption)
   }
 
   def remove(tx: ErgoTransaction): ErgoMemPool = {
     val wtx = pool.transactionsRegistry.get(tx.id)
     val updStats = wtx.map(wgtx => stats.add(System.currentTimeMillis(), wgtx))
       .getOrElse(MemPoolStatistics(System.currentTimeMillis(), 0, System.currentTimeMillis()))
-    new ErgoMemPool(pool.remove(tx), updStats)
+    new ErgoMemPool(pool.remove(tx), updStats, sortingOption)
   }
 
   def filter(condition: ErgoTransaction => Boolean): ErgoMemPool = {
-    new ErgoMemPool(pool.filter(condition), stats)
+    new ErgoMemPool(pool.filter(condition), stats, sortingOption)
   }
 
   def filter(txs: Seq[ErgoTransaction]): ErgoMemPool = filter(t => !txs.exists(_.id == t.id))
@@ -105,7 +115,7 @@ class ErgoMemPool private[mempool](pool: OrderedTxPool,
     * @param tx - Transaction to invalidate
     */
   def invalidate(tx: ErgoTransaction): ErgoMemPool = {
-    new ErgoMemPool(pool.invalidate(tx), stats)
+    new ErgoMemPool(pool.invalidate(tx), stats, sortingOption)
   }
 
   /**
@@ -116,61 +126,75 @@ class ErgoMemPool private[mempool](pool: OrderedTxPool,
   // Check if transaction is double-spending inputs spent in the mempool.
   // If so, the new transacting is replacing older ones if it has bigger weight (fee/byte) than them on average.
   // Otherwise, the new transaction being rejected.
-  private def acceptIfNoDoubleSpend(tx: ErgoTransaction): (ErgoMemPool, ProcessingOutcome) = {
+  private def acceptIfNoDoubleSpend(tx: ErgoTransaction, cost: Int): (ErgoMemPool, ProcessingOutcome) = {
+    val feeFactor = sortingOption match {
+      case SortingOption.FeePerByte => tx.size
+      case SortingOption.FeePerCycle => cost
+    }
+
     val doubleSpendingWtxs = tx.inputs.flatMap { inp =>
       pool.inputs.get(inp.boxId)
     }.toSet
 
-    if(doubleSpendingWtxs.nonEmpty) {
-      val ownWtx = weighted(tx)
+    if (doubleSpendingWtxs.nonEmpty) {
+      val ownWtx = weighted(tx, feeFactor)
       val doubleSpendingTotalWeight = doubleSpendingWtxs.map(_.weight).sum / doubleSpendingWtxs.size
       if (ownWtx.weight > doubleSpendingTotalWeight) {
         val doubleSpendingTxs = doubleSpendingWtxs.map(wtx => pool.orderedTransactions(wtx)).toSeq
-        new ErgoMemPool(pool.put(tx).remove(doubleSpendingTxs), stats) -> ProcessingOutcome.Accepted
+        new ErgoMemPool(pool.put(tx, feeFactor).remove(doubleSpendingTxs), stats, sortingOption) -> ProcessingOutcome.Accepted
       } else {
         this -> ProcessingOutcome.DoubleSpendingLoser(doubleSpendingWtxs.map(_.id))
       }
     } else {
-      new ErgoMemPool(pool.put(tx), stats) -> ProcessingOutcome.Accepted
+      val poolSizeLimit = nodeSettings.mempoolCapacity
+      if (pool.size == poolSizeLimit &&
+        weighted(tx, feeFactor).weight <= pool.orderedTransactions.lastKey.weight) {
+        this -> ProcessingOutcome.Declined(new Exception("Transaction pays less than any other in the pool being full"))
+      } else {
+        new ErgoMemPool(pool.put(tx, feeFactor), stats, sortingOption) -> ProcessingOutcome.Accepted
+      }
     }
   }
 
   def process(tx: ErgoTransaction, state: ErgoState[_]): (ErgoMemPool, ProcessingOutcome) = {
     log.info(s"Processing mempool transaction: $tx")
 
     val blacklistedTransactions = nodeSettings.blacklistedTransactions
-    if(blacklistedTransactions.nonEmpty && blacklistedTransactions.contains(tx.id)) {
-      new ErgoMemPool(pool.invalidate(tx), stats) -> ProcessingOutcome.Invalidated(new Exception("blacklisted tx"))
+    if (blacklistedTransactions.nonEmpty && blacklistedTransactions.contains(tx.id)) {
+      new ErgoMemPool(pool.invalidate(tx), stats, sortingOption) -> ProcessingOutcome.Invalidated(new Exception("blacklisted tx"))
     } else {
       val fee = extractFee(tx)
       val minFee = settings.nodeSettings.minimalFeeAmount
       val canAccept = pool.canAccept(tx)
 
       if (fee >= minFee) {
         if (canAccept) {
+          val costLimit = nodeSettings.maxTransactionCost
           state match {
             case utxo: UtxoState =>
               // Allow proceeded transaction to spend outputs of pooled transactions.
               val utxoWithPool = utxo.withTransactions(getAll)
               if (tx.inputIds.forall(inputBoxId => utxoWithPool.boxById(inputBoxId).isDefined)) {
-                utxoWithPool.validateWithCost(tx, Some(utxo.stateContext), nodeSettings.maxTransactionCost, None).fold(
-                  ex => new ErgoMemPool(pool.invalidate(tx), stats) -> ProcessingOutcome.Invalidated(ex),
-                  _ => acceptIfNoDoubleSpend(tx)
-                )
+                utxoWithPool.validateWithCost(tx, Some(utxo.stateContext), costLimit, None) match {
+                  case Success(cost) => acceptIfNoDoubleSpend(tx, cost)
+                  case Failure(ex) => new ErgoMemPool(pool.invalidate(tx), stats, sortingOption) -> ProcessingOutcome.Invalidated(ex)
+                }
               } else {
                 this -> ProcessingOutcome.Declined(new Exception("not all utxos in place yet"))
               }
             case validator: TransactionValidation =>
               // transaction validation currently works only for UtxoState, so this branch currently
               // will not be triggered probably
-              validator.validateWithCost(tx, nodeSettings.maxTransactionCost).fold(
-                ex => new ErgoMemPool(pool.invalidate(tx), stats) -> ProcessingOutcome.Invalidated(ex),
-                _ => acceptIfNoDoubleSpend(tx)
-              )
+              validator.validateWithCost(tx, costLimit) match {
+                case Success(cost) => acceptIfNoDoubleSpend(tx, cost)
+                case Failure(ex) => new ErgoMemPool(pool.invalidate(tx), stats, sortingOption) -> ProcessingOutcome.Invalidated(ex)
+              }
             case _ =>
               // Accept transaction in case of "digest" state. Transactions are not downloaded in this mode from other
               // peers though, so such transactions can come from the local wallet only.
-              acceptIfNoDoubleSpend(tx)
+              //
+              // We pass fake cost in this case, as there's no real competition between local transactions only anyway
+              acceptIfNoDoubleSpend(tx, cost = FakeFeeFactor)
           }
         } else {
           this -> ProcessingOutcome.Declined(
@@ -216,6 +240,7 @@ class ErgoMemPool private[mempool](pool: OrderedTxPool,
     * Calculate position in mempool corresponding to the specified fee and
     * estimate time of serving this transaction based on average rate of placing
     * transactions in blockchain
+    *
     * @param txFee  - transaction fee
     * @param txSize - size of transaction (in bytes)
     * @return average time for this transaction to be placed in block
@@ -244,7 +269,36 @@ class ErgoMemPool private[mempool](pool: OrderedTxPool,
   def getReader: ErgoMemPoolReader = this
 }
 
-object ErgoMemPool {
+object ErgoMemPool extends ScorexLogging {
+
+  /**
+   * Hierarchy of sorting strategies for mempool transactions
+   */
+  sealed trait SortingOption
+
+  object SortingOption {
+    /**
+      * Sort transactions by fee paid for transaction size, so fee/byte
+      */
+    case object FeePerByte extends SortingOption
+
+    /**
+      * Sort transactions by fee paid for transaction contracts validation cost, so fee/execution unit
+      */
+    case object FeePerCycle extends SortingOption
+
+    /**
+      * @return randomly chosen mempool sorting strategy
+      */
+    def random(): SortingOption = {
+      if (Random.nextBoolean()) {
+        FeePerByte
+      } else {
+        FeePerCycle
+      }
+    }
+  }
+
 
   sealed trait ProcessingOutcome
 
@@ -258,6 +312,7 @@ object ErgoMemPool {
     /**
       * Class signalling that a valid transaction was rejected as it is double-spending inputs of mempool transactions
       * and has no bigger weight (fee/byte) than them on average.
+      *
       * @param winnerTxIds - identifiers of transactions won in replace-by-fee auction
       */
     case class DoubleSpendingLoser(winnerTxIds: Set[ModifierId]) extends ProcessingOutcome
@@ -277,11 +332,20 @@ object ErgoMemPool {
 
   /**
     * Create empty mempool
+    *
     * @param settings - node settings (to get mempool settings from)
+    * @param sortingOption - how to sort transactions (by size or execution cost)
     * @return empty mempool
     */
-  def empty(settings: ErgoSettings): ErgoMemPool =
+  def empty(settings: ErgoSettings, sortingOption: SortingOption = SortingOption.random()): ErgoMemPool = {
+    sortingOption match {
+      case SortingOption.FeePerByte => log.info("Sorting mempool by fee-per-byte")
+      case SortingOption.FeePerCycle => log.info("Sorting mempool by fee-per-cycle")
+    }
     new ErgoMemPool(OrderedTxPool.empty(settings),
-      MemPoolStatistics(System.currentTimeMillis(), 0, System.currentTimeMillis()))(settings)
+      MemPoolStatistics(System.currentTimeMillis(), 0, System.currentTimeMillis()),
+      sortingOption
+    )(settings)
+  }
 
 }