Merge pull request #308 from ergoplatform/i303

Node crash fix

build.sbt

@@ -14,7 +14,7 @@ resolvers ++= Seq("Sonatype Releases" at "https://oss.sonatype.org/content/repos
   "Typesafe maven releases" at "http://repo.typesafe.com/typesafe/maven-releases/",
   "Sonatype Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots/")
 
-val scorexVersion = "d5c4180c-SNAPSHOT"
+val scorexVersion = "effd499c-SNAPSHOT"
 
 libraryDependencies ++= Seq(
   "org.scorexfoundation" %% "scrypto" % "2.1.2",

lock.sbt

@@ -49,7 +49,7 @@ dependencyOverrides in ThisBuild ++= Seq(
   "org.scalacheck" % "scalacheck_2.12" % "1.13.5",
   "org.scorexfoundation" % "avl-iodb_2.12" % "0.2.14",
   "org.scorexfoundation" % "iodb_2.12" % "0.3.2",
-  "org.scorexfoundation" % "scorex-core_2.12" % "d5c4180c-SNAPSHOT",
+  "org.scorexfoundation" % "scorex-core_2.12" % "effd499c-SNAPSHOT",
   "org.scorexfoundation" % "scrypto_2.12" % "2.1.2",
   "org.scorexfoundation" % "sigma-state_2.12" % "0.9.3",
   "org.slf4j" % "slf4j-api" % "1.8.0-beta1",
@@ -61,4 +61,4 @@ dependencyOverrides in ThisBuild ++= Seq(
   "org.typelevel" % "macro-compat_2.12" % "1.1.1",
   "org.whispersystems" % "curve25519-java" % "0.5.0"
 )
-// LIBRARY_DEPENDENCIES_HASH 5d2ab45def58eece6d403c3b6dd781213afad31f
+// LIBRARY_DEPENDENCIES_HASH 7d405ddb9d88825e46f3cb9f988354e2c253876d

src/main/resources/logback.xml

@@ -1,18 +1,41 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <configuration>
+    <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator"/>
 
-    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+    <property name="default.pattern" value="%d{HH:mm:ss.SSS} %-5level [%.25thread] %logger{26} - %msg%n"/>
+    <!--<property name="logback.file.final-directory" value="${logback.file.directory:-${ergo.directory}}"/>-->
+
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <target>System.out</target>
         <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
-            <level>DEBUG</level>
+            <level>${logback.stdout.level:-INFO}</level>
         </filter>
         <encoder>
-            <pattern>[%thread] >> [%-5level] %logger{36} >> %d{HH:mm:ss.SSS} %msg%n</pattern>
+            <pattern>${logback.pattern:-${default.pattern}}</pattern>
         </encoder>
     </appender>
 
-    <root>
-        <appender-ref ref="CONSOLE"/>
-    </root>
+    <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>${logback.file.level:-DEBUG}</level>
+        </filter>
+        <file>ergo.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <!-- daily rollover -->
+            <fileNamePattern>ergo.%d{yyyy-MM-dd}.log.gz</fileNamePattern>
+
+            <!-- keep 30 days' worth of history capped at 1GB total size -->
+            <maxHistory>30</maxHistory>
+            <totalSizeCap>1GB</totalSizeCap>
+        </rollingPolicy>
 
+        <encoder>
+            <pattern>${default.pattern}</pattern>
+        </encoder>
+    </appender>
+
+    <root level="TRACE">
+        <appender-ref ref="STDOUT"/>
+        <appender-ref ref="FILE"/>
+    </root>
 </configuration>

src/main/scala/org/ergoplatform/ErgoApp.scala

@@ -30,6 +30,7 @@ class ErgoApp(args: Seq[String]) extends Application {
   implicit val ec: ExecutionContextExecutor = actorSystem.dispatcher
 
   lazy val ergoSettings: ErgoSettings = ErgoSettings.read(args.headOption)
+
   lazy val emission = new CoinsEmission(ergoSettings.chainSettings.monetary)
 
   override implicit lazy val settings: ScorexSettings = ergoSettings.scorexSettings

src/main/scala/org/ergoplatform/local/ErgoMiner.scala

@@ -199,6 +199,7 @@ object ErgoMiner extends ScorexLogging {
                      emission: CoinsEmission): ErgoTransaction = {
     state.emissionBox() match {
       case Some(emissionBox) =>
+        assert(state.boxById(emissionBox.id).isDefined, s"Emission box ${Algos.encode(emissionBox.id)} missed")
         ErgoMiner.createCoinbase(height, feeBoxes, emissionBox, minerProp, emission)
       case None =>
         // TODO extract fees when emission is finished

src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala

@@ -54,22 +54,32 @@ trait ErgoHistoryReader
     bestFullBlockIdOpt.flatMap(id => typedModifierById[Header](id)).flatMap(getFullBlock)
 
   /**
-    * Get ErgoPersistentModifier by it's id if it is in history
+    * @param id - modifier id
+    * @return semantically valid ErgoPersistentModifier with the given id it is in history
     */
   override def modifierById(id: ModifierId): Option[ErgoPersistentModifier] = {
-    historyStorage.modifierById(id)
-      .ensuring(_.forall(_.id sameElements id), s"Modifier ${Algos.encode(id)} id is incorrect")
-  }
+    if (isSemanticallyValid(id) != ModifierSemanticValidity.Invalid) {
+      historyStorage.modifierById(id)
+    } else {
+      None
+    }
+  }.ensuring(_.forall(_.id sameElements id), s"Modifier ${Algos.encode(id)} id is incorrect")
 
   /**
-    * Get ErgoPersistentModifier of type T by it's id if it is in history
+    *
+    * @param id - modifier id
+    * @tparam T - expected Type
+    * @return semantically valid ErgoPersistentModifier of type T with the given id it is in history
     */
   @SuppressWarnings(Array("IsInstanceOf"))
   def typedModifierById[T <: ErgoPersistentModifier](id: ModifierId): Option[T] = modifierById(id) match {
     case Some(m: T@unchecked) if m.isInstanceOf[T] => Some(m)
     case _ => None
   }
 
+
+  override def contains(id: ModifierId): Boolean = historyStorage.contains(id)
+
   /**
     * Id of best block to mine
     */

src/main/scala/org/ergoplatform/nodeView/history/storage/FilesObjectsStore.scala

@@ -25,5 +25,7 @@ class FilesObjectsStore(dir: String) extends ObjectsStore {
     Files.delete(path(id))
   }
 
+  override def contains(id: ModifierId): Boolean = Files.exists(path(id))
+
   private def path(id: ModifierId) = Paths.get(dir + "/" + Algos.encode(id))
 }

src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala

@@ -17,7 +17,7 @@ class HistoryStorage(indexStore: Store, objectsStore: ObjectsStore) extends Scor
         case Success(b) =>
           Some(b)
         case Failure(e) =>
-          log.warn(s"Failed to parse block ${encoder.encode(id)} from db (bytes are: ${bBytes.mkString("-")}): ", e)
+          log.warn(s"Failed to parse modifier ${encoder.encode(id)} from db (bytes are: ${bBytes.mkString("-")}): ", e)
           None
       }
     }
@@ -26,7 +26,7 @@ class HistoryStorage(indexStore: Store, objectsStore: ObjectsStore) extends Scor
 
   def get(id: ModifierId): Option[Array[Byte]] = objectsStore.get(id)
 
-  def contains(id: ModifierId): Boolean = get(id).isDefined
+  def contains(id: ModifierId): Boolean = objectsStore.contains(id)
 
   def insert(id: ByteArrayWrapper,
              indexesToInsert: Seq[(ByteArrayWrapper, ByteArrayWrapper)],

src/main/scala/org/ergoplatform/nodeView/history/storage/ObjectsStore.scala

@@ -13,4 +13,6 @@ trait ObjectsStore {
 
   def put(m: ErgoPersistentModifier): Try[Unit]
 
+  def contains(id: ModifierId): Boolean
+
 }

src/test/scala/org/ergoplatform/nodeView/ErgoNodeViewHolderSpecification.scala

@@ -270,45 +270,59 @@ class ErgoNodeViewHolderSpecification extends ErgoPropertyTest with BeforeAndAft
     val (us, bh) = createUtxoState(Some(nodeViewRef))
     val genesis = validFullBlock(parentOpt = None, us, bh)
     val wusAfterGenesis = WrappedUtxoState(us, bh, stateConstants).applyModifier(genesis).get
-
-    nodeViewRef ! LocallyGeneratedModifier(genesis.header)
+    // TODO looks like another bug is still present here, see https://github.com/ergoplatform/ergo/issues/309
     if (nodeViewConfig.verifyTransactions) {
-      nodeViewRef ! LocallyGeneratedModifier(genesis.blockTransactions)
-      nodeViewRef ! LocallyGeneratedModifier(genesis.aDProofs.get)
-    }
+      nodeViewRef ! LocallyGeneratedModifier(genesis.header)
+      if (nodeViewConfig.verifyTransactions) {
+        nodeViewRef ! LocallyGeneratedModifier(genesis.blockTransactions)
+        nodeViewRef ! LocallyGeneratedModifier(genesis.aDProofs.get)
+      }
 
-    val block = validFullBlock(Some(genesis.header), wusAfterGenesis)
-    val wusAfterBlock = wusAfterGenesis.applyModifier(block).get
+      val block = validFullBlock(Some(genesis.header), wusAfterGenesis)
+      val wusAfterBlock = wusAfterGenesis.applyModifier(block).get
 
-    nodeViewRef ! LocallyGeneratedModifier(block.header)
-    if (nodeViewConfig.verifyTransactions) {
-      nodeViewRef ! LocallyGeneratedModifier(block.blockTransactions)
-      nodeViewRef ! LocallyGeneratedModifier(block.aDProofs.get)
-      nodeViewRef ! rootHash(nodeViewConfig)
-      expectMsg(Algos.encode(wusAfterBlock.rootHash))
-    }
+      nodeViewRef ! LocallyGeneratedModifier(block.header)
+      if (nodeViewConfig.verifyTransactions) {
+        nodeViewRef ! LocallyGeneratedModifier(block.blockTransactions)
+        nodeViewRef ! LocallyGeneratedModifier(block.aDProofs.get)
+        nodeViewRef ! rootHash(nodeViewConfig)
+        expectMsg(Algos.encode(wusAfterBlock.rootHash))
+      }
 
-    nodeViewRef ! bestHeaderOpt(nodeViewConfig)
-    expectMsg(Some(block.header))
+      nodeViewRef ! bestHeaderOpt(nodeViewConfig)
+      expectMsg(Some(block.header))
 
-    val brokenBlock = validFullBlock(Some(block.header), wusAfterBlock)
-    val headTx = brokenBlock.blockTransactions.txs.head
-    val newInput = headTx.inputs.head.copy(boxId = ADKey @@ Algos.hash("wrong input"))
+      val brokenBlock = generateInvalidFullBlock(block.header, wusAfterBlock)
+      nodeViewRef ! LocallyGeneratedModifier(brokenBlock.header)
+      nodeViewRef ! LocallyGeneratedModifier(brokenBlock.blockTransactions)
+      nodeViewRef ! LocallyGeneratedModifier(brokenBlock.aDProofs.get)
 
-    nodeViewRef ! LocallyGeneratedModifier(brokenBlock.header)
+      val brokenBlock2 = generateInvalidFullBlock(block.header, wusAfterBlock)
+      brokenBlock2.header should not be brokenBlock.header
+      nodeViewRef ! LocallyGeneratedModifier(brokenBlock2.header)
+      nodeViewRef ! LocallyGeneratedModifier(brokenBlock2.blockTransactions)
+      nodeViewRef ! LocallyGeneratedModifier(brokenBlock2.aDProofs.get)
 
-    val brokenTransactions = brokenBlock.blockTransactions
-      .copy(txs = headTx.copy(inputs = newInput +: headTx.inputs.tail) +: brokenBlock.blockTransactions.txs.tail)
-    if (nodeViewConfig.verifyTransactions) {
-      nodeViewRef ! LocallyGeneratedModifier(brokenTransactions)
-      nodeViewRef ! LocallyGeneratedModifier(brokenBlock.aDProofs.get)
       nodeViewRef ! bestFullBlock(nodeViewConfig)
       expectMsg(Some(block))
+      nodeViewRef ! rootHash(nodeViewConfig)
+      expectMsg(Algos.encode(wusAfterBlock.rootHash))
+      nodeViewRef ! bestHeaderOpt(nodeViewConfig)
+      expectMsg(Some(block.header))
+
     }
+  }
 
-    nodeViewRef ! bestHeaderOpt(nodeViewConfig)
-    //TODO Note and verify!
-    expectMsg(Some(brokenBlock.header))
+  private def generateInvalidFullBlock(parentHeader: Header, parentState: WrappedUtxoState) = {
+    val brokenBlockIn = validFullBlock(Some(parentHeader), parentState)
+    val headTx = brokenBlockIn.blockTransactions.txs.head
+    val newInput = headTx.inputs.head.copy(boxId = ADKey @@ Algos.hash("wrong input"))
+    val brokenTransactionsIn = brokenBlockIn.blockTransactions
+      .copy(txs = headTx.copy(inputs = newInput +: headTx.inputs.tail) +: brokenBlockIn.blockTransactions.txs.tail)
+    val brokenHeader = brokenBlockIn.header.copy(transactionsRoot = brokenTransactionsIn.digest)
+    val brokenTransactions = brokenTransactionsIn.copy(headerId = brokenHeader.id)
+    val brokenProofs = brokenBlockIn.aDProofs.get.copy(headerId = brokenHeader.id)
+    ErgoFullBlock(brokenHeader, brokenTransactions, Some(brokenProofs))
   }
 
   private val t8 = TestCase("switching for a better chain") { fixture =>