From 5760cf05709ab2d55be26073522df71addcfa242 Mon Sep 17 00:00:00 2001 From: catena Date: Mon, 18 Jun 2018 17:53:22 +0300 Subject: [PATCH 1/8] Update scorex version --- build.sbt | 2 +- lock.sbt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index e9ca2b1802..2de3185e81 100644 --- a/build.sbt +++ b/build.sbt @@ -14,7 +14,7 @@ resolvers ++= Seq("Sonatype Releases" at "https://oss.sonatype.org/content/repos "Typesafe maven releases" at "http://repo.typesafe.com/typesafe/maven-releases/", "Sonatype Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots/") -val scorexVersion = "d5c4180c-SNAPSHOT" +val scorexVersion = "effd499c-SNAPSHOT" libraryDependencies ++= Seq( "org.scorexfoundation" %% "scrypto" % "2.1.2", diff --git a/lock.sbt b/lock.sbt index 92f49ec0de..a87d3f29c1 100644 --- a/lock.sbt +++ b/lock.sbt @@ -49,7 +49,7 @@ dependencyOverrides in ThisBuild ++= Seq( "org.scalacheck" % "scalacheck_2.12" % "1.13.5", "org.scorexfoundation" % "avl-iodb_2.12" % "0.2.14", "org.scorexfoundation" % "iodb_2.12" % "0.3.2", - "org.scorexfoundation" % "scorex-core_2.12" % "d5c4180c-SNAPSHOT", + "org.scorexfoundation" % "scorex-core_2.12" % "effd499c-SNAPSHOT", "org.scorexfoundation" % "scrypto_2.12" % "2.1.2", "org.scorexfoundation" % "sigma-state_2.12" % "0.9.3", "org.slf4j" % "slf4j-api" % "1.8.0-beta1", @@ -61,4 +61,4 @@ dependencyOverrides in ThisBuild ++= Seq( "org.typelevel" % "macro-compat_2.12" % "1.1.1", "org.whispersystems" % "curve25519-java" % "0.5.0" ) -// LIBRARY_DEPENDENCIES_HASH 5d2ab45def58eece6d403c3b6dd781213afad31f +// LIBRARY_DEPENDENCIES_HASH 7d405ddb9d88825e46f3cb9f988354e2c253876d From f8382b6253a9a41e1297c53ad5971a720b111806 Mon Sep 17 00:00:00 2001 From: catena Date: Mon, 18 Jun 2018 18:43:17 +0300 Subject: [PATCH 2/8] Assert emission box is defined --- src/main/scala/org/ergoplatform/local/ErgoMiner.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/scala/org/ergoplatform/local/ErgoMiner.scala b/src/main/scala/org/ergoplatform/local/ErgoMiner.scala index 5f0ed34130..6b2e2312cd 100644 --- a/src/main/scala/org/ergoplatform/local/ErgoMiner.scala +++ b/src/main/scala/org/ergoplatform/local/ErgoMiner.scala @@ -199,6 +199,7 @@ object ErgoMiner extends ScorexLogging { emission: CoinsEmission): ErgoTransaction = { state.emissionBox() match { case Some(emissionBox) => + assert(state.boxById(emissionBox.id).isDefined, s"Emission box ${Algos.encode(emissionBox.id)} missed") ErgoMiner.createCoinbase(height, feeBoxes, emissionBox, minerProp, emission) case None => // TODO extract fees when emission is finished From 435c2cf40425eb4bdc378bff9e69fa87c8c7d74c Mon Sep 17 00:00:00 2001 From: catena Date: Mon, 18 Jun 2018 18:53:20 +0300 Subject: [PATCH 3/8] More checks to invalid full block application --- .../ergoplatform/nodeView/ErgoNodeViewHolderSpecification.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/scala/org/ergoplatform/nodeView/ErgoNodeViewHolderSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/ErgoNodeViewHolderSpecification.scala index e10f0ca0a2..3ed2432534 100644 --- a/src/test/scala/org/ergoplatform/nodeView/ErgoNodeViewHolderSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/ErgoNodeViewHolderSpecification.scala @@ -304,6 +304,8 @@ class ErgoNodeViewHolderSpecification extends ErgoPropertyTest with BeforeAndAft nodeViewRef ! LocallyGeneratedModifier(brokenBlock.aDProofs.get) nodeViewRef ! bestFullBlock(nodeViewConfig) expectMsg(Some(block)) + nodeViewRef ! rootHash(nodeViewConfig) + expectMsg(Algos.encode(wusAfterBlock.rootHash)) } nodeViewRef ! bestHeaderOpt(nodeViewConfig) From fe2622bd803018b00e81144b28f460fae694fc17 Mon Sep 17 00:00:00 2001 From: catena Date: Tue, 19 Jun 2018 13:19:26 +0300 Subject: [PATCH 4/8] Broken test for invalid block application --- .../ErgoNodeViewHolderSpecification.scala | 71 +++++++++++-------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/src/test/scala/org/ergoplatform/nodeView/ErgoNodeViewHolderSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/ErgoNodeViewHolderSpecification.scala index 3ed2432534..6c811afa24 100644 --- a/src/test/scala/org/ergoplatform/nodeView/ErgoNodeViewHolderSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/ErgoNodeViewHolderSpecification.scala @@ -270,47 +270,59 @@ class ErgoNodeViewHolderSpecification extends ErgoPropertyTest with BeforeAndAft val (us, bh) = createUtxoState(Some(nodeViewRef)) val genesis = validFullBlock(parentOpt = None, us, bh) val wusAfterGenesis = WrappedUtxoState(us, bh, stateConstants).applyModifier(genesis).get - - nodeViewRef ! LocallyGeneratedModifier(genesis.header) + // TODO looks like another bug is still present here, see https://github.com/ergoplatform/ergo/issues/309 if (nodeViewConfig.verifyTransactions) { - nodeViewRef ! LocallyGeneratedModifier(genesis.blockTransactions) - nodeViewRef ! LocallyGeneratedModifier(genesis.aDProofs.get) - } + nodeViewRef ! LocallyGeneratedModifier(genesis.header) + if (nodeViewConfig.verifyTransactions) { + nodeViewRef ! LocallyGeneratedModifier(genesis.blockTransactions) + nodeViewRef ! LocallyGeneratedModifier(genesis.aDProofs.get) + } - val block = validFullBlock(Some(genesis.header), wusAfterGenesis) - val wusAfterBlock = wusAfterGenesis.applyModifier(block).get + val block = validFullBlock(Some(genesis.header), wusAfterGenesis) + val wusAfterBlock = wusAfterGenesis.applyModifier(block).get - nodeViewRef ! LocallyGeneratedModifier(block.header) - if (nodeViewConfig.verifyTransactions) { - nodeViewRef ! LocallyGeneratedModifier(block.blockTransactions) - nodeViewRef ! LocallyGeneratedModifier(block.aDProofs.get) - nodeViewRef ! rootHash(nodeViewConfig) - expectMsg(Algos.encode(wusAfterBlock.rootHash)) - } + nodeViewRef ! LocallyGeneratedModifier(block.header) + if (nodeViewConfig.verifyTransactions) { + nodeViewRef ! LocallyGeneratedModifier(block.blockTransactions) + nodeViewRef ! LocallyGeneratedModifier(block.aDProofs.get) + nodeViewRef ! rootHash(nodeViewConfig) + expectMsg(Algos.encode(wusAfterBlock.rootHash)) + } - nodeViewRef ! bestHeaderOpt(nodeViewConfig) - expectMsg(Some(block.header)) + nodeViewRef ! bestHeaderOpt(nodeViewConfig) + expectMsg(Some(block.header)) - val brokenBlock = validFullBlock(Some(block.header), wusAfterBlock) - val headTx = brokenBlock.blockTransactions.txs.head - val newInput = headTx.inputs.head.copy(boxId = ADKey @@ Algos.hash("wrong input")) + val brokenBlock = generateInvalidFullBlock(block.header, wusAfterBlock) + nodeViewRef ! LocallyGeneratedModifier(brokenBlock.header) + nodeViewRef ! LocallyGeneratedModifier(brokenBlock.blockTransactions) + nodeViewRef ! LocallyGeneratedModifier(brokenBlock.aDProofs.get) - nodeViewRef ! LocallyGeneratedModifier(brokenBlock.header) + val brokenBlock2 = generateInvalidFullBlock(block.header, wusAfterBlock) + brokenBlock2.header should not be brokenBlock.header + nodeViewRef ! LocallyGeneratedModifier(brokenBlock2.header) + nodeViewRef ! LocallyGeneratedModifier(brokenBlock2.blockTransactions) + nodeViewRef ! LocallyGeneratedModifier(brokenBlock2.aDProofs.get) - val brokenTransactions = brokenBlock.blockTransactions - .copy(txs = headTx.copy(inputs = newInput +: headTx.inputs.tail) +: brokenBlock.blockTransactions.txs.tail) - if (nodeViewConfig.verifyTransactions) { - nodeViewRef ! LocallyGeneratedModifier(brokenTransactions) - nodeViewRef ! LocallyGeneratedModifier(brokenBlock.aDProofs.get) nodeViewRef ! bestFullBlock(nodeViewConfig) expectMsg(Some(block)) nodeViewRef ! rootHash(nodeViewConfig) expectMsg(Algos.encode(wusAfterBlock.rootHash)) + nodeViewRef ! bestHeaderOpt(nodeViewConfig) + expectMsg(Some(block.header)) + } + } - nodeViewRef ! bestHeaderOpt(nodeViewConfig) - //TODO Note and verify! - expectMsg(Some(brokenBlock.header)) + private def generateInvalidFullBlock(parentHeader: Header, parentState: WrappedUtxoState) = { + val brokenBlockIn = validFullBlock(Some(parentHeader), parentState) + val headTx = brokenBlockIn.blockTransactions.txs.head + val newInput = headTx.inputs.head.copy(boxId = ADKey @@ Algos.hash("wrong input")) + val brokenTransactionsIn = brokenBlockIn.blockTransactions + .copy(txs = headTx.copy(inputs = newInput +: headTx.inputs.tail) +: brokenBlockIn.blockTransactions.txs.tail) + val brokenHeader = brokenBlockIn.header.copy(transactionsRoot = brokenTransactionsIn.digest) + val brokenTransactions = brokenTransactionsIn.copy(headerId = brokenHeader.id) + val brokenProofs = brokenBlockIn.aDProofs.get.copy(headerId = brokenHeader.id) + ErgoFullBlock(brokenHeader, brokenTransactions, Some(brokenProofs)) } private val t8 = TestCase("switching for a better chain") { fixture => @@ -471,7 +483,8 @@ class ErgoNodeViewHolderSpecification extends ErgoPropertyTest with BeforeAndAft } } - val cases: List[TestCase] = List(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11) + // val cases: List[TestCase] = List(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11) + val cases: List[TestCase] = List(t7) allConfigs.foreach { c => cases.foreach { t => From 84d94e4936eeb957ba72dd40753bb2da24eba053 Mon Sep 17 00:00:00 2001 From: catena Date: Tue, 19 Jun 2018 15:08:26 +0300 Subject: [PATCH 5/8] Fix link switch on block invalidation. close #303 --- .../nodeView/history/ErgoHistoryReader.scala | 20 ++++++++++++++----- .../history/storage/FilesObjectsStore.scala | 2 ++ .../history/storage/ObjectsStore.scala | 2 ++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala index 6dd9a3923b..ea62faaefc 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala @@ -54,15 +54,22 @@ trait ErgoHistoryReader bestFullBlockIdOpt.flatMap(id => typedModifierById[Header](id)).flatMap(getFullBlock) /** - * Get ErgoPersistentModifier by it's id if it is in history + * @param id - modifier id + * @return semantically valid ErgoPersistentModifier with the given id it is in history */ override def modifierById(id: ModifierId): Option[ErgoPersistentModifier] = { - historyStorage.modifierById(id) - .ensuring(_.forall(_.id sameElements id), s"Modifier ${Algos.encode(id)} id is incorrect") - } + if (isSemanticallyValid(id) != ModifierSemanticValidity.Invalid) { + historyStorage.modifierById(id) + } else { + None + } + }.ensuring(_.forall(_.id sameElements id), s"Modifier ${Algos.encode(id)} id is incorrect") /** - * Get ErgoPersistentModifier of type T by it's id if it is in history + * + * @param id - modifier id + * @tparam T - expected Type + * @return semantically valid ErgoPersistentModifier of type T with the given id it is in history */ @SuppressWarnings(Array("IsInstanceOf")) def typedModifierById[T <: ErgoPersistentModifier](id: ModifierId): Option[T] = modifierById(id) match { @@ -70,6 +77,9 @@ trait ErgoHistoryReader case _ => None } + + override def contains(id: ModifierId): Boolean = historyStorage.contains(id) + /** * Id of best block to mine */ diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/FilesObjectsStore.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/FilesObjectsStore.scala index e490eafc45..e4bf4928ad 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/FilesObjectsStore.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/FilesObjectsStore.scala @@ -25,5 +25,7 @@ class FilesObjectsStore(dir: String) extends ObjectsStore { Files.delete(path(id)) } + override def contains(id: ModifierId): Boolean = Files.exists(path(id)) + private def path(id: ModifierId) = Paths.get(dir + "/" + Algos.encode(id)) } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/ObjectsStore.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/ObjectsStore.scala index aae0f60e78..1fa35123e7 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/ObjectsStore.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/ObjectsStore.scala @@ -13,4 +13,6 @@ trait ObjectsStore { def put(m: ErgoPersistentModifier): Try[Unit] + def contains(id: ModifierId): Boolean + } From c6a6aa26f5ae83fdb7594c32bfecea8c238659cd Mon Sep 17 00:00:00 2001 From: catena Date: Tue, 19 Jun 2018 15:33:14 +0300 Subject: [PATCH 6/8] Do not read all bytes when only check for contains --- .../nodeView/history/storage/HistoryStorage.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index 8d5fae0122..dc4f5b1d30 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -17,7 +17,7 @@ class HistoryStorage(indexStore: Store, objectsStore: ObjectsStore) extends Scor case Success(b) => Some(b) case Failure(e) => - log.warn(s"Failed to parse block ${encoder.encode(id)} from db (bytes are: ${bBytes.mkString("-")}): ", e) + log.warn(s"Failed to parse modifier ${encoder.encode(id)} from db (bytes are: ${bBytes.mkString("-")}): ", e) None } } @@ -26,7 +26,7 @@ class HistoryStorage(indexStore: Store, objectsStore: ObjectsStore) extends Scor def get(id: ModifierId): Option[Array[Byte]] = objectsStore.get(id) - def contains(id: ModifierId): Boolean = get(id).isDefined + def contains(id: ModifierId): Boolean = objectsStore.contains(id) def insert(id: ByteArrayWrapper, indexesToInsert: Seq[(ByteArrayWrapper, ByteArrayWrapper)], From 22f77636cd92a55ef409ecda275b2ced43f9b308 Mon Sep 17 00:00:00 2001 From: catena Date: Tue, 19 Jun 2018 15:38:02 +0300 Subject: [PATCH 7/8] log rotation --- src/main/resources/logback.xml | 35 +++++++++++++++---- src/main/scala/org/ergoplatform/ErgoApp.scala | 1 + 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index c9619d6857..112975002b 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -1,18 +1,41 @@ + - + + + + System.out - DEBUG + ${logback.stdout.level:-INFO} - [%thread] >> [%-5level] %logger{36} >> %d{HH:mm:ss.SSS} %msg%n + ${logback.pattern:-${default.pattern}} - - - + + + ${logback.file.level:-DEBUG} + + ergo.log + + + ergo.%d{yyyy-MM-dd}.log.gz + + + 30 + 1GB + + + ${default.pattern} + + + + + + + diff --git a/src/main/scala/org/ergoplatform/ErgoApp.scala b/src/main/scala/org/ergoplatform/ErgoApp.scala index 8ca1e70155..c550610267 100644 --- a/src/main/scala/org/ergoplatform/ErgoApp.scala +++ b/src/main/scala/org/ergoplatform/ErgoApp.scala @@ -30,6 +30,7 @@ class ErgoApp(args: Seq[String]) extends Application { implicit val ec: ExecutionContextExecutor = actorSystem.dispatcher lazy val ergoSettings: ErgoSettings = ErgoSettings.read(args.headOption) + lazy val emission = new CoinsEmission(ergoSettings.chainSettings.monetary) override implicit lazy val settings: ScorexSettings = ergoSettings.scorexSettings From 825815f55c2c2e616720e2fbcbbdebbb4b1c202d Mon Sep 17 00:00:00 2001 From: catena Date: Tue, 19 Jun 2018 17:14:01 +0300 Subject: [PATCH 8/8] ErgoNodeViewHolderSpecification cleanup --- .../nodeView/ErgoNodeViewHolderSpecification.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/scala/org/ergoplatform/nodeView/ErgoNodeViewHolderSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/ErgoNodeViewHolderSpecification.scala index 6c811afa24..0e5bd1c922 100644 --- a/src/test/scala/org/ergoplatform/nodeView/ErgoNodeViewHolderSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/ErgoNodeViewHolderSpecification.scala @@ -483,8 +483,7 @@ class ErgoNodeViewHolderSpecification extends ErgoPropertyTest with BeforeAndAft } } - // val cases: List[TestCase] = List(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11) - val cases: List[TestCase] = List(t7) + val cases: List[TestCase] = List(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11) allConfigs.foreach { c => cases.foreach { t =>