From ff3112d4d69a4fd0d1c26ba868a302a0a3555bf8 Mon Sep 17 00:00:00 2001 From: capdev Date: Wed, 18 Sep 2019 14:24:38 +0500 Subject: [PATCH 01/39] draft --- .../history/HistoryApplicatorTest.scala | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala new file mode 100644 index 0000000000..a3245c5fcc --- /dev/null +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -0,0 +1,137 @@ +package encry.modifiers.history + +import java.io.File + +import akka.actor.{ActorRef, ActorSystem, Props} +import akka.testkit.{ImplicitSender, TestKit, TestProbe} +import akka.util.Timeout +import encry.EncryApp.settings +import encry.modifiers.InstanceFactory +import encry.network.DeliveryManager.FullBlockChainIsSynced +import encry.network.ModifiersToNetworkUtils +import encry.network.NodeViewSynchronizer.ReceivableMessages.UpdatedHistory +import encry.settings.{EncryAppSettings, Settings} +import encry.storage.VersionalStorage +import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} +import encry.storage.iodb.versionalIODB.IODBWrapper +import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} +import encry.utils.{FileHelper, TestHelper} +import encry.view.actors +import encry.view.actors.{HistoryApplicator, StateApplicator} +import encry.view.actors.HistoryApplicator.ModifierToHistoryAppending +import encry.view.actors.NodeViewHolder.ReceivableMessages.{LocallyGeneratedBlock, ModifierFromRemote} +import encry.view.actors.StateApplicator.NotificationAboutSuccessfullyAppliedModifier +import encry.view.history.History +import encry.view.state.{BoxHolder, UtxoState} +import encry.view.wallet.EncryWallet +import io.iohk.iodb.LSMStore +import org.encryfoundation.common.modifiers.history.{Block, Payload, PayloadProtoSerializer} +import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import org.encryfoundation.common.utils.constants.TestNetConstants +import org.iq80.leveldb.Options +import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} + +import scala.concurrent.duration._ +class HistoryApplicatorTest extends TestKit(ActorSystem()) + with WordSpecLike + with ImplicitSender + with InstanceFactory + with Matchers + with BeforeAndAfterAll + with Settings { + + override def afterAll: Unit = { + TestKit.shutdownActorSystem(system) + } + + implicit val timeout: FiniteDuration = 5 seconds + + def utxoFromBoxHolder(bh: BoxHolder, dir: File, settings: EncryAppSettings): UtxoState = { + val storage = settings.storage.state match { + case VersionalStorage.IODB => + IODBWrapper(new LSMStore(dir, keepVersions = TestNetConstants.DefaultKeepVersions)) + case VersionalStorage.LevelDB => + val levelDBInit = LevelDbFactory.factory.open(dir, new Options) + VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settings.levelDB, keySize = 32)) + } + storage.insert( + StorageVersion @@ Array.fill(32)(0: Byte), + bh.boxes.values.map(bx => (StorageKey !@@ bx.id, StorageValue @@ bx.bytes)).toList + ) + new UtxoState(storage, settings.constants) + } + + "HistoryApplicator" should { + "add Modifier" in { + + val dir = FileHelper.getRandomTempDir + val history: History = generateDummyHistory(settings) + val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) + + val bxs = TestHelper.genAssetBoxes + val bh = BoxHolder(bxs) + val state = utxoFromBoxHolder(bh, FileHelper.getRandomTempDir, settings) + + val nodeViewHolder = TestProbe() + val influx = TestProbe() + + // val historyBlocks = (0 until 10).foldLeft(history) { + // case (prevHistory, _) => + // val block: Block = generateNextBlock(prevHistory) + // prevHistory.append(block.header).get._1.append(block.payload).get._1.reportModifierIsValid(block) + // } + + val historyBlocks = (0 until 10).foldLeft(history, Seq.empty[Block]) { + case ((prevHistory, blocks), _) => + val block: Block = generateNextBlock(prevHistory) + prevHistory.append(block.header) + prevHistory.append(block.payload) + (prevHistory.reportModifierIsValid(block), blocks :+ block) + } + val payload = Payload(ModifierId @@ scorex.utils.Random.randomBytes(), Seq(coinbaseTransaction)) + + val block: Block = generateNextBlock(history) + + val modifiers: Map[ModifierId, Array[Byte]] = historyBlocks._2.map { b => + b.payload.id -> PayloadProtoSerializer.toProto(b.payload).toByteArray + }.toMap// + (payload.id -> PayloadProtoSerializer.toProto(payload).toByteArray) + + val historyApplicator: ActorRef = system.actorOf( + Props(new HistoryApplicator(history, state, wallet, settings, nodeViewHolder.ref, Some(influx.ref)) { + override def receive: Receive = { + val r = super.receive + println(s"currentNumberOfAppliedModifiers: $currentNumberOfAppliedModifiers") + println(s"modifiersQueue: $modifiersQueue") + r + }}) + //HistoryApplicator.props(history, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) + // .withDispatcher("history-applicator-dispatcher") + , "historyApplicator") + +// import akka.actor.ActorDSL._ +// val captureEventsActor = actor { +// new Act { +// become { +// case FullBlockChainIsSynced() => println(s"fullBlockChainIsSynced") +// } +// } +// } + + val logProbe = TestProbe() + system.eventStream.subscribe(logProbe.ref, classOf[FullBlockChainIsSynced]) + + // val probe = TestProbe() +// probe.watch(historyApplicator) + + val modifier = ModifiersToNetworkUtils.fromProto(Payload.modifierTypeId, PayloadProtoSerializer.toProto(block.payload).toByteArray) + +// historyApplicator ! ModifierFromRemote(modifier.get) + historyApplicator ! LocallyGeneratedBlock(block) + + //Thread.sleep(100000) + logProbe.expectMsg(FullBlockChainIsSynced()) + + } + } + +} From be4aa2cf37e4357ef5caf2f85e7db65cabd429e6 Mon Sep 17 00:00:00 2001 From: capdev Date: Wed, 18 Sep 2019 17:57:29 +0500 Subject: [PATCH 02/39] add chain synced test --- .../history/HistoryApplicatorTest.scala | 117 +++++++++++------- 1 file changed, 74 insertions(+), 43 deletions(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index a3245c5fcc..1ddfc17c14 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -30,8 +30,10 @@ import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.encryfoundation.common.utils.constants.TestNetConstants import org.iq80.leveldb.Options import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} +import akka.testkit.{TestKit, TestActorRef} import scala.concurrent.duration._ + class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike with ImplicitSender @@ -44,9 +46,9 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) TestKit.shutdownActorSystem(system) } - implicit val timeout: FiniteDuration = 5 seconds + val timeout: FiniteDuration = 5 seconds - def utxoFromBoxHolder(bh: BoxHolder, dir: File, settings: EncryAppSettings): UtxoState = { + def utxoFromBoxHolder(boxHolder: BoxHolder, dir: File, settings: EncryAppSettings): UtxoState = { val storage = settings.storage.state match { case VersionalStorage.IODB => IODBWrapper(new LSMStore(dir, keepVersions = TestNetConstants.DefaultKeepVersions)) @@ -56,34 +58,66 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) } storage.insert( StorageVersion @@ Array.fill(32)(0: Byte), - bh.boxes.values.map(bx => (StorageKey !@@ bx.id, StorageValue @@ bx.bytes)).toList + boxHolder.boxes.values.map(bx => (StorageKey !@@ bx.id, StorageValue @@ bx.bytes)).toList ) new UtxoState(storage, settings.constants) } - "HistoryApplicator" should { - "add Modifier" in { + "HistoryApplicator add locall generated block" should { + "chain synced" in { val dir = FileHelper.getRandomTempDir val history: History = generateDummyHistory(settings) val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) val bxs = TestHelper.genAssetBoxes - val bh = BoxHolder(bxs) - val state = utxoFromBoxHolder(bh, FileHelper.getRandomTempDir, settings) + val boxHolder = BoxHolder(bxs) + val state = utxoFromBoxHolder(boxHolder, FileHelper.getRandomTempDir, settings) val nodeViewHolder = TestProbe() val influx = TestProbe() - // val historyBlocks = (0 until 10).foldLeft(history) { - // case (prevHistory, _) => - // val block: Block = generateNextBlock(prevHistory) - // prevHistory.append(block.header).get._1.append(block.payload).get._1.reportModifierIsValid(block) - // } + val historyBlocks = (0 until 10).foldLeft(history, Seq.empty[Block]) { + case ((prevHistory, blocks), _) => + val block: Block = generateNextBlock(prevHistory) + prevHistory.append(block.header) + prevHistory.append(block.payload) + (prevHistory.reportModifierIsValid(block), blocks :+ block) + } + + val block: Block = generateNextBlock(history) + + val historyApplicator: TestActorRef[HistoryApplicator] = + TestActorRef[HistoryApplicator]( + HistoryApplicator.props(history, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) + ) + + system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) + + historyApplicator ! LocallyGeneratedBlock(block) + + expectMsg(timeout, FullBlockChainIsSynced()) + } + } + + "HistoryApplicator add locall generated block" should { + "check queues" in { + + val dir = FileHelper.getRandomTempDir + val history: History = generateDummyHistory(settings) + val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) + + val bxs = TestHelper.genAssetBoxes + val boxHolder = BoxHolder(bxs) + val state = utxoFromBoxHolder(boxHolder, FileHelper.getRandomTempDir, settings) + + val nodeViewHolder = TestProbe() + val influx = TestProbe() val historyBlocks = (0 until 10).foldLeft(history, Seq.empty[Block]) { case ((prevHistory, blocks), _) => val block: Block = generateNextBlock(prevHistory) + //prevHistory.append(block.header).get._1.append(block.payload).get._1.reportModifierIsValid(block) prevHistory.append(block.header) prevHistory.append(block.payload) (prevHistory.reportModifierIsValid(block), blocks :+ block) @@ -94,42 +128,39 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val modifiers: Map[ModifierId, Array[Byte]] = historyBlocks._2.map { b => b.payload.id -> PayloadProtoSerializer.toProto(b.payload).toByteArray - }.toMap// + (payload.id -> PayloadProtoSerializer.toProto(payload).toByteArray) - - val historyApplicator: ActorRef = system.actorOf( - Props(new HistoryApplicator(history, state, wallet, settings, nodeViewHolder.ref, Some(influx.ref)) { - override def receive: Receive = { - val r = super.receive - println(s"currentNumberOfAppliedModifiers: $currentNumberOfAppliedModifiers") - println(s"modifiersQueue: $modifiersQueue") - r - }}) - //HistoryApplicator.props(history, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) - // .withDispatcher("history-applicator-dispatcher") - , "historyApplicator") - -// import akka.actor.ActorDSL._ -// val captureEventsActor = actor { -// new Act { -// become { -// case FullBlockChainIsSynced() => println(s"fullBlockChainIsSynced") -// } -// } -// } - - val logProbe = TestProbe() - system.eventStream.subscribe(logProbe.ref, classOf[FullBlockChainIsSynced]) - - // val probe = TestProbe() -// probe.watch(historyApplicator) + }.toMap // + (payload.id -> PayloadProtoSerializer.toProto(payload).toByteArray) + + val historyApplicator: TestActorRef[HistoryApplicator] = + TestActorRef[HistoryApplicator]( + HistoryApplicator.props(history, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) + ) + + // val historyApplicator: ActorRef = system.actorOf( + // Props(new HistoryApplicator(history, state, wallet, settings, nodeViewHolder.ref, Some(influx.ref)) { + // + // override def getModifierForApplying(): Unit = { + // println(s"-------------------") + // println(s"locallyGeneratedModifiers: ${locallyGeneratedModifiers.size}") + // //locallyGeneratedModifiers.size shouldBe 2 + // println(s"getModifierForApplying()") + // super.getModifierForApplying() + // println(s"currentNumberOfAppliedModifiers: $currentNumberOfAppliedModifiers") + // //currentNumberOfAppliedModifiers shouldBe 1 + // } + // }) + // ) + + system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) val modifier = ModifiersToNetworkUtils.fromProto(Payload.modifierTypeId, PayloadProtoSerializer.toProto(block.payload).toByteArray) -// historyApplicator ! ModifierFromRemote(modifier.get) + //historyApplicator ! ModifierFromRemote(modifier.get) historyApplicator ! LocallyGeneratedBlock(block) - //Thread.sleep(100000) - logProbe.expectMsg(FullBlockChainIsSynced()) + expectMsgType[ModifierToHistoryAppending] + + historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 + historyApplicator.underlyingActor.locallyGeneratedModifiers.size shouldBe 1 } } From 68749411fed701b3bc0f126f44c12dfb4b625257 Mon Sep 17 00:00:00 2001 From: capdev Date: Thu, 19 Sep 2019 12:23:17 +0500 Subject: [PATCH 03/39] add check queues --- .../encry/view/actors/HistoryApplicator.scala | 7 +- .../history/HistoryApplicatorTest.scala | 141 ++++++++---------- 2 files changed, 67 insertions(+), 81 deletions(-) diff --git a/src/main/scala/encry/view/actors/HistoryApplicator.scala b/src/main/scala/encry/view/actors/HistoryApplicator.scala index c1950e31cd..a967774d5a 100644 --- a/src/main/scala/encry/view/actors/HistoryApplicator.scala +++ b/src/main/scala/encry/view/actors/HistoryApplicator.scala @@ -157,10 +157,6 @@ class HistoryApplicator(history: History, context.system.eventStream.publish(DownloadRequest(tid, id)) } - type ModifierIdAsKey = scala.collection.mutable.WrappedArray.ofByte - - def toKey(id: ModifierId): ModifierIdAsKey = new mutable.WrappedArray.ofByte(id) - } object HistoryApplicator { @@ -183,6 +179,9 @@ object HistoryApplicator { case otherwise => 4 }) + type ModifierIdAsKey = scala.collection.mutable.WrappedArray.ofByte + def toKey(id: ModifierId): ModifierIdAsKey = new mutable.WrappedArray.ofByte(id) + def props(history: History, setting: EncryAppSettings, state: UtxoState, diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 1ddfc17c14..90897bb84d 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -9,28 +9,29 @@ import encry.EncryApp.settings import encry.modifiers.InstanceFactory import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.ModifiersToNetworkUtils -import encry.network.NodeViewSynchronizer.ReceivableMessages.UpdatedHistory +import encry.network.NodeViewSynchronizer.ReceivableMessages.{SemanticallySuccessfulModifier, UpdatedHistory} import encry.settings.{EncryAppSettings, Settings} import encry.storage.VersionalStorage import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} import encry.storage.iodb.versionalIODB.IODBWrapper import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} import encry.utils.{FileHelper, TestHelper} -import encry.view.actors +import encry.view.{ModifiersCache, actors} import encry.view.actors.{HistoryApplicator, StateApplicator} -import encry.view.actors.HistoryApplicator.ModifierToHistoryAppending +import encry.view.actors.HistoryApplicator.{ModifierToHistoryAppending, StartModifiersApplicationOnStateApplicator} import encry.view.actors.NodeViewHolder.ReceivableMessages.{LocallyGeneratedBlock, ModifierFromRemote} import encry.view.actors.StateApplicator.NotificationAboutSuccessfullyAppliedModifier import encry.view.history.History import encry.view.state.{BoxHolder, UtxoState} import encry.view.wallet.EncryWallet import io.iohk.iodb.LSMStore -import org.encryfoundation.common.modifiers.history.{Block, Payload, PayloadProtoSerializer} +import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload, PayloadProtoSerializer} import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.encryfoundation.common.utils.constants.TestNetConstants import org.iq80.leveldb.Options import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} -import akka.testkit.{TestKit, TestActorRef} +import akka.testkit.{TestActorRef, TestKit} +import org.encryfoundation.common.modifiers.PersistentModifier import scala.concurrent.duration._ @@ -63,48 +64,52 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) new UtxoState(storage, settings.constants) } - "HistoryApplicator add locall generated block" should { - "chain synced" in { - - val dir = FileHelper.getRandomTempDir - val history: History = generateDummyHistory(settings) - val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) - - val bxs = TestHelper.genAssetBoxes - val boxHolder = BoxHolder(bxs) - val state = utxoFromBoxHolder(boxHolder, FileHelper.getRandomTempDir, settings) - - val nodeViewHolder = TestProbe() - val influx = TestProbe() - - val historyBlocks = (0 until 10).foldLeft(history, Seq.empty[Block]) { - case ((prevHistory, blocks), _) => - val block: Block = generateNextBlock(prevHistory) - prevHistory.append(block.header) - prevHistory.append(block.payload) - (prevHistory.reportModifierIsValid(block), blocks :+ block) - } - - val block: Block = generateNextBlock(history) - - val historyApplicator: TestActorRef[HistoryApplicator] = - TestActorRef[HistoryApplicator]( - HistoryApplicator.props(history, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) - ) - - system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) - - historyApplicator ! LocallyGeneratedBlock(block) - - expectMsg(timeout, FullBlockChainIsSynced()) - } + def blockToModifier(block: Block): (PersistentModifier, PersistentModifier) = { + (ModifiersToNetworkUtils.fromProto(Header.modifierTypeId, HeaderProtoSerializer.toProto(block.header).toByteArray).get, + ModifiersToNetworkUtils.fromProto(Payload.modifierTypeId, PayloadProtoSerializer.toProto(block.payload).toByteArray).get) } - "HistoryApplicator add locall generated block" should { +// "HistoryApplicator add locall generated block" should { +// "chain synced" in { +// +// val dir = FileHelper.getRandomTempDir +// val history: History = generateDummyHistory(settings) +// val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) +// +// val bxs = TestHelper.genAssetBoxes +// val boxHolder = BoxHolder(bxs) +// val state = utxoFromBoxHolder(boxHolder, FileHelper.getRandomTempDir, settings) +// +// val nodeViewHolder = TestProbe() +// val influx = TestProbe() +// +// val historyBlocks = (0 until 10).foldLeft(history, Seq.empty[Block]) { +// case ((prevHistory, blocks), _) => +// val block: Block = generateNextBlock(prevHistory) +// prevHistory.append(block.header) +// prevHistory.append(block.payload) +// (prevHistory.reportModifierIsValid(block), blocks :+ block) +// } +// +// val block: Block = generateNextBlock(history) +// +// val historyApplicator: TestActorRef[HistoryApplicator] = +// TestActorRef[HistoryApplicator]( +// HistoryApplicator.props(history, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) +// ) +// +// system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) +// +// historyApplicator ! LocallyGeneratedBlock(block) +// +// expectMsg(timeout, FullBlockChainIsSynced()) +// } +// } + + "HistoryApplicator" should { "check queues" in { val dir = FileHelper.getRandomTempDir - val history: History = generateDummyHistory(settings) val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) val bxs = TestHelper.genAssetBoxes @@ -114,53 +119,35 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val nodeViewHolder = TestProbe() val influx = TestProbe() - val historyBlocks = (0 until 10).foldLeft(history, Seq.empty[Block]) { - case ((prevHistory, blocks), _) => - val block: Block = generateNextBlock(prevHistory) - //prevHistory.append(block.header).get._1.append(block.payload).get._1.reportModifierIsValid(block) - prevHistory.append(block.header) - prevHistory.append(block.payload) - (prevHistory.reportModifierIsValid(block), blocks :+ block) - } - val payload = Payload(ModifierId @@ scorex.utils.Random.randomBytes(), Seq(coinbaseTransaction)) + val initialHistory: History = generateDummyHistory(settings) - val block: Block = generateNextBlock(history) + val block1: Block = generateNextBlock(initialHistory) - val modifiers: Map[ModifierId, Array[Byte]] = historyBlocks._2.map { b => - b.payload.id -> PayloadProtoSerializer.toProto(b.payload).toByteArray - }.toMap // + (payload.id -> PayloadProtoSerializer.toProto(payload).toByteArray) + val (headerMod1, payloadMod1) = blockToModifier(block1) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( - HistoryApplicator.props(history, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) + HistoryApplicator.props(initialHistory, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) ) - // val historyApplicator: ActorRef = system.actorOf( - // Props(new HistoryApplicator(history, state, wallet, settings, nodeViewHolder.ref, Some(influx.ref)) { - // - // override def getModifierForApplying(): Unit = { - // println(s"-------------------") - // println(s"locallyGeneratedModifiers: ${locallyGeneratedModifiers.size}") - // //locallyGeneratedModifiers.size shouldBe 2 - // println(s"getModifierForApplying()") - // super.getModifierForApplying() - // println(s"currentNumberOfAppliedModifiers: $currentNumberOfAppliedModifiers") - // //currentNumberOfAppliedModifiers shouldBe 1 - // } - // }) - // ) + historyApplicator ! ModifierFromRemote(headerMod1) - system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) + ModifiersCache.put(HistoryApplicator.toKey(headerMod1.id), headerMod1, initialHistory) + ModifiersCache.put(HistoryApplicator.toKey(payloadMod1.id), payloadMod1, initialHistory) - val modifier = ModifiersToNetworkUtils.fromProto(Payload.modifierTypeId, PayloadProtoSerializer.toProto(block.payload).toByteArray) + println(s"modifiersQueue.size: ${historyApplicator.underlyingActor.modifiersQueue.size}") + println(s"currentNumberOfAppliedModifiers: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") + historyApplicator.underlyingActor.modifiersQueue.size shouldBe 1 + historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 1 - //historyApplicator ! ModifierFromRemote(modifier.get) - historyApplicator ! LocallyGeneratedBlock(block) + historyApplicator ! StateApplicator.RequestNextModifier - expectMsgType[ModifierToHistoryAppending] + println(s"modifiersQueue.size: ${historyApplicator.underlyingActor.modifiersQueue.size}") + println(s"currentNumberOfAppliedModifiers: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") + historyApplicator.underlyingActor.modifiersQueue.size shouldBe 0 + historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 1 - historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 - historyApplicator.underlyingActor.locallyGeneratedModifiers.size shouldBe 1 +// expectMsgType[StartModifiersApplicationOnStateApplicator](timeout) } } From 965a61cb912140ad54d7a5802d172d8671609843 Mon Sep 17 00:00:00 2001 From: capdev Date: Thu, 19 Sep 2019 14:17:19 +0500 Subject: [PATCH 04/39] add checks for queues and history --- .../history/HistoryApplicatorTest.scala | 51 ++++++++++--------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 90897bb84d..217ec1571e 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -32,6 +32,7 @@ import org.iq80.leveldb.Options import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} import akka.testkit.{TestActorRef, TestKit} import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.utils.Algos import scala.concurrent.duration._ @@ -47,7 +48,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) TestKit.shutdownActorSystem(system) } - val timeout: FiniteDuration = 5 seconds + val timeout: FiniteDuration = 10 seconds def utxoFromBoxHolder(boxHolder: BoxHolder, dir: File, settings: EncryAppSettings): UtxoState = { val storage = settings.storage.state match { @@ -64,9 +65,11 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) new UtxoState(storage, settings.constants) } - def blockToModifier(block: Block): (PersistentModifier, PersistentModifier) = { - (ModifiersToNetworkUtils.fromProto(Header.modifierTypeId, HeaderProtoSerializer.toProto(block.header).toByteArray).get, - ModifiersToNetworkUtils.fromProto(Payload.modifierTypeId, PayloadProtoSerializer.toProto(block.payload).toByteArray).get) + def blockToModifiers(block: Block): Seq[PersistentModifier] = { + Seq( + ModifiersToNetworkUtils.fromProto(Header.modifierTypeId, HeaderProtoSerializer.toProto(block.header).toByteArray).get, + ModifiersToNetworkUtils.fromProto(Payload.modifierTypeId, PayloadProtoSerializer.toProto(block.payload).toByteArray).get + ) } // "HistoryApplicator add locall generated block" should { @@ -109,46 +112,44 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) "HistoryApplicator" should { "check queues" in { + val nodeViewHolder = TestProbe() + val influx = TestProbe() + val dir = FileHelper.getRandomTempDir val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) - val bxs = TestHelper.genAssetBoxes val boxHolder = BoxHolder(bxs) val state = utxoFromBoxHolder(boxHolder, FileHelper.getRandomTempDir, settings) - - val nodeViewHolder = TestProbe() - val influx = TestProbe() - val initialHistory: History = generateDummyHistory(settings) - val block1: Block = generateNextBlock(initialHistory) - - val (headerMod1, payloadMod1) = blockToModifier(block1) + val modifiers = blockToModifiers(block1) + val modifierIds = modifiers.map(_.id) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( HistoryApplicator.props(initialHistory, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) ) - historyApplicator ! ModifierFromRemote(headerMod1) - - ModifiersCache.put(HistoryApplicator.toKey(headerMod1.id), headerMod1, initialHistory) - ModifiersCache.put(HistoryApplicator.toKey(payloadMod1.id), payloadMod1, initialHistory) + modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) - println(s"modifiersQueue.size: ${historyApplicator.underlyingActor.modifiersQueue.size}") - println(s"currentNumberOfAppliedModifiers: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") - historyApplicator.underlyingActor.modifiersQueue.size shouldBe 1 - historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 1 + val modifiersQueue = historyApplicator.underlyingActor.modifiersQueue + val currentNumberOfAppliedModifiers = historyApplicator.underlyingActor.currentNumberOfAppliedModifiers historyApplicator ! StateApplicator.RequestNextModifier - println(s"modifiersQueue.size: ${historyApplicator.underlyingActor.modifiersQueue.size}") - println(s"currentNumberOfAppliedModifiers: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") - historyApplicator.underlyingActor.modifiersQueue.size shouldBe 0 - historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 1 + val modifiersQueue1 = historyApplicator.underlyingActor.modifiersQueue + val currentNumberOfAppliedModifiers1 = historyApplicator.underlyingActor.currentNumberOfAppliedModifiers + val history = historyApplicator.underlyingActor.history + + expectMsgType[StartModifiersApplicationOnStateApplicator](timeout) -// expectMsgType[StartModifiersApplicationOnStateApplicator](timeout) + modifiersQueue.size shouldBe 2 + currentNumberOfAppliedModifiers shouldBe 2 + modifiersQueue1.size shouldBe 0 + currentNumberOfAppliedModifiers1 shouldBe 2 + assert(modifierIds.forall(history.isModifierDefined)) + assert(modifierIds.map(Algos.encode) == modifiersQueue.map(_._1)) } } From f45b0a320d42963c79a8fe666434cb62ef5a85e4 Mon Sep 17 00:00:00 2001 From: capdev Date: Fri, 20 Sep 2019 11:43:42 +0500 Subject: [PATCH 05/39] move bench.Utils to EncryCore add check queue for rollback limit test add to HistoryApplicator debug prints --- .../test/scala/benches/HistoryBenches.scala | 2 +- .../SerializedAssetTransactionBenchmark.scala | 2 +- .../benches/SerializedDataTxBenchmark.scala | 2 +- .../SerializedMonetaryTxBenchmark.scala | 2 +- .../src/test/scala/benches/StateBenches.scala | 2 +- .../scala/benches/StateRollbackBench.scala | 2 +- .../benches/VersionalLevelDBBanches.scala | 4 +- .../main/scala/encry/utils/ChainUtils.scala | 6 +- .../scala/encry/utils/FileHelper.scala | 0 .../encry/view/actors/HistoryApplicator.scala | 42 ++-- .../history/HistoryApplicatorTest.scala | 232 ++++++++++++------ 11 files changed, 189 insertions(+), 107 deletions(-) rename benchmarks/src/test/scala/benches/Utils.scala => src/main/scala/encry/utils/ChainUtils.scala (99%) rename src/{test => main}/scala/encry/utils/FileHelper.scala (100%) diff --git a/benchmarks/src/test/scala/benches/HistoryBenches.scala b/benchmarks/src/test/scala/benches/HistoryBenches.scala index 99e5e52ddf..02e6e23462 100644 --- a/benchmarks/src/test/scala/benches/HistoryBenches.scala +++ b/benchmarks/src/test/scala/benches/HistoryBenches.scala @@ -4,7 +4,7 @@ import java.io.File import java.util.concurrent.TimeUnit import benches.HistoryBenches.HistoryBenchState -import benches.Utils.{generateHistory, generateNextBlockValidForHistory, getRandomTempDir} +import encry.utils.ChainUtils.{generateHistory, generateNextBlockValidForHistory, getRandomTempDir} import encry.EncryApp import encry.settings.EncryAppSettings import encry.view.history.History diff --git a/benchmarks/src/test/scala/benches/SerializedAssetTransactionBenchmark.scala b/benchmarks/src/test/scala/benches/SerializedAssetTransactionBenchmark.scala index df3e1047db..31f51f179d 100644 --- a/benchmarks/src/test/scala/benches/SerializedAssetTransactionBenchmark.scala +++ b/benchmarks/src/test/scala/benches/SerializedAssetTransactionBenchmark.scala @@ -3,7 +3,7 @@ package benches import java.util.concurrent.TimeUnit import benches.SerializedAssetTransactionBenchmark.SerializedAssetBenchState -import benches.Utils._ +import encry.utils.ChainUtils._ import encryBenchmark.{BenchSettings, Settings} import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionSerializer} import org.encryfoundation.common.modifiers.state.box.AssetBox diff --git a/benchmarks/src/test/scala/benches/SerializedDataTxBenchmark.scala b/benchmarks/src/test/scala/benches/SerializedDataTxBenchmark.scala index 8ab3ef0621..425700c263 100644 --- a/benchmarks/src/test/scala/benches/SerializedDataTxBenchmark.scala +++ b/benchmarks/src/test/scala/benches/SerializedDataTxBenchmark.scala @@ -3,7 +3,7 @@ package benches import java.util.concurrent.TimeUnit import benches.SerializedDataTxBenchmark.SerializedDataBenchState -import benches.Utils._ +import encry.utils.ChainUtils._ import encryBenchmark.{BenchSettings, Settings} import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionSerializer} import org.encryfoundation.common.modifiers.state.box.AssetBox diff --git a/benchmarks/src/test/scala/benches/SerializedMonetaryTxBenchmark.scala b/benchmarks/src/test/scala/benches/SerializedMonetaryTxBenchmark.scala index 404c5a9f8c..2525bd4507 100644 --- a/benchmarks/src/test/scala/benches/SerializedMonetaryTxBenchmark.scala +++ b/benchmarks/src/test/scala/benches/SerializedMonetaryTxBenchmark.scala @@ -3,7 +3,7 @@ package benches import java.util.concurrent.TimeUnit import benches.SerializedMonetaryTxBenchmark.SerializedMonetaryBenchState -import benches.Utils._ +import encry.utils.ChainUtils._ import encryBenchmark.{BenchSettings, Settings} import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionSerializer} import org.encryfoundation.common.modifiers.state.box.AssetBox diff --git a/benchmarks/src/test/scala/benches/StateBenches.scala b/benchmarks/src/test/scala/benches/StateBenches.scala index 9aa45760b2..8ad8418676 100644 --- a/benchmarks/src/test/scala/benches/StateBenches.scala +++ b/benchmarks/src/test/scala/benches/StateBenches.scala @@ -5,7 +5,7 @@ import java.util.concurrent.TimeUnit import benches.StateBenches.StateBenchState import org.openjdk.jmh.annotations._ -import benches.Utils._ +import encry.utils.ChainUtils._ import encry.EncryApp import encry.settings.EncryAppSettings import encry.storage.VersionalStorage diff --git a/benchmarks/src/test/scala/benches/StateRollbackBench.scala b/benchmarks/src/test/scala/benches/StateRollbackBench.scala index 812fc160f9..e0989c2af8 100644 --- a/benchmarks/src/test/scala/benches/StateRollbackBench.scala +++ b/benchmarks/src/test/scala/benches/StateRollbackBench.scala @@ -4,7 +4,7 @@ import java.io.File import java.util.concurrent.TimeUnit import benches.StateRollbackBench.StateRollbackState -import benches.Utils._ +import encry.utils.ChainUtils._ import encry.storage.VersionalStorage import encry.utils.CoreTaggedTypes.VersionTag import encry.view.state.{BoxHolder, UtxoState} diff --git a/benchmarks/src/test/scala/benches/VersionalLevelDBBanches.scala b/benchmarks/src/test/scala/benches/VersionalLevelDBBanches.scala index e8f8bebdec..896957536f 100644 --- a/benchmarks/src/test/scala/benches/VersionalLevelDBBanches.scala +++ b/benchmarks/src/test/scala/benches/VersionalLevelDBBanches.scala @@ -4,7 +4,7 @@ import java.util.concurrent.TimeUnit import benches.VersionalLevelDBBanches.VersionalLevelDBState import encry.settings.LevelDBSettings import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VersionalLevelDBCompanion} -import encry.utils.FileHelper +import encry.utils.{ChainUtils, FileHelper} import org.iq80.leveldb.Options import org.openjdk.jmh.annotations.{Benchmark, Mode, Scope, State} import org.openjdk.jmh.infra.Blackhole @@ -55,7 +55,7 @@ object VersionalLevelDBBanches { //val elems1k = Utils.generateRandomLevelDbElemsWithoutDeletions(1000, 100) //val elems5k = Utils.generateRandomLevelDbElemsWithoutDeletions(5000, 100) - val elems10k = Utils.generateRandomLevelDbElemsWithoutDeletions(10000, 100) + val elems10k = ChainUtils.generateRandomLevelDbElemsWithoutDeletions(10000, 100) //val elems30k = Utils.generateRandomLevelDbElemsWithoutDeletions(30000, 100) } } \ No newline at end of file diff --git a/benchmarks/src/test/scala/benches/Utils.scala b/src/main/scala/encry/utils/ChainUtils.scala similarity index 99% rename from benchmarks/src/test/scala/benches/Utils.scala rename to src/main/scala/encry/utils/ChainUtils.scala index 0d8e54d3be..235bb8c024 100644 --- a/benchmarks/src/test/scala/benches/Utils.scala +++ b/src/main/scala/encry/utils/ChainUtils.scala @@ -1,4 +1,4 @@ -package benches +package encry.utils import java.io.File @@ -11,7 +11,6 @@ import encry.storage.VersionalStorage.{StorageKey, StorageType, StorageValue, St import encry.storage.iodb.versionalIODB.IODBWrapper import encry.storage.levelDb.versionalLevelDB.VersionalLevelDBCompanion.{LevelDBVersion, VersionalLevelDbKey, VersionalLevelDbValue} import encry.storage.levelDb.versionalLevelDB._ -import encry.utils.{FileHelper, Mnemonic, NetworkTimeProvider} import encry.view.history.History import encry.view.history.storage.HistoryStorage import encry.view.state.{BoxHolder, UtxoState} @@ -34,7 +33,7 @@ import scorex.utils.Random import scala.collection.immutable import scala.util.{Random => R} -object Utils extends Settings with StrictLogging { +object ChainUtils extends Settings with StrictLogging { val mnemonicKey: String = "index another island accuse valid aerobic little absurd bunker keep insect scissors" val privKey: PrivateKey25519 = createPrivKey(Some(mnemonicKey)) @@ -399,7 +398,6 @@ object Utils extends Settings with StrictLogging { } def generateHistory(settings: EncryAppSettings, file: File): History = { - val indexStore: LSMStore = new LSMStore(FileHelper.getRandomTempDir, keepVersions = 0) val objectsStore: LSMStore = new LSMStore(FileHelper.getRandomTempDir, keepVersions = 0) val levelDBInit = LevelDbFactory.factory.open(FileHelper.getRandomTempDir, new Options) diff --git a/src/test/scala/encry/utils/FileHelper.scala b/src/main/scala/encry/utils/FileHelper.scala similarity index 100% rename from src/test/scala/encry/utils/FileHelper.scala rename to src/main/scala/encry/utils/FileHelper.scala diff --git a/src/main/scala/encry/view/actors/HistoryApplicator.scala b/src/main/scala/encry/view/actors/HistoryApplicator.scala index a967774d5a..37068c64e5 100644 --- a/src/main/scala/encry/view/actors/HistoryApplicator.scala +++ b/src/main/scala/encry/view/actors/HistoryApplicator.scala @@ -28,7 +28,7 @@ import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.immutable.Queue import scala.collection.mutable -class HistoryApplicator(history: History, +class HistoryApplicator(val history: History, state: UtxoState, wallet: EncryWallet, setting: EncryAppSettings, @@ -49,10 +49,12 @@ class HistoryApplicator(history: History, override def receive: Receive = { case ModifierFromRemote(mod) if !history.isModifierDefined(mod.id) && !ModifiersCache.contains(toKey(mod.id)) => + println("ModifierFromRemote") ModifiersCache.put(toKey(mod.id), mod, history) getModifierForApplying() case ModifierFromRemote(modifier) => + println(s"history.isModifierDefined(mod.id): ${history.isModifierDefined(modifier.id)} ModifiersCache.contains(toKey(mod.id): ${ModifiersCache.contains(toKey(modifier.id))}") logger.info(s"Modifier ${modifier.encodedId} contains in history or in modifiers cache. Reject it.") case LocallyGeneratedBlock(block) => @@ -76,6 +78,7 @@ class HistoryApplicator(history: History, case Right(progressInfo) if progressInfo.toApply.nonEmpty => logger.info(s"Modifier ${modifier.encodedId} successfully applied to history.") modifiersQueue = modifiersQueue.enqueue(modifier.encodedId -> progressInfo) + println(s"modifiersQueue.enqueue ${modifiersQueue.size}") logger.info(s"New element put into queue. Current queue size is ${modifiersQueue.length}." + s"Current number of applied modifiers is $currentNumberOfAppliedModifiers.") influxRef.foreach(ref => @@ -88,11 +91,13 @@ class HistoryApplicator(history: History, walletApplicator ! WalletNeedRollbackTo(VersionTag !@@ progressInfo.branchPoint.get) if (progressInfo.toRemove.nonEmpty) nodeViewHolder ! TransactionsForWallet(progressInfo.toRemove) + println("history.append success") stateApplicator ! NotificationAboutNewModifier getModifierForApplying() case Right(progressInfo) => logger.info(s"Progress info is empty after appending to the state.") if (!isLocallyGenerated) requestDownloads(progressInfo) + println("SemanticallySuccessfulModifier") context.system.eventStream.publish(SemanticallySuccessfulModifier(modifier)) currentNumberOfAppliedModifiers -= 1 getModifierForApplying() @@ -105,11 +110,13 @@ class HistoryApplicator(history: History, sender() ! StartModifiersApplicationOnStateApplicator(pi, IndexedSeq.empty[PersistentModifier]) modifiersQueue = newQueue } + println(s"modifiersQueue.dequeue ${modifiersQueue.size}") case NotificationAboutSuccessfullyAppliedModifier => if (history.isFullChainSynced) { logger.info(s"BlockChain is synced on state applicator at height ${history.getBestHeaderHeight}!") ModifiersCache.setChainSynced() + println("FullBlockChainIsSynced") context.system.eventStream.publish(FullBlockChainIsSynced()) } currentNumberOfAppliedModifiers -= 1 @@ -131,23 +138,26 @@ class HistoryApplicator(history: History, case nonsense => logger.info(s"History applicator actor got from $sender message $nonsense.") } - def getModifierForApplying(): Unit = if (currentNumberOfAppliedModifiers < setting.levelDB.maxVersions) { - logger.debug(s"It's possible to append new modifier to history. Trying to get new one from the cache.") - if (locallyGeneratedModifiers.nonEmpty) locallyGeneratedModifiers.dequeueOption.foreach { - case (modifier, newQueue) => - locallyGeneratedModifiers = newQueue + def getModifierForApplying(): Unit = { + println(s"currentNumberOfAppliedModifiers: $currentNumberOfAppliedModifiers") + if (currentNumberOfAppliedModifiers < setting.levelDB.maxVersions) { + logger.debug(s"It's possible to append new modifier to history. Trying to get new one from the cache.") + if (locallyGeneratedModifiers.nonEmpty) locallyGeneratedModifiers.dequeueOption.foreach { + case (modifier, newQueue) => + locallyGeneratedModifiers = newQueue + currentNumberOfAppliedModifiers += 1 + logger.debug(s"Found new local modifier ${modifier.encodedId} with type ${modifier.modifierTypeId}." + + s"currentNumberOfAppliedModifiers is $currentNumberOfAppliedModifiers." + + s" Current locally generated modifiers size ${locallyGeneratedModifiers.size}.") + self ! ModifierToHistoryAppending(modifier, isLocallyGenerated = true) + } + else ModifiersCache.popCandidate(history).foreach { modifier => currentNumberOfAppliedModifiers += 1 - logger.debug(s"Found new local modifier ${modifier.encodedId} with type ${modifier.modifierTypeId}." + + logger.debug(s"Found new modifier ${modifier.encodedId} with type ${modifier.modifierTypeId}." + s"currentNumberOfAppliedModifiers is $currentNumberOfAppliedModifiers." + - s" Current locally generated modifiers size ${locallyGeneratedModifiers.size}.") - self ! ModifierToHistoryAppending(modifier, isLocallyGenerated = true) - } - else ModifiersCache.popCandidate(history).foreach { modifier => - currentNumberOfAppliedModifiers += 1 - logger.debug(s"Found new modifier ${modifier.encodedId} with type ${modifier.modifierTypeId}." + - s"currentNumberOfAppliedModifiers is $currentNumberOfAppliedModifiers." + - s" Current mod cache size ${ModifiersCache.size}") - self ! ModifierToHistoryAppending(modifier) + s" Current mod cache size ${ModifiersCache.size}") + self ! ModifierToHistoryAppending(modifier) + } } } diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 217ec1571e..8b0c7e1436 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -15,7 +15,8 @@ import encry.storage.VersionalStorage import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} import encry.storage.iodb.versionalIODB.IODBWrapper import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} -import encry.utils.{FileHelper, TestHelper} +import encry.utils.{FileHelper, TestHelper, Utils} +import encry.utils.ChainUtils.{privKey, _} import encry.view.{ModifiersCache, actors} import encry.view.actors.{HistoryApplicator, StateApplicator} import encry.view.actors.HistoryApplicator.{ModifierToHistoryAppending, StartModifiersApplicationOnStateApplicator} @@ -26,12 +27,14 @@ import encry.view.state.{BoxHolder, UtxoState} import encry.view.wallet.EncryWallet import io.iohk.iodb.LSMStore import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload, PayloadProtoSerializer} -import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import org.encryfoundation.common.utils.TaggedTypes.{Height, ModifierId} import org.encryfoundation.common.utils.constants.TestNetConstants import org.iq80.leveldb.Options -import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} +import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} import akka.testkit.{TestActorRef, TestKit} +import io.circe.Decoder.state import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.state.box.AssetBox import org.encryfoundation.common.utils.Algos import scala.concurrent.duration._ @@ -39,118 +42,189 @@ import scala.concurrent.duration._ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike with ImplicitSender - with InstanceFactory - with Matchers with BeforeAndAfterAll + with Matchers + with InstanceFactory + with OneInstancePerTest with Settings { override def afterAll: Unit = { TestKit.shutdownActorSystem(system) } - val timeout: FiniteDuration = 10 seconds - - def utxoFromBoxHolder(boxHolder: BoxHolder, dir: File, settings: EncryAppSettings): UtxoState = { - val storage = settings.storage.state match { - case VersionalStorage.IODB => - IODBWrapper(new LSMStore(dir, keepVersions = TestNetConstants.DefaultKeepVersions)) - case VersionalStorage.LevelDB => - val levelDBInit = LevelDbFactory.factory.open(dir, new Options) - VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settings.levelDB, keySize = 32)) + def genHistoryBlocks(initialHistory: History, count: Int, append: Boolean): (History, Seq[Block]) = { + (0 until count).foldLeft(initialHistory, Seq.empty[Block]) { + case ((prevHistory, blocks), _) => + val block: Block = generateNextBlock(prevHistory) + if(append) { + prevHistory.append(block.header) + prevHistory.append(block.payload) + } + (prevHistory.reportModifierIsValid(block), blocks :+ block) } - storage.insert( - StorageVersion @@ Array.fill(32)(0: Byte), - boxHolder.boxes.values.map(bx => (StorageKey !@@ bx.id, StorageValue @@ bx.bytes)).toList - ) - new UtxoState(storage, settings.constants) } - def blockToModifiers(block: Block): Seq[PersistentModifier] = { - Seq( - ModifiersToNetworkUtils.fromProto(Header.modifierTypeId, HeaderProtoSerializer.toProto(block.header).toByteArray).get, - ModifiersToNetworkUtils.fromProto(Payload.modifierTypeId, PayloadProtoSerializer.toProto(block.payload).toByteArray).get - ) - } + def blockToModifier(block: Block): Seq[PersistentModifier] = Seq(block.header, block.payload) -// "HistoryApplicator add locall generated block" should { -// "chain synced" in { -// -// val dir = FileHelper.getRandomTempDir -// val history: History = generateDummyHistory(settings) -// val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) -// -// val bxs = TestHelper.genAssetBoxes -// val boxHolder = BoxHolder(bxs) -// val state = utxoFromBoxHolder(boxHolder, FileHelper.getRandomTempDir, settings) -// -// val nodeViewHolder = TestProbe() -// val influx = TestProbe() -// -// val historyBlocks = (0 until 10).foldLeft(history, Seq.empty[Block]) { -// case ((prevHistory, blocks), _) => -// val block: Block = generateNextBlock(prevHistory) -// prevHistory.append(block.header) -// prevHistory.append(block.payload) -// (prevHistory.reportModifierIsValid(block), blocks :+ block) -// } -// -// val block: Block = generateNextBlock(history) -// -// val historyApplicator: TestActorRef[HistoryApplicator] = -// TestActorRef[HistoryApplicator]( -// HistoryApplicator.props(history, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) -// ) -// -// system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) + val numberOfInputsInOneTransaction = 1//10 + val transactionsNumberInEachBlock = 100//1000 + val numberOfOutputsInOneTransaction = 2//10 + val initialboxCount = 100//100000 + + val dir: File = FileHelper.getRandomTempDir + val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) + val bxs: IndexedSeq[AssetBox] = TestHelper.genAssetBoxes + val boxHolder: BoxHolder = BoxHolder(bxs) + val genesisBlock: Block = generateGenesisBlockValidForHistory//generateGenesisBlock(Height @@ 0) + val state: UtxoState = utxoFromBoxHolder(boxHolder, dir, None, settings, VersionalStorage.LevelDB) + .applyModifier(genesisBlock).right.get + + val initHistory: History = generateDummyHistory(settings) + + initHistory.append(genesisBlock.header) + initHistory.append(genesisBlock.payload) + val initialHistory: History = initHistory.reportModifierIsValid(genesisBlock) + + val block: Block = generateNextBlock(initialHistory) + +// val initialBoxes: IndexedSeq[AssetBox] = (0 until initialboxCount).map(nonce => +// genHardcodedBox(privKey.publicImage.address.address, nonce) +// ) +// val boxesHolder: BoxHolder = BoxHolder(initialBoxes) +// val state: UtxoState = utxoFromBoxHolder(boxesHolder, dir, None, settings, VersionalStorage.LevelDB) +// val genesisBlock: Block = generateGenesisBlockValidForState(state) +// val genesisState = state.applyModifier(genesisBlock).right.get // -// historyApplicator ! LocallyGeneratedBlock(block) +// val block: Block = generateNextBlockValidForState(genesisBlock, genesisState, +// initialBoxes.take(transactionsNumberInEachBlock * numberOfInputsInOneTransaction), +// transactionsNumberInEachBlock, numberOfInputsInOneTransaction, +// numberOfOutputsInOneTransaction) // -// expectMsg(timeout, FullBlockChainIsSynced()) +// val stateGenerationResults: (Vector[Block], Block, UtxoState, IndexedSeq[AssetBox]) = +// (0 until 10).foldLeft(Vector[Block](), genesisBlock, genesisState, initialBoxes) { +// case ((blocks, block, stateL, boxes), _) => +// val nextBlock: Block = generateNextBlockValidForState(block, stateL, +// boxes.take(transactionsNumberInEachBlock * numberOfInputsInOneTransaction), +// transactionsNumberInEachBlock, numberOfInputsInOneTransaction, +// numberOfOutputsInOneTransaction) +// val stateN: UtxoState = stateL.applyModifier(nextBlock).right.get +// (blocks :+ nextBlock, nextBlock, stateN, boxes.drop(transactionsNumberInEachBlock * numberOfInputsInOneTransaction)) // } -// } +// +// val chain: Vector[Block] = genesisBlock +: stateGenerationResults._1 - "HistoryApplicator" should { - "check queues" in { + val nodeViewHolder = TestProbe() + val influx = TestProbe() - val nodeViewHolder = TestProbe() - val influx = TestProbe() + val timeout: FiniteDuration = 10 seconds - val dir = FileHelper.getRandomTempDir - val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) - val bxs = TestHelper.genAssetBoxes - val boxHolder = BoxHolder(bxs) - val state = utxoFromBoxHolder(boxHolder, FileHelper.getRandomTempDir, settings) - val initialHistory: History = generateDummyHistory(settings) - val block1: Block = generateNextBlock(initialHistory) - val modifiers = blockToModifiers(block1) - val modifierIds = modifiers.map(_.id) + // "HistoryApplicator add locall generated block" should { + // "chain synced" in { + // + // val dir = FileHelper.getRandomTempDir + // val history: History = generateDummyHistory(settings) + // val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) + // + // val bxs = TestHelper.genAssetBoxes + // val boxHolder = BoxHolder(bxs) + // val state = utxoFromBoxHolder(boxHolder, FileHelper.getRandomTempDir, settings) + // + // val nodeViewHolder = TestProbe() + // val influx = TestProbe() + // + // val historyBlocks = (0 until 10).foldLeft(history, Seq.empty[Block]) { + // case ((prevHistory, blocks), _) => + // val block: Block = generateNextBlock(prevHistory) + // prevHistory.append(block.header) + // prevHistory.append(block.payload) + // (prevHistory.reportModifierIsValid(block), blocks :+ block) + // } + // + // val block: Block = generateNextBlock(history) + // + // val historyApplicator: TestActorRef[HistoryApplicator] = + // TestActorRef[HistoryApplicator]( + // HistoryApplicator.props(history, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) + // ) + // + // system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) + // + // historyApplicator ! LocallyGeneratedBlock(block) + // + // expectMsg(timeout, FullBlockChainIsSynced()) + // } + // } + "HistoryApplicator" should { + "check queues" in { val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( HistoryApplicator.props(initialHistory, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) ) + val modifiers: Seq[PersistentModifier] = blockToModifier(block) + val modifierIds: Seq[ModifierId] = modifiers.map(_.id) + modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) val modifiersQueue = historyApplicator.underlyingActor.modifiersQueue - val currentNumberOfAppliedModifiers = historyApplicator.underlyingActor.currentNumberOfAppliedModifiers + modifiersQueue.size shouldBe 2 + historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 historyApplicator ! StateApplicator.RequestNextModifier - val modifiersQueue1 = historyApplicator.underlyingActor.modifiersQueue - val currentNumberOfAppliedModifiers1 = historyApplicator.underlyingActor.currentNumberOfAppliedModifiers + historyApplicator.underlyingActor.modifiersQueue shouldBe 0 + historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 + val history = historyApplicator.underlyingActor.history expectMsgType[StartModifiersApplicationOnStateApplicator](timeout) - modifiersQueue.size shouldBe 2 - currentNumberOfAppliedModifiers shouldBe 2 - modifiersQueue1.size shouldBe 0 - currentNumberOfAppliedModifiers1 shouldBe 2 - assert(modifierIds.forall(history.isModifierDefined)) assert(modifierIds.map(Algos.encode) == modifiersQueue.map(_._1)) } } + "HistoryApplicator" should { + "check queue for rollback limit" in { + + val initHistory1: History = generateDummyHistory(settings) + initHistory1.append(genesisBlock.header) + initHistory1.append(genesisBlock.payload) + val initialHistory1: History = initHistory.reportModifierIsValid(genesisBlock) + + val historyApplicator: TestActorRef[HistoryApplicator] = + TestActorRef[HistoryApplicator]( + HistoryApplicator.props(initialHistory1, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) + //.withDispatcher("history-applicator-dispatcher"), "historyApplicator" + ) + + val modifiers: Seq[PersistentModifier] = + genHistoryBlocks(initialHistory, /*settings.levelDB.maxVersions + */ 10, false)._2 + .flatMap(blockToModifier) + + system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) + + modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) + + // println(s"modifiersQueue: ${historyApplicator.underlyingActor.modifiersQueue.size}") + // println(s"currentNumberOfAppliedModifiers: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") + + // historyApplicator.underlyingActor.modifiersQueue shouldBe 0 + // historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 + + //val modifiersQueue = historyApplicator.underlyingActor.modifiersQueue + //val history = historyApplicator.underlyingActor.history + + //expectMsgType[StartModifiersApplicationOnStateApplicator](timeout) + + Thread.sleep(30000) + + //expectMsg(timeout, FullBlockChainIsSynced()) + + // assert(modifierIds.forall(history.isModifierDefined)) + // assert(modifierIds.map(Algos.encode) == modifiersQueue.map(_._1)) + } + } + } From 699fc1cd079afee3a7f5f18a489c9d187111b3f2 Mon Sep 17 00:00:00 2001 From: capdev Date: Fri, 20 Sep 2019 12:37:18 +0500 Subject: [PATCH 06/39] try create right chain --- .../history/HistoryApplicatorTest.scala | 99 ++++++++++--------- 1 file changed, 54 insertions(+), 45 deletions(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 8b0c7e1436..1fa097990b 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -27,16 +27,20 @@ import encry.view.state.{BoxHolder, UtxoState} import encry.view.wallet.EncryWallet import io.iohk.iodb.LSMStore import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload, PayloadProtoSerializer} -import org.encryfoundation.common.utils.TaggedTypes.{Height, ModifierId} +import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height, ModifierId} import org.encryfoundation.common.utils.constants.TestNetConstants import org.iq80.leveldb.Options import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} import akka.testkit.{TestActorRef, TestKit} +import encry.modifiers.mempool.TransactionFactory import io.circe.Decoder.state +import org.encryfoundation.common.crypto.PrivateKey25519 import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.modifiers.state.box.AssetBox import org.encryfoundation.common.utils.Algos +import scala.collection.Seq import scala.concurrent.duration._ class HistoryApplicatorTest extends TestKit(ActorSystem()) @@ -52,14 +56,12 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) TestKit.shutdownActorSystem(system) } - def genHistoryBlocks(initialHistory: History, count: Int, append: Boolean): (History, Seq[Block]) = { + def genHistoryBlocks(initialHistory: History, count: Int): (History, Seq[Block]) = { (0 until count).foldLeft(initialHistory, Seq.empty[Block]) { case ((prevHistory, blocks), _) => val block: Block = generateNextBlock(prevHistory) - if(append) { prevHistory.append(block.header) prevHistory.append(block.payload) - } (prevHistory.reportModifierIsValid(block), blocks :+ block) } } @@ -69,49 +71,58 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val numberOfInputsInOneTransaction = 1//10 val transactionsNumberInEachBlock = 100//1000 val numberOfOutputsInOneTransaction = 2//10 - val initialboxCount = 100//100000 + val initialboxCount = 10//100000 val dir: File = FileHelper.getRandomTempDir val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) - val bxs: IndexedSeq[AssetBox] = TestHelper.genAssetBoxes - val boxHolder: BoxHolder = BoxHolder(bxs) - val genesisBlock: Block = generateGenesisBlockValidForHistory//generateGenesisBlock(Height @@ 0) - val state: UtxoState = utxoFromBoxHolder(boxHolder, dir, None, settings, VersionalStorage.LevelDB) - .applyModifier(genesisBlock).right.get - - val initHistory: History = generateDummyHistory(settings) - - initHistory.append(genesisBlock.header) - initHistory.append(genesisBlock.payload) - val initialHistory: History = initHistory.reportModifierIsValid(genesisBlock) - - val block: Block = generateNextBlock(initialHistory) - -// val initialBoxes: IndexedSeq[AssetBox] = (0 until initialboxCount).map(nonce => -// genHardcodedBox(privKey.publicImage.address.address, nonce) -// ) -// val boxesHolder: BoxHolder = BoxHolder(initialBoxes) -// val state: UtxoState = utxoFromBoxHolder(boxesHolder, dir, None, settings, VersionalStorage.LevelDB) -// val genesisBlock: Block = generateGenesisBlockValidForState(state) -// val genesisState = state.applyModifier(genesisBlock).right.get -// +// val bxs: IndexedSeq[AssetBox] = TestHelper.genAssetBoxes +// val boxHolder: BoxHolder = BoxHolder(bxs) +// val genesisBlock: Block = generateGenesisBlockValidForHistory//generateGenesisBlock(Height @@ 0) +// val state: UtxoState = utxoFromBoxHolder(boxHolder, dir, None, settings, VersionalStorage.LevelDB) +// .applyModifier(genesisBlock).right.get + + val initialBoxes: IndexedSeq[AssetBox] = (0 until initialboxCount).map(nonce => + genHardcodedBox(privKey.publicImage.address.address, nonce) + ) + val boxesHolder: BoxHolder = BoxHolder(initialBoxes) + val state: UtxoState = utxoFromBoxHolder(boxesHolder, dir, None, settings, VersionalStorage.LevelDB) + val genesisBlock: Block = generateGenesisBlockValidForState(state) + val genesisState: UtxoState = state.applyModifier(genesisBlock).right.get + // val block: Block = generateNextBlockValidForState(genesisBlock, genesisState, // initialBoxes.take(transactionsNumberInEachBlock * numberOfInputsInOneTransaction), // transactionsNumberInEachBlock, numberOfInputsInOneTransaction, // numberOfOutputsInOneTransaction) -// -// val stateGenerationResults: (Vector[Block], Block, UtxoState, IndexedSeq[AssetBox]) = -// (0 until 10).foldLeft(Vector[Block](), genesisBlock, genesisState, initialBoxes) { -// case ((blocks, block, stateL, boxes), _) => -// val nextBlock: Block = generateNextBlockValidForState(block, stateL, -// boxes.take(transactionsNumberInEachBlock * numberOfInputsInOneTransaction), -// transactionsNumberInEachBlock, numberOfInputsInOneTransaction, -// numberOfOutputsInOneTransaction) -// val stateN: UtxoState = stateL.applyModifier(nextBlock).right.get -// (blocks :+ nextBlock, nextBlock, stateN, boxes.drop(transactionsNumberInEachBlock * numberOfInputsInOneTransaction)) -// } -// -// val chain: Vector[Block] = genesisBlock +: stateGenerationResults._1 + + val stateGenerationResults: (List[(Block, Block)], Block, UtxoState, IndexedSeq[AssetBox]) = + (0 until 10).foldLeft(List.empty[(Block, Block)], genesisBlock, state, initialBoxes) { + case ((blocks, block, stateL, boxes), _) => + val nextBlockMainChain: Block = generateNextBlockForStateWithSpendingAllPreviousBoxes( + block, + stateL, + block.payload.txs.flatMap(_.newBoxes.map(_.asInstanceOf[AssetBox])).toIndexedSeq) + val nextBlockFork: Block = generateNextBlockForStateWithSpendingAllPreviousBoxes( + block, + stateL, + block.payload.txs.flatMap(_.newBoxes.map(_.asInstanceOf[AssetBox])).toIndexedSeq, + addDiff = Difficulty @@ BigInt(100) + ) + val stateN: UtxoState = stateL.applyModifier(nextBlockMainChain).right.get + (blocks :+ (nextBlockMainChain, nextBlockFork), + nextBlockMainChain, stateN, boxes.drop(transactionsNumberInEachBlock * numberOfInputsInOneTransaction) + ) + } + + val chain: List[Block] = genesisBlock +: stateGenerationResults._1.map(_._1) + val forkBlocks: List[Block] = genesisBlock +: stateGenerationResults._1.map(_._2) + //state = stateGenerationResults._3 + +// val initHistory: History = generateDummyHistory(settings) +// initHistory.append(genesisBlock.header) +// initHistory.append(genesisBlock.payload) +// val initialHistory: History = initHistory.reportModifierIsValid(genesisBlock) + +// val block: Block = generateNextBlock(initialHistory) val nodeViewHolder = TestProbe() val influx = TestProbe() @@ -189,18 +200,16 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) "check queue for rollback limit" in { val initHistory1: History = generateDummyHistory(settings) - initHistory1.append(genesisBlock.header) - initHistory1.append(genesisBlock.payload) - val initialHistory1: History = initHistory.reportModifierIsValid(genesisBlock) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( - HistoryApplicator.props(initialHistory1, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) + HistoryApplicator.props(initHistory1, settings, stateGenerationResults._3, wallet, nodeViewHolder.ref, Some(influx.ref)) //.withDispatcher("history-applicator-dispatcher"), "historyApplicator" ) val modifiers: Seq[PersistentModifier] = - genHistoryBlocks(initialHistory, /*settings.levelDB.maxVersions + */ 10, false)._2 + chain + //genHistoryBlocks(initialHistory, /*settings.levelDB.maxVersions + */ 10)._2 .flatMap(blockToModifier) system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) From 8ab890b1e1d5e20d0dd5dc2c20b25eec555b53ee Mon Sep 17 00:00:00 2001 From: capdev Date: Fri, 20 Sep 2019 18:26:30 +0500 Subject: [PATCH 07/39] fix timestamps create valid chain fix UtxoState.applyModifier off powScheme.verify --- .../src/test/scala/benches/StateBenches.scala | 2 +- .../scala/benches/StateRollbackBench.scala | 2 +- src/main/scala/encry/utils/ChainUtils.scala | 21 ++- .../history/HistoryModifiersValidator.scala | 6 +- .../scala/encry/view/state/UtxoState.scala | 13 +- .../encry/modifiers/InstanceFactory.scala | 7 +- .../history/HistoryApplicatorTest.scala | 158 ++++++++---------- 7 files changed, 101 insertions(+), 108 deletions(-) diff --git a/benchmarks/src/test/scala/benches/StateBenches.scala b/benchmarks/src/test/scala/benches/StateBenches.scala index 8ad8418676..e076e9ed41 100644 --- a/benchmarks/src/test/scala/benches/StateBenches.scala +++ b/benchmarks/src/test/scala/benches/StateBenches.scala @@ -73,7 +73,7 @@ object StateBenches extends BenchSettings { ) val boxesHolder: BoxHolder = BoxHolder(initialBoxes) var state: UtxoState = utxoFromBoxHolder(boxesHolder, tmpDir, None, settings, VersionalStorage.LevelDB) - val genesisBlock: Block = generateGenesisBlockValidForState(state) + val genesisBlock: Block = generateGenesisBlockValidForState state = state.applyModifier(genesisBlock).right.get diff --git a/benchmarks/src/test/scala/benches/StateRollbackBench.scala b/benchmarks/src/test/scala/benches/StateRollbackBench.scala index e0989c2af8..26eea4c59c 100644 --- a/benchmarks/src/test/scala/benches/StateRollbackBench.scala +++ b/benchmarks/src/test/scala/benches/StateRollbackBench.scala @@ -66,7 +66,7 @@ object StateRollbackBench extends BenchSettings { ) val boxesHolder: BoxHolder = BoxHolder(initialBoxes) var state: UtxoState = utxoFromBoxHolder(boxesHolder, tmpDir, None, settings, VersionalStorage.LevelDB) - val genesisBlock: Block = generateGenesisBlockValidForState(state) + val genesisBlock: Block = generateGenesisBlockValidForState state = state.applyModifier(genesisBlock).right.get diff --git a/src/main/scala/encry/utils/ChainUtils.scala b/src/main/scala/encry/utils/ChainUtils.scala index 235bb8c024..bc4437970b 100644 --- a/src/main/scala/encry/utils/ChainUtils.scala +++ b/src/main/scala/encry/utils/ChainUtils.scala @@ -5,7 +5,7 @@ import java.io.File import akka.actor.ActorRef import com.typesafe.scalalogging.StrictLogging import encry.modifiers.mempool.TransactionFactory -import encry.settings.{Settings, EncryAppSettings} +import encry.settings.{EncryAppSettings, Settings} import encry.storage.VersionalStorage import encry.storage.VersionalStorage.{StorageKey, StorageType, StorageValue, StorageVersion} import encry.storage.iodb.versionalIODB.IODBWrapper @@ -60,7 +60,7 @@ object ChainUtils extends Settings with StrictLogging { ) :: acc } - def generateGenesisBlockValidForState(state: UtxoState): Block = { + def generateGenesisBlockValidForState: Block = { val txs = Seq(coinbaseTransaction(0)) val header = genHeader.copy( parentId = Header.GenesisParentId, @@ -69,6 +69,18 @@ object ChainUtils extends Settings with StrictLogging { Block(header, Payload(header.id, txs)) } + def generateGenesisBlock(genesisHeight: Height): Block = { + val txs: Seq[Transaction] = Seq(ChainUtils.coinbaseTransaction(0)) + val txsRoot: Digest32 = Payload.rootHash(txs.map(_.id)) + val header = genHeader.copy( + timestamp = System.currentTimeMillis(), + parentId = Header.GenesisParentId, + height = genesisHeight, + transactionsRoot = txsRoot + ) + Block(header, Payload(header.id, txs)) + } + def generateGenesisBlockValidForHistory: Block = { val header = genHeader.copy(parentId = Header.GenesisParentId, height = settings.constants.GenesisHeight) Block(header, Payload(header.id, Seq(coinbaseTransaction))) @@ -113,13 +125,14 @@ object ChainUtils extends Settings with StrictLogging { box: Seq[AssetBox], splitCoef: Int = 2, addDiff: Difficulty = Difficulty @@ BigInt(0)): Block = { + val timestamp = System.currentTimeMillis() val transactions: Seq[Transaction] = box.indices.foldLeft(box, Seq.empty[Transaction]) { case ((boxes, transactionsL), _) => val tx: Transaction = defaultPaymentTransactionScratch( privKey, fee = 1, - timestamp = 11L, + timestamp, useBoxes = IndexedSeq(boxes.head), recipient = privKey.publicImage.address.address, amount = boxes.head.amount - 1, @@ -132,7 +145,7 @@ object ChainUtils extends Settings with StrictLogging { 1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), - System.currentTimeMillis(), + timestamp, prevBlock.header.height + 1, R.nextLong(), Difficulty @@ (BigInt(1) + addDiff), diff --git a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala index 8951496ab6..9d46b017c1 100644 --- a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala +++ b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala @@ -72,9 +72,9 @@ trait HistoryModifiersValidator extends HistoryApi with Settings { _ <- Either.cond(heightOf(h.parentId).exists(h => getBestHeaderHeight - h < settings.constants.MaxRollbackDepth), (), HeaderFatalValidationError(s"Header ${h.encodedId} has height greater than max roll back depth")) powSchemeValidationResult = powScheme.verify(h) - _ <- Either.cond(powSchemeValidationResult.isRight, (), - HeaderFatalValidationError(s"Wrong proof-of-work solution in header ${h.encodedId}" + - s" caused: $powSchemeValidationResult")) +// _ <- Either.cond(powSchemeValidationResult.isRight, (), +// HeaderFatalValidationError(s"Wrong proof-of-work solution in header ${h.encodedId}" + +// s" caused: $powSchemeValidationResult")) _ <- Either.cond(isSemanticallyValid(h.parentId) != ModifierSemanticValidity.Invalid, (), HeaderFatalValidationError(s"Header ${h.encodedId} is semantically invalid")) _ <- Either.cond(h.timestamp - timeProvider.estimatedTime <= settings.constants.MaxTimeDrift, (), diff --git a/src/main/scala/encry/view/state/UtxoState.scala b/src/main/scala/encry/view/state/UtxoState.scala index 0f70f1ef6f..53ff239327 100644 --- a/src/main/scala/encry/view/state/UtxoState.scala +++ b/src/main/scala/encry/view/state/UtxoState.scala @@ -53,7 +53,10 @@ final case class UtxoState(storage: VersionalStorage, constants: Constants) val result = mod match { case header: Header => logger.info(s"\n\nStarting to applyModifier as a header: ${Algos.encode(mod.id)} to state at height ${header.height}") - UtxoState(storage, constants).asRight[List[ModifierApplyError]] + val newState: UtxoState = UtxoState(storage, constants) + newState.height = height + newState.lastBlockTimestamp = header.timestamp + newState.asRight[List[ModifierApplyError]] case block: Block => logger.info(s"\n\nStarting to applyModifier as a Block: ${Algos.encode(mod.id)} to state at height ${block.header.height}") val lastTxId = block.payload.txs.last.id @@ -80,10 +83,10 @@ final case class UtxoState(storage: VersionalStorage, constants: Constants) combinedStateChange.inputsToDb.toList ) logger.info(s"Time of insert: ${(System.currentTimeMillis() - insertTimestart)/1000L} s") - UtxoState( - storage, - constants - ).asRight[List[ModifierApplyError]] + val newState: UtxoState = UtxoState(storage, constants) + newState.height = Height @@ block.header.height + newState.lastBlockTimestamp = block.header.timestamp + newState.asRight[List[ModifierApplyError]] } ) } diff --git a/src/test/scala/encry/modifiers/InstanceFactory.scala b/src/test/scala/encry/modifiers/InstanceFactory.scala index 2444c6f56b..0cd2d3fa6b 100755 --- a/src/test/scala/encry/modifiers/InstanceFactory.scala +++ b/src/test/scala/encry/modifiers/InstanceFactory.scala @@ -4,7 +4,7 @@ import encry.modifiers.mempool._ import encry.modifiers.state.Keys import encry.settings.{EncryAppSettings, NodeSettings} import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} -import encry.utils.{EncryGenerator, FileHelper, NetworkTimeProvider, TestHelper} +import encry.utils.{ChainUtils, EncryGenerator, FileHelper, NetworkTimeProvider, TestHelper} import encry.view.history.History import encry.view.history.storage.HistoryStorage import io.iohk.iodb.LSMStore @@ -14,7 +14,6 @@ import org.encryfoundation.common.modifiers.state.box.{AssetBox, EncryPropositio import org.encryfoundation.common.modifiers.state.box.Box.Amount import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{Height, _} - import org.encryfoundation.prismlang.compiler.CompiledContract import org.encryfoundation.prismlang.core.Ast.Expr import org.encryfoundation.prismlang.core.{Ast, Types} @@ -47,14 +46,14 @@ trait InstanceFactory extends Keys with EncryGenerator { } def generateGenesisBlock(genesisHeight: Height): Block = { - val txs: Seq[Transaction] = Seq(coinbaseTransaction) + val txs: Seq[Transaction] = Seq(ChainUtils.coinbaseTransaction(0)) val txsRoot: Digest32 = Payload.rootHash(txs.map(_.id)) val header = genHeader.copy( parentId = Header.GenesisParentId, height = genesisHeight, transactionsRoot = txsRoot ) - Block(header, Payload(header.id, Seq(coinbaseTransaction))) + Block(header, Payload(header.id, txs)) } def paymentTransactionDynamic: Transaction = { diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 1fa097990b..89334f1409 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -2,43 +2,25 @@ package encry.modifiers.history import java.io.File -import akka.actor.{ActorRef, ActorSystem, Props} -import akka.testkit.{ImplicitSender, TestKit, TestProbe} -import akka.util.Timeout -import encry.EncryApp.settings +import akka.actor.ActorSystem +import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} import encry.modifiers.InstanceFactory -import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.ModifiersToNetworkUtils -import encry.network.NodeViewSynchronizer.ReceivableMessages.{SemanticallySuccessfulModifier, UpdatedHistory} -import encry.settings.{EncryAppSettings, Settings} +import encry.settings.Settings import encry.storage.VersionalStorage -import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} -import encry.storage.iodb.versionalIODB.IODBWrapper -import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} -import encry.utils.{FileHelper, TestHelper, Utils} -import encry.utils.ChainUtils.{privKey, _} -import encry.view.{ModifiersCache, actors} +import encry.utils.ChainUtils._ +import encry.utils.{ChainUtils, FileHelper} +import encry.view.actors.HistoryApplicator.StartModifiersApplicationOnStateApplicator +import encry.view.actors.NodeViewHolder.ReceivableMessages.ModifierFromRemote import encry.view.actors.{HistoryApplicator, StateApplicator} -import encry.view.actors.HistoryApplicator.{ModifierToHistoryAppending, StartModifiersApplicationOnStateApplicator} -import encry.view.actors.NodeViewHolder.ReceivableMessages.{LocallyGeneratedBlock, ModifierFromRemote} -import encry.view.actors.StateApplicator.NotificationAboutSuccessfullyAppliedModifier import encry.view.history.History import encry.view.state.{BoxHolder, UtxoState} import encry.view.wallet.EncryWallet -import io.iohk.iodb.LSMStore -import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload, PayloadProtoSerializer} -import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height, ModifierId} -import org.encryfoundation.common.utils.constants.TestNetConstants -import org.iq80.leveldb.Options -import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -import akka.testkit.{TestActorRef, TestKit} -import encry.modifiers.mempool.TransactionFactory -import io.circe.Decoder.state -import org.encryfoundation.common.crypto.PrivateKey25519 import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.state.box.AssetBox import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.{Height, ModifierId} +import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} import scala.collection.Seq import scala.concurrent.duration._ @@ -66,69 +48,60 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) } } - def blockToModifier(block: Block): Seq[PersistentModifier] = Seq(block.header, block.payload) + def blockToModifiers(block: Block): Seq[PersistentModifier] = Seq(block.header, block.payload) + + def generateChain(blockQty: Int): (UtxoState, List[Block]) = { + val numberOfInputsInOneTransaction = 1//10 + val transactionsNumberInEachBlock = 100//1000 + val initialboxCount = 10//100000 + + val initialBoxes: IndexedSeq[AssetBox] = (0 until initialboxCount).map(nonce => + genHardcodedBox(privKey.publicImage.address.address, nonce) + ) + val boxesHolder: BoxHolder = BoxHolder(initialBoxes) + + val genesisBlock: Block = ChainUtils.generateGenesisBlock(Height @@ 0) + + val state: UtxoState = utxoFromBoxHolder(boxesHolder, dir, None, settings, VersionalStorage.LevelDB) + .applyModifier(genesisBlock).right.get + + val stateGenerationResults: (List[Block], Block, UtxoState, IndexedSeq[AssetBox]) = + (0 until blockQty).foldLeft(List.empty[Block], genesisBlock, state, initialBoxes) { + case ((blocks, block, stateL, boxes), _) => + val nextBlockMainChain: Block = generateNextBlockForStateWithSpendingAllPreviousBoxes( + block, stateL, block.payload.txs.flatMap(_.newBoxes.map(_.asInstanceOf[AssetBox])).toIndexedSeq) + val stateN: UtxoState = stateL.applyModifier(nextBlockMainChain).right.get + (blocks :+ nextBlockMainChain, + nextBlockMainChain, stateN, boxes.drop(transactionsNumberInEachBlock * numberOfInputsInOneTransaction) + ) + } + (state, genesisBlock +: stateGenerationResults._1) + } - val numberOfInputsInOneTransaction = 1//10 - val transactionsNumberInEachBlock = 100//1000 - val numberOfOutputsInOneTransaction = 2//10 - val initialboxCount = 10//100000 + def printIds(blocks: List[Block]) = { + blocks.foreach { b => + println(s"header.timestamp: ${b.header.timestamp}") + println(s"header.id: ${Algos.encode(b.header.id)}") + println(s"header.payloadId: ${Algos.encode(b.header.payloadId)}") + println(s"payload.id: ${Algos.encode(b.payload.id)}") + println(s"payload.headerId: ${Algos.encode(b.payload.headerId)}") + } + } val dir: File = FileHelper.getRandomTempDir val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) -// val bxs: IndexedSeq[AssetBox] = TestHelper.genAssetBoxes -// val boxHolder: BoxHolder = BoxHolder(bxs) -// val genesisBlock: Block = generateGenesisBlockValidForHistory//generateGenesisBlock(Height @@ 0) -// val state: UtxoState = utxoFromBoxHolder(boxHolder, dir, None, settings, VersionalStorage.LevelDB) -// .applyModifier(genesisBlock).right.get - - val initialBoxes: IndexedSeq[AssetBox] = (0 until initialboxCount).map(nonce => - genHardcodedBox(privKey.publicImage.address.address, nonce) - ) - val boxesHolder: BoxHolder = BoxHolder(initialBoxes) - val state: UtxoState = utxoFromBoxHolder(boxesHolder, dir, None, settings, VersionalStorage.LevelDB) - val genesisBlock: Block = generateGenesisBlockValidForState(state) - val genesisState: UtxoState = state.applyModifier(genesisBlock).right.get - -// val block: Block = generateNextBlockValidForState(genesisBlock, genesisState, -// initialBoxes.take(transactionsNumberInEachBlock * numberOfInputsInOneTransaction), -// transactionsNumberInEachBlock, numberOfInputsInOneTransaction, -// numberOfOutputsInOneTransaction) - - val stateGenerationResults: (List[(Block, Block)], Block, UtxoState, IndexedSeq[AssetBox]) = - (0 until 10).foldLeft(List.empty[(Block, Block)], genesisBlock, state, initialBoxes) { - case ((blocks, block, stateL, boxes), _) => - val nextBlockMainChain: Block = generateNextBlockForStateWithSpendingAllPreviousBoxes( - block, - stateL, - block.payload.txs.flatMap(_.newBoxes.map(_.asInstanceOf[AssetBox])).toIndexedSeq) - val nextBlockFork: Block = generateNextBlockForStateWithSpendingAllPreviousBoxes( - block, - stateL, - block.payload.txs.flatMap(_.newBoxes.map(_.asInstanceOf[AssetBox])).toIndexedSeq, - addDiff = Difficulty @@ BigInt(100) - ) - val stateN: UtxoState = stateL.applyModifier(nextBlockMainChain).right.get - (blocks :+ (nextBlockMainChain, nextBlockFork), - nextBlockMainChain, stateN, boxes.drop(transactionsNumberInEachBlock * numberOfInputsInOneTransaction) - ) - } - - val chain: List[Block] = genesisBlock +: stateGenerationResults._1.map(_._1) - val forkBlocks: List[Block] = genesisBlock +: stateGenerationResults._1.map(_._2) - //state = stateGenerationResults._3 -// val initHistory: History = generateDummyHistory(settings) -// initHistory.append(genesisBlock.header) -// initHistory.append(genesisBlock.payload) -// val initialHistory: History = initHistory.reportModifierIsValid(genesisBlock) - -// val block: Block = generateNextBlock(initialHistory) + val initialHistory: History = generateDummyHistory(settings) val nodeViewHolder = TestProbe() val influx = TestProbe() val timeout: FiniteDuration = 10 seconds + //printIds(chain) + //println(s"${settings.constants.RetargetingEpochsQty}") + //println(s"${settings.constants.EpochLength}") + // "HistoryApplicator add locall generated block" should { // "chain synced" in { // @@ -168,16 +141,22 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) "HistoryApplicator" should { "check queues" in { + + val (state, chain) = generateChain(1) + + val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) + val modifierIds = modifiers.map(_.id) + val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( HistoryApplicator.props(initialHistory, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) ) - val modifiers: Seq[PersistentModifier] = blockToModifier(block) - val modifierIds: Seq[ModifierId] = modifiers.map(_.id) - modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) + println(s"modifiersQueue: ${historyApplicator.underlyingActor.modifiersQueue.size}") + println(s"currentNumberOfAppliedModifiers: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") + val modifiersQueue = historyApplicator.underlyingActor.modifiersQueue modifiersQueue.size shouldBe 2 historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 @@ -199,20 +178,19 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) "HistoryApplicator" should { "check queue for rollback limit" in { - val initHistory1: History = generateDummyHistory(settings) + println(s"maxVersions: ${settings.levelDB.maxVersions}") + + val (state, chain) = generateChain(120) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( - HistoryApplicator.props(initHistory1, settings, stateGenerationResults._3, wallet, nodeViewHolder.ref, Some(influx.ref)) + HistoryApplicator.props(initialHistory, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) //.withDispatcher("history-applicator-dispatcher"), "historyApplicator" ) - val modifiers: Seq[PersistentModifier] = - chain - //genHistoryBlocks(initialHistory, /*settings.levelDB.maxVersions + */ 10)._2 - .flatMap(blockToModifier) + val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) - system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) + //system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) From 77143d8ccbf893237b2d6993deb456c2878e535c Mon Sep 17 00:00:00 2001 From: capdev Date: Mon, 23 Sep 2019 10:21:05 +0500 Subject: [PATCH 08/39] debug prints --- .../encry/view/actors/HistoryApplicator.scala | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/scala/encry/view/actors/HistoryApplicator.scala b/src/main/scala/encry/view/actors/HistoryApplicator.scala index 37068c64e5..cea8d4b6ee 100644 --- a/src/main/scala/encry/view/actors/HistoryApplicator.scala +++ b/src/main/scala/encry/view/actors/HistoryApplicator.scala @@ -49,12 +49,12 @@ class HistoryApplicator(val history: History, override def receive: Receive = { case ModifierFromRemote(mod) if !history.isModifierDefined(mod.id) && !ModifiersCache.contains(toKey(mod.id)) => - println("ModifierFromRemote") + //println("ModifierFromRemote") ModifiersCache.put(toKey(mod.id), mod, history) getModifierForApplying() case ModifierFromRemote(modifier) => - println(s"history.isModifierDefined(mod.id): ${history.isModifierDefined(modifier.id)} ModifiersCache.contains(toKey(mod.id): ${ModifiersCache.contains(toKey(modifier.id))}") + //println(s"history.isModifierDefined(mod.id): ${history.isModifierDefined(modifier.id)} ModifiersCache.contains(toKey(mod.id): ${ModifiersCache.contains(toKey(modifier.id))}") logger.info(s"Modifier ${modifier.encodedId} contains in history or in modifiers cache. Reject it.") case LocallyGeneratedBlock(block) => @@ -78,7 +78,7 @@ class HistoryApplicator(val history: History, case Right(progressInfo) if progressInfo.toApply.nonEmpty => logger.info(s"Modifier ${modifier.encodedId} successfully applied to history.") modifiersQueue = modifiersQueue.enqueue(modifier.encodedId -> progressInfo) - println(s"modifiersQueue.enqueue ${modifiersQueue.size}") + //println(s"modifiersQueue.enqueue ${modifiersQueue.size}") logger.info(s"New element put into queue. Current queue size is ${modifiersQueue.length}." + s"Current number of applied modifiers is $currentNumberOfAppliedModifiers.") influxRef.foreach(ref => @@ -91,13 +91,13 @@ class HistoryApplicator(val history: History, walletApplicator ! WalletNeedRollbackTo(VersionTag !@@ progressInfo.branchPoint.get) if (progressInfo.toRemove.nonEmpty) nodeViewHolder ! TransactionsForWallet(progressInfo.toRemove) - println("history.append success") + //println("history.append success") stateApplicator ! NotificationAboutNewModifier getModifierForApplying() case Right(progressInfo) => logger.info(s"Progress info is empty after appending to the state.") if (!isLocallyGenerated) requestDownloads(progressInfo) - println("SemanticallySuccessfulModifier") + //println("SemanticallySuccessfulModifier") context.system.eventStream.publish(SemanticallySuccessfulModifier(modifier)) currentNumberOfAppliedModifiers -= 1 getModifierForApplying() @@ -110,13 +110,13 @@ class HistoryApplicator(val history: History, sender() ! StartModifiersApplicationOnStateApplicator(pi, IndexedSeq.empty[PersistentModifier]) modifiersQueue = newQueue } - println(s"modifiersQueue.dequeue ${modifiersQueue.size}") + //println(s"modifiersQueue.dequeue ${modifiersQueue.size}") case NotificationAboutSuccessfullyAppliedModifier => if (history.isFullChainSynced) { logger.info(s"BlockChain is synced on state applicator at height ${history.getBestHeaderHeight}!") ModifiersCache.setChainSynced() - println("FullBlockChainIsSynced") + //println("FullBlockChainIsSynced") context.system.eventStream.publish(FullBlockChainIsSynced()) } currentNumberOfAppliedModifiers -= 1 @@ -140,6 +140,7 @@ class HistoryApplicator(val history: History, def getModifierForApplying(): Unit = { println(s"currentNumberOfAppliedModifiers: $currentNumberOfAppliedModifiers") + println(s"modifiersQueue.size: ${modifiersQueue.size}") if (currentNumberOfAppliedModifiers < setting.levelDB.maxVersions) { logger.debug(s"It's possible to append new modifier to history. Trying to get new one from the cache.") if (locallyGeneratedModifiers.nonEmpty) locallyGeneratedModifiers.dequeueOption.foreach { From 7d76dd8eecd5c2780426f63441a8ac330e51e310 Mon Sep 17 00:00:00 2001 From: capdev Date: Mon, 23 Sep 2019 17:14:46 +0500 Subject: [PATCH 09/39] add check rollback for invalid state test --- src/main/scala/encry/utils/ChainUtils.scala | 98 ++++++++++++++++- .../encry/view/actors/HistoryApplicator.scala | 12 ++- .../encry/view/actors/StateApplicator.scala | 2 + .../history/HistoryApplicatorTest.scala | 101 ++++++++++++------ 4 files changed, 169 insertions(+), 44 deletions(-) diff --git a/src/main/scala/encry/utils/ChainUtils.scala b/src/main/scala/encry/utils/ChainUtils.scala index bc4437970b..ee7e3775a5 100644 --- a/src/main/scala/encry/utils/ChainUtils.scala +++ b/src/main/scala/encry/utils/ChainUtils.scala @@ -4,6 +4,8 @@ import java.io.File import akka.actor.ActorRef import com.typesafe.scalalogging.StrictLogging +import encry.EncryApp.timeProvider +import encry.consensus.EncrySupplyController import encry.modifiers.mempool.TransactionFactory import encry.settings.{EncryAppSettings, Settings} import encry.storage.VersionalStorage @@ -11,6 +13,7 @@ import encry.storage.VersionalStorage.{StorageKey, StorageType, StorageValue, St import encry.storage.iodb.versionalIODB.IODBWrapper import encry.storage.levelDb.versionalLevelDB.VersionalLevelDBCompanion.{LevelDBVersion, VersionalLevelDbKey, VersionalLevelDbValue} import encry.storage.levelDb.versionalLevelDB._ +import encry.utils.NetworkTime.Time import encry.view.history.History import encry.view.history.storage.HistoryStorage import encry.view.state.{BoxHolder, UtxoState} @@ -70,7 +73,17 @@ object ChainUtils extends Settings with StrictLogging { } def generateGenesisBlock(genesisHeight: Height): Block = { - val txs: Seq[Transaction] = Seq(ChainUtils.coinbaseTransaction(0)) + val supplyTotal: Amount = settings.constants.InitialEmissionAmount + + val coinbaseTrans = TransactionFactory.coinbaseTransactionScratch( + privKey.publicImage, + System.currentTimeMillis(), + supplyTotal, + amount = 0L, + height = genesisHeight + ) + + val txs: Seq[Transaction] = Seq(coinbaseTrans) val txsRoot: Digest32 = Payload.rootHash(txs.map(_.id)) val header = genHeader.copy( timestamp = System.currentTimeMillis(), @@ -154,6 +167,83 @@ object ChainUtils extends Settings with StrictLogging { Block(header, Payload(header.id, transactions)) } + def generateNextBlockSpend(prevBlock: Block, + state: UtxoState, + box: Seq[AssetBox], + amount: Amount + ): Block = { + + val transactions: Seq[Transaction] = Seq( + defaultPaymentTransactionScratch( + privKey, + fee = 0L, + timestamp = System.currentTimeMillis(), + useBoxes = box.toIndexedSeq, + recipient = randomAddress, + amount = amount, + numOfOutputs = 1 + ), + defaultPaymentTransactionScratch( + privKey, + fee = 0L, + timestamp = System.currentTimeMillis(), + useBoxes = box.toIndexedSeq, + recipient = randomAddress, + amount = amount, + numOfOutputs = 1 + ), + coinbaseTransaction(prevBlock.header.height + 1) + ) + + val header = Header( + 1.toByte, + prevBlock.id, + Payload.rootHash(transactions.map(_.id)), + System.currentTimeMillis(), + prevBlock.header.height + 1, + R.nextLong(), + Difficulty @@ BigInt(1), + EquihashSolution(Seq(1, 3)) + ) + Block(header, Payload(header.id, transactions)) + } + + def generateNextBlockForStateWithInvalidTrans(prevBlock: Block, + state: UtxoState, + box: Seq[AssetBox], + splitCoef: Int = 2, + addDiff: Difficulty = Difficulty @@ BigInt(0)): Block = { + val timestamp = System.currentTimeMillis() + + val transactions: Seq[Transaction] = box.indices.foldLeft(box, Seq.empty[Transaction]) { + case ((boxes, transactionsL), _) => + val errorAmount = 1000000000L + println(s"amount: ${boxes.head.amount - 1 + errorAmount}") + val tx: Transaction = defaultPaymentTransactionScratch( + privKey, + fee = 1, + timestamp, + useBoxes = IndexedSeq(boxes.head), + recipient = privKey.publicImage.address.address, + amount = boxes.head.amount - 1 + errorAmount, + numOfOutputs = splitCoef + ) + (boxes.tail, transactionsL :+ tx) + }._2.filter(tx => state.validate(tx).isRight) ++ Seq(coinbaseTransaction(prevBlock.header.height + 1)) + logger.info(s"Number of generated transactions: ${transactions.size}.") + val header = Header( + 1.toByte, + prevBlock.id, + Payload.rootHash(transactions.map(_.id)), + timestamp, + prevBlock.header.height + 1, + R.nextLong(), + Difficulty @@ (BigInt(1) + addDiff), + EquihashSolution(Seq(1, 3)) + ) + Block(header, Payload(header.id, transactions)) + } + def generateNextBlockValidForHistory(history: History, difficultyDiff: BigInt = 0, prevBlock: Option[Block], @@ -234,7 +324,7 @@ object ChainUtils extends Settings with StrictLogging { def coinbaseTransaction(height: Int): Transaction = TransactionFactory.coinbaseTransactionScratch( privKey.publicImage, System.currentTimeMillis(), - supply = 10000000L, + supply = settings.constants.InitialEmissionAmount, amount = 1L, height = Height @@ height ) @@ -420,8 +510,8 @@ object ChainUtils extends Settings with StrictLogging { val ntp: NetworkTimeProvider = new NetworkTimeProvider(settings.ntp) new History { - override val historyStorage: HistoryStorage = storage - override val timeProvider: NetworkTimeProvider = ntp + override val historyStorage: HistoryStorage = storage + override val timeProvider: NetworkTimeProvider = ntp } } diff --git a/src/main/scala/encry/view/actors/HistoryApplicator.scala b/src/main/scala/encry/view/actors/HistoryApplicator.scala index cea8d4b6ee..e6a157a3cc 100644 --- a/src/main/scala/encry/view/actors/HistoryApplicator.scala +++ b/src/main/scala/encry/view/actors/HistoryApplicator.scala @@ -72,6 +72,7 @@ class HistoryApplicator(val history: History, history.append(modifier) match { case Left(ex) => currentNumberOfAppliedModifiers -= 1 + println("unsuccessfully applied to history") logger.info(s"Modifier ${modifier.encodedId} unsuccessfully applied to history with exception ${ex.getMessage}." + s" Current currentNumberOfAppliedModifiers $currentNumberOfAppliedModifiers.") context.system.eventStream.publish(SyntacticallyFailedModification(modifier, List(HistoryApplyError(ex.getMessage)))) @@ -91,13 +92,13 @@ class HistoryApplicator(val history: History, walletApplicator ! WalletNeedRollbackTo(VersionTag !@@ progressInfo.branchPoint.get) if (progressInfo.toRemove.nonEmpty) nodeViewHolder ! TransactionsForWallet(progressInfo.toRemove) - //println("history.append success") + println("history.append success") stateApplicator ! NotificationAboutNewModifier getModifierForApplying() case Right(progressInfo) => logger.info(s"Progress info is empty after appending to the state.") if (!isLocallyGenerated) requestDownloads(progressInfo) - //println("SemanticallySuccessfulModifier") + println("SemanticallySuccessfulModifier") context.system.eventStream.publish(SemanticallySuccessfulModifier(modifier)) currentNumberOfAppliedModifiers -= 1 getModifierForApplying() @@ -116,7 +117,7 @@ class HistoryApplicator(val history: History, if (history.isFullChainSynced) { logger.info(s"BlockChain is synced on state applicator at height ${history.getBestHeaderHeight}!") ModifiersCache.setChainSynced() - //println("FullBlockChainIsSynced") + println("FullBlockChainIsSynced") context.system.eventStream.publish(FullBlockChainIsSynced()) } currentNumberOfAppliedModifiers -= 1 @@ -130,6 +131,7 @@ class HistoryApplicator(val history: History, getModifierForApplying() case NeedToReportAsInValid(block) => + println(s"NeedToReportAsInValid") logger.info(s"History got message NeedToReportAsInValid for block ${block.encodedId}.") currentNumberOfAppliedModifiers -= 1 val (_, newProgressInfo: ProgressInfo) = history.reportModifierIsInvalid(block) @@ -139,8 +141,8 @@ class HistoryApplicator(val history: History, } def getModifierForApplying(): Unit = { - println(s"currentNumberOfAppliedModifiers: $currentNumberOfAppliedModifiers") - println(s"modifiersQueue.size: ${modifiersQueue.size}") + //println(s"currentNumberOfAppliedModifiers: $currentNumberOfAppliedModifiers") + //println(s"modifiersQueue.size: ${modifiersQueue.size}") if (currentNumberOfAppliedModifiers < setting.levelDB.maxVersions) { logger.debug(s"It's possible to append new modifier to history. Trying to get new one from the cache.") if (locallyGeneratedModifiers.nonEmpty) locallyGeneratedModifiers.dequeueOption.foreach { diff --git a/src/main/scala/encry/view/actors/StateApplicator.scala b/src/main/scala/encry/view/actors/StateApplicator.scala index eb450cdf62..c7072795eb 100644 --- a/src/main/scala/encry/view/actors/StateApplicator.scala +++ b/src/main/scala/encry/view/actors/StateApplicator.scala @@ -126,6 +126,7 @@ class StateApplicator(settings: EncryAppSettings, block: Block, transactions: Seq[Transaction]): Receive = { case TransactionValidatedSuccessfully => + println("TransactionValidatedSuccessfully") transactionsValidatorsNumber -= 1 if (transactionsValidatorsNumber == 0) { val combinedStateChange = combineAll(transactions.toList.map(UtxoState.tx2StateChange)) @@ -154,6 +155,7 @@ class StateApplicator(settings: EncryAppSettings, } case TransactionValidatedFailure(tx, ex) => + println("TransactionValidatedFailure") logger.info(s"Transaction ${tx.encodedId} failed in validation by state.") context.children.foreach(_ ! Kill) context.system.eventStream.publish(SemanticallyFailedModification(block, List(StateModifierApplyError(s"$ex")))) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 89334f1409..f4c6279762 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -5,25 +5,31 @@ import java.io.File import akka.actor.ActorSystem import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} import encry.modifiers.InstanceFactory +import encry.network.DeliveryManager.FullBlockChainIsSynced +import encry.network.DeliveryManagerTests.DMUtils.generateNextBlock import encry.settings.Settings import encry.storage.VersionalStorage -import encry.utils.ChainUtils._ -import encry.utils.{ChainUtils, FileHelper} +import encry.utils.ChainUtils.{coinbaseTransaction, defaultPaymentTransactionScratch, privKey, _} +import encry.utils.{ChainUtils, FileHelper, TestHelper} import encry.view.actors.HistoryApplicator.StartModifiersApplicationOnStateApplicator import encry.view.actors.NodeViewHolder.ReceivableMessages.ModifierFromRemote import encry.view.actors.{HistoryApplicator, StateApplicator} import encry.view.history.History import encry.view.state.{BoxHolder, UtxoState} import encry.view.wallet.EncryWallet +import org.encryfoundation.common.crypto.equihash.EquihashSolution import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.Block +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.modifiers.state.box.AssetBox import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{Height, ModifierId} +import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height, ModifierId} import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +import scorex.crypto.hash.Digest32 import scala.collection.Seq import scala.concurrent.duration._ +import scala.util.Random class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike @@ -42,18 +48,18 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) (0 until count).foldLeft(initialHistory, Seq.empty[Block]) { case ((prevHistory, blocks), _) => val block: Block = generateNextBlock(prevHistory) - prevHistory.append(block.header) - prevHistory.append(block.payload) + prevHistory.append(block.header) + prevHistory.append(block.payload) (prevHistory.reportModifierIsValid(block), blocks :+ block) } } def blockToModifiers(block: Block): Seq[PersistentModifier] = Seq(block.header, block.payload) - def generateChain(blockQty: Int): (UtxoState, List[Block]) = { - val numberOfInputsInOneTransaction = 1//10 - val transactionsNumberInEachBlock = 100//1000 - val initialboxCount = 10//100000 + def generateChain(blockQty: Int, genInvalidState: Boolean = false): (UtxoState, UtxoState, List[Block]) = { + val numberOfInputsInOneTransaction = 1 //10 + val transactionsNumberInEachBlock = 1 //1000 + val initialboxCount = 1 //100000 val initialBoxes: IndexedSeq[AssetBox] = (0 until initialboxCount).map(nonce => genHardcodedBox(privKey.publicImage.address.address, nonce) @@ -68,14 +74,19 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val stateGenerationResults: (List[Block], Block, UtxoState, IndexedSeq[AssetBox]) = (0 until blockQty).foldLeft(List.empty[Block], genesisBlock, state, initialBoxes) { case ((blocks, block, stateL, boxes), _) => - val nextBlockMainChain: Block = generateNextBlockForStateWithSpendingAllPreviousBoxes( - block, stateL, block.payload.txs.flatMap(_.newBoxes.map(_.asInstanceOf[AssetBox])).toIndexedSeq) - val stateN: UtxoState = stateL.applyModifier(nextBlockMainChain).right.get - (blocks :+ nextBlockMainChain, - nextBlockMainChain, stateN, boxes.drop(transactionsNumberInEachBlock * numberOfInputsInOneTransaction) + val nextBlock: Block = + if (genInvalidState) + generateNextBlockForStateWithInvalidTrans(block, stateL, + block.payload.txs.flatMap(_.newBoxes.map(_.asInstanceOf[AssetBox])).toIndexedSeq) + else + generateNextBlockSpend(block, stateL, + block.payload.txs.flatMap(_.newBoxes.map(_.asInstanceOf[AssetBox])).toIndexedSeq) + val stateN: UtxoState = stateL.applyModifier(nextBlock).right.get + (blocks :+ nextBlock, + nextBlock, stateN, boxes.drop(transactionsNumberInEachBlock * numberOfInputsInOneTransaction) ) } - (state, genesisBlock +: stateGenerationResults._1) + (state, stateGenerationResults._3, genesisBlock +: stateGenerationResults._1) } def printIds(blocks: List[Block]) = { @@ -142,7 +153,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) "HistoryApplicator" should { "check queues" in { - val (state, chain) = generateChain(1) + val (_, state, chain) = generateChain(1) val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) val modifierIds = modifiers.map(_.id) @@ -178,40 +189,60 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) "HistoryApplicator" should { "check queue for rollback limit" in { - println(s"maxVersions: ${settings.levelDB.maxVersions}") - - val (state, chain) = generateChain(120) + val (_, state, chain) = generateChain(settings.levelDB.maxVersions + 30) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( HistoryApplicator.props(initialHistory, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) - //.withDispatcher("history-applicator-dispatcher"), "historyApplicator" ) + system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) - //system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) - modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) - // println(s"modifiersQueue: ${historyApplicator.underlyingActor.modifiersQueue.size}") - // println(s"currentNumberOfAppliedModifiers: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") + historyApplicator.underlyingActor.modifiersQueue.size should be <= settings.levelDB.maxVersions + //println(s"modifiersQueue: ${historyApplicator.underlyingActor.modifiersQueue.size}") + + expectMsg(timeout, FullBlockChainIsSynced()) + } + } + + "HistoryApplicator" should { + "check rollback for invalid state" in { + + val (initState, state, chain) = generateChain(1, false) - // historyApplicator.underlyingActor.modifiersQueue shouldBe 0 - // historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 + state.applyModifier(chain(1)) - //val modifiersQueue = historyApplicator.underlyingActor.modifiersQueue - //val history = historyApplicator.underlyingActor.history + initialHistory.append(chain(0).header) + initialHistory.append(chain(0).payload) + val history = initialHistory.reportModifierIsValid(chain(0)) - //expectMsgType[StartModifiersApplicationOnStateApplicator](timeout) +// history.append(chain(1).header) +// history.append(chain(1).payload) +// val history1 = history.reportModifierIsValid(chain(1)) +// println(history.testApplicable(chain(1).header).right.get) - Thread.sleep(30000) +// val invalidState = state.applyModifier(chain(0)).right.get - //expectMsg(timeout, FullBlockChainIsSynced()) + val historyApplicator: TestActorRef[HistoryApplicator] = + TestActorRef[HistoryApplicator]( + HistoryApplicator.props(history, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) + ) + system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) + + //val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) - // assert(modifierIds.forall(history.isModifierDefined)) - // assert(modifierIds.map(Algos.encode) == modifiersQueue.map(_._1)) + chain.foreach(b => println(Algos.encode(b.header.id))) + + blockToModifiers(chain(1)).foreach(historyApplicator ! ModifierFromRemote(_)) + //modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) + + historyApplicator.underlyingActor.modifiersQueue.size should be <= settings.levelDB.maxVersions + //println(s"modifiersQueue: ${historyApplicator.underlyingActor.modifiersQueue.size}") + + expectMsg(timeout, FullBlockChainIsSynced()) //Thread.sleep(60000) } } - } From b2be0566ad88b078d5f2b7423385acd4fb35a8b7 Mon Sep 17 00:00:00 2001 From: capdev Date: Mon, 23 Sep 2019 18:04:37 +0500 Subject: [PATCH 10/39] fix gen chain --- src/main/scala/encry/utils/ChainUtils.scala | 45 +----------------- .../history/HistoryApplicatorTest.scala | 47 +++++++++---------- 2 files changed, 24 insertions(+), 68 deletions(-) diff --git a/src/main/scala/encry/utils/ChainUtils.scala b/src/main/scala/encry/utils/ChainUtils.scala index ee7e3775a5..4b13b6ab21 100644 --- a/src/main/scala/encry/utils/ChainUtils.scala +++ b/src/main/scala/encry/utils/ChainUtils.scala @@ -167,47 +167,6 @@ object ChainUtils extends Settings with StrictLogging { Block(header, Payload(header.id, transactions)) } - def generateNextBlockSpend(prevBlock: Block, - state: UtxoState, - box: Seq[AssetBox], - amount: Amount - ): Block = { - - val transactions: Seq[Transaction] = Seq( - defaultPaymentTransactionScratch( - privKey, - fee = 0L, - timestamp = System.currentTimeMillis(), - useBoxes = box.toIndexedSeq, - recipient = randomAddress, - amount = amount, - numOfOutputs = 1 - ), - defaultPaymentTransactionScratch( - privKey, - fee = 0L, - timestamp = System.currentTimeMillis(), - useBoxes = box.toIndexedSeq, - recipient = randomAddress, - amount = amount, - numOfOutputs = 1 - ), - coinbaseTransaction(prevBlock.header.height + 1) - ) - - val header = Header( - 1.toByte, - prevBlock.id, - Payload.rootHash(transactions.map(_.id)), - System.currentTimeMillis(), - prevBlock.header.height + 1, - R.nextLong(), - Difficulty @@ BigInt(1), - EquihashSolution(Seq(1, 3)) - ) - Block(header, Payload(header.id, transactions)) - } - def generateNextBlockForStateWithInvalidTrans(prevBlock: Block, state: UtxoState, box: Seq[AssetBox], @@ -217,15 +176,13 @@ object ChainUtils extends Settings with StrictLogging { val transactions: Seq[Transaction] = box.indices.foldLeft(box, Seq.empty[Transaction]) { case ((boxes, transactionsL), _) => - val errorAmount = 1000000000L - println(s"amount: ${boxes.head.amount - 1 + errorAmount}") val tx: Transaction = defaultPaymentTransactionScratch( privKey, fee = 1, timestamp, useBoxes = IndexedSeq(boxes.head), recipient = privKey.publicImage.address.address, - amount = boxes.head.amount - 1 + errorAmount, + amount = boxes.head.amount * 2, //invalid numOfOutputs = splitCoef ) (boxes.tail, transactionsL :+ tx) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index f4c6279762..18e003977b 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -56,7 +56,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) def blockToModifiers(block: Block): Seq[PersistentModifier] = Seq(block.header, block.payload) - def generateChain(blockQty: Int, genInvalidState: Boolean = false): (UtxoState, UtxoState, List[Block]) = { + def generateChain(blockQty: Int, genInvalid: Boolean = false): (UtxoState, UtxoState, List[Block]) = { val numberOfInputsInOneTransaction = 1 //10 val transactionsNumberInEachBlock = 1 //1000 val initialboxCount = 1 //100000 @@ -75,13 +75,13 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) (0 until blockQty).foldLeft(List.empty[Block], genesisBlock, state, initialBoxes) { case ((blocks, block, stateL, boxes), _) => val nextBlock: Block = - if (genInvalidState) + if (genInvalid) generateNextBlockForStateWithInvalidTrans(block, stateL, block.payload.txs.flatMap(_.newBoxes.map(_.asInstanceOf[AssetBox])).toIndexedSeq) else - generateNextBlockSpend(block, stateL, + generateNextBlockForStateWithSpendingAllPreviousBoxes(block, stateL, block.payload.txs.flatMap(_.newBoxes.map(_.asInstanceOf[AssetBox])).toIndexedSeq) - val stateN: UtxoState = stateL.applyModifier(nextBlock).right.get + val stateN: UtxoState = if (genInvalid) stateL else stateL.applyModifier(nextBlock).right.get (blocks :+ nextBlock, nextBlock, stateN, boxes.drop(transactionsNumberInEachBlock * numberOfInputsInOneTransaction) ) @@ -89,6 +89,14 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) (state, stateGenerationResults._3, genesisBlock +: stateGenerationResults._1) } + def applyBlocksToHistory(history: History, blocks: Seq[Block]): History = + blocks.foldLeft(history) { + case (prevHistory, block) => + prevHistory.append(block.header) + prevHistory.append(block.payload) + prevHistory.reportModifierIsValid(block) + } + def printIds(blocks: List[Block]) = { blocks.foreach { b => println(s"header.timestamp: ${b.header.timestamp}") @@ -189,11 +197,11 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) "HistoryApplicator" should { "check queue for rollback limit" in { - val (_, state, chain) = generateChain(settings.levelDB.maxVersions + 30) + val (initState, state, chain) = generateChain(settings.levelDB.maxVersions + 30) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( - HistoryApplicator.props(initialHistory, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) + HistoryApplicator.props(initialHistory, settings, initState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) @@ -211,20 +219,13 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) "HistoryApplicator" should { "check rollback for invalid state" in { - val (initState, state, chain) = generateChain(1, false) + val (genesisState, state, chain) = generateChain(3, genInvalid = true) - state.applyModifier(chain(1)) + val history = applyBlocksToHistory(initialHistory, chain) + println(s"history.height: ${history.getBestBlockHeight}") - initialHistory.append(chain(0).header) - initialHistory.append(chain(0).payload) - val history = initialHistory.reportModifierIsValid(chain(0)) - -// history.append(chain(1).header) -// history.append(chain(1).payload) -// val history1 = history.reportModifierIsValid(chain(1)) -// println(history.testApplicable(chain(1).header).right.get) - -// val invalidState = state.applyModifier(chain(0)).right.get + // println(history.testApplicable(chain(1).header).right.get) + // val invalidState = state.applyModifier(chain(0)).right.get val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( @@ -232,17 +233,15 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) ) system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) - //val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) - chain.foreach(b => println(Algos.encode(b.header.id))) - blockToModifiers(chain(1)).foreach(historyApplicator ! ModifierFromRemote(_)) - //modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) + //blockToModifiers(chain(1)).foreach(historyApplicator ! ModifierFromRemote(_)) + chain.flatMap(blockToModifiers).foreach(historyApplicator ! ModifierFromRemote(_)) - historyApplicator.underlyingActor.modifiersQueue.size should be <= settings.levelDB.maxVersions + //historyApplicator.underlyingActor.modifiersQueue.size should be <= settings.levelDB.maxVersions //println(s"modifiersQueue: ${historyApplicator.underlyingActor.modifiersQueue.size}") - expectMsg(timeout, FullBlockChainIsSynced()) //Thread.sleep(60000) + expectMsg(timeout, FullBlockChainIsSynced())//Thread.sleep(60000) } } } From 5b29bc87c0eb6a5457e791a3806c338e871dcd3e Mon Sep 17 00:00:00 2001 From: capdev Date: Mon, 23 Sep 2019 18:20:03 +0500 Subject: [PATCH 11/39] reafctor HistoryApplicator add locall generated block test --- .../history/HistoryApplicatorTest.scala | 61 ++++++------------- 1 file changed, 20 insertions(+), 41 deletions(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 18e003977b..67b81277d4 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -12,7 +12,7 @@ import encry.storage.VersionalStorage import encry.utils.ChainUtils.{coinbaseTransaction, defaultPaymentTransactionScratch, privKey, _} import encry.utils.{ChainUtils, FileHelper, TestHelper} import encry.view.actors.HistoryApplicator.StartModifiersApplicationOnStateApplicator -import encry.view.actors.NodeViewHolder.ReceivableMessages.ModifierFromRemote +import encry.view.actors.NodeViewHolder.ReceivableMessages.{LocallyGeneratedBlock, ModifierFromRemote} import encry.view.actors.{HistoryApplicator, StateApplicator} import encry.view.history.History import encry.view.state.{BoxHolder, UtxoState} @@ -117,46 +117,25 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val timeout: FiniteDuration = 10 seconds - //printIds(chain) - //println(s"${settings.constants.RetargetingEpochsQty}") - //println(s"${settings.constants.EpochLength}") - - // "HistoryApplicator add locall generated block" should { - // "chain synced" in { - // - // val dir = FileHelper.getRandomTempDir - // val history: History = generateDummyHistory(settings) - // val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) - // - // val bxs = TestHelper.genAssetBoxes - // val boxHolder = BoxHolder(bxs) - // val state = utxoFromBoxHolder(boxHolder, FileHelper.getRandomTempDir, settings) - // - // val nodeViewHolder = TestProbe() - // val influx = TestProbe() - // - // val historyBlocks = (0 until 10).foldLeft(history, Seq.empty[Block]) { - // case ((prevHistory, blocks), _) => - // val block: Block = generateNextBlock(prevHistory) - // prevHistory.append(block.header) - // prevHistory.append(block.payload) - // (prevHistory.reportModifierIsValid(block), blocks :+ block) - // } - // - // val block: Block = generateNextBlock(history) - // - // val historyApplicator: TestActorRef[HistoryApplicator] = - // TestActorRef[HistoryApplicator]( - // HistoryApplicator.props(history, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) - // ) - // - // system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) - // - // historyApplicator ! LocallyGeneratedBlock(block) - // - // expectMsg(timeout, FullBlockChainIsSynced()) - // } - // } + "HistoryApplicator add locall generated block" should { + "chain sync" in { + + val blockQty = 10 + + val (genesisState, state, chain) = generateChain(blockQty) + + val historyApplicator: TestActorRef[HistoryApplicator] = + TestActorRef[HistoryApplicator]( + HistoryApplicator.props(initialHistory, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) + ) + + system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) + + chain.foreach(historyApplicator ! LocallyGeneratedBlock(_)) + + (0 until (blockQty + 1) * 2).map(n => expectMsg(timeout, FullBlockChainIsSynced())) + } + } "HistoryApplicator" should { "check queues" in { From ad8903572f813cd0abacbad33901b85ce6e8d242 Mon Sep 17 00:00:00 2001 From: capdev Date: Tue, 24 Sep 2019 14:24:06 +0500 Subject: [PATCH 12/39] add checkFullBlockChainIsSynced refactoring cleanup --- .../encry/view/actors/HistoryApplicator.scala | 8 ++- .../encry/view/actors/StateApplicator.scala | 8 ++- .../history/HistoryApplicatorTest.scala | 63 +++++++++++-------- 3 files changed, 47 insertions(+), 32 deletions(-) diff --git a/src/main/scala/encry/view/actors/HistoryApplicator.scala b/src/main/scala/encry/view/actors/HistoryApplicator.scala index e6a157a3cc..68191cf612 100644 --- a/src/main/scala/encry/view/actors/HistoryApplicator.scala +++ b/src/main/scala/encry/view/actors/HistoryApplicator.scala @@ -72,9 +72,9 @@ class HistoryApplicator(val history: History, history.append(modifier) match { case Left(ex) => currentNumberOfAppliedModifiers -= 1 - println("unsuccessfully applied to history") logger.info(s"Modifier ${modifier.encodedId} unsuccessfully applied to history with exception ${ex.getMessage}." + s" Current currentNumberOfAppliedModifiers $currentNumberOfAppliedModifiers.") + println("SyntacticallyFailedModification") context.system.eventStream.publish(SyntacticallyFailedModification(modifier, List(HistoryApplyError(ex.getMessage)))) case Right(progressInfo) if progressInfo.toApply.nonEmpty => logger.info(s"Modifier ${modifier.encodedId} successfully applied to history.") @@ -92,7 +92,7 @@ class HistoryApplicator(val history: History, walletApplicator ! WalletNeedRollbackTo(VersionTag !@@ progressInfo.branchPoint.get) if (progressInfo.toRemove.nonEmpty) nodeViewHolder ! TransactionsForWallet(progressInfo.toRemove) - println("history.append success") + //println("history.append success") stateApplicator ! NotificationAboutNewModifier getModifierForApplying() case Right(progressInfo) => @@ -131,11 +131,12 @@ class HistoryApplicator(val history: History, getModifierForApplying() case NeedToReportAsInValid(block) => - println(s"NeedToReportAsInValid") + //println(s"NeedToReportAsInValid") logger.info(s"History got message NeedToReportAsInValid for block ${block.encodedId}.") currentNumberOfAppliedModifiers -= 1 val (_, newProgressInfo: ProgressInfo) = history.reportModifierIsInvalid(block) sender() ! NewProgressInfoAfterMarkingAsInValid(newProgressInfo) + //println(s"ha history.height: ${history.getBestBlockHeight}") case nonsense => logger.info(s"History applicator actor got from $sender message $nonsense.") } @@ -167,6 +168,7 @@ class HistoryApplicator(val history: History, def requestDownloads(pi: ProgressInfo): Unit = pi.toDownload.foreach { case (tid, id) => if (tid != Transaction.modifierTypeId) logger.debug(s"HistoryApplicator call requestDownloads for modifier ${Algos.encode(id)} of type $tid") + println("DownloadRequest") context.system.eventStream.publish(DownloadRequest(tid, id)) } diff --git a/src/main/scala/encry/view/actors/StateApplicator.scala b/src/main/scala/encry/view/actors/StateApplicator.scala index c7072795eb..ccc48d313d 100644 --- a/src/main/scala/encry/view/actors/StateApplicator.scala +++ b/src/main/scala/encry/view/actors/StateApplicator.scala @@ -47,6 +47,7 @@ class StateApplicator(settings: EncryAppSettings, case header: Header => state.lastBlockTimestamp = header.timestamp historyApplicator ! NeedToReportAsValid(header) + println("stateApplicator.SemanticallySuccessfulModifier") context.system.eventStream.publish(SemanticallySuccessfulModifier(header)) val newToApply: List[PersistentModifier] = toApply.drop(1) if (newToApply.nonEmpty) { @@ -126,7 +127,6 @@ class StateApplicator(settings: EncryAppSettings, block: Block, transactions: Seq[Transaction]): Receive = { case TransactionValidatedSuccessfully => - println("TransactionValidatedSuccessfully") transactionsValidatorsNumber -= 1 if (transactionsValidatorsNumber == 0) { val combinedStateChange = combineAll(transactions.toList.map(UtxoState.tx2StateChange)) @@ -135,6 +135,7 @@ class StateApplicator(settings: EncryAppSettings, combinedStateChange.outputsToDb.toList, combinedStateChange.inputsToDb.toList ) + println("stateApplicator.SemanticallySuccessfulModifier") context.system.eventStream.publish(SemanticallySuccessfulModifier(block)) historyApplicator ! NeedToReportAsValid(block) influxRef.foreach { ref => @@ -155,9 +156,9 @@ class StateApplicator(settings: EncryAppSettings, } case TransactionValidatedFailure(tx, ex) => - println("TransactionValidatedFailure") logger.info(s"Transaction ${tx.encodedId} failed in validation by state.") context.children.foreach(_ ! Kill) + println("stateApplicator.SemanticallyFailedModification") context.system.eventStream.publish(SemanticallyFailedModification(block, List(StateModifierApplyError(s"$ex")))) historyApplicator ! NeedToReportAsInValid(block) context.become(awaitingNewProgressInfo(block, ui, toApply)) @@ -200,10 +201,12 @@ class StateApplicator(settings: EncryAppSettings, else Success(state) -> suffixApplied stateToApplyTry match { case Failure(exception) => + println("stateApplicator.RollbackFailed") context.system.eventStream.publish(RollbackFailed(branchPointOpt)) EncryApp.forceStopApplication(500, s"Rollback failed: $exception.") case Success(stateToApply) => logger.info(s"Successfully applied to the state. Starting modifiers applying.") + println("stateApplicator.RollbackSucceed") context.system.eventStream.publish(RollbackSucceed(branchPointOpt)) self ! StartModifiersApplying context.become(modifierApplication( @@ -225,6 +228,7 @@ class StateApplicator(settings: EncryAppSettings, def requestDownloads(pi: ProgressInfo): Unit = pi.toDownload.foreach { case (tid, id) => if (tid != Transaction.modifierTypeId) logger.debug(s"StateApplicator call requestDownloads for modifier ${Algos.encode(id)} of type $tid") + println("stateApplicator.DownloadRequest") context.system.eventStream.publish(DownloadRequest(tid, id)) } } diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 67b81277d4..b0f493090d 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -7,11 +7,13 @@ import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} import encry.modifiers.InstanceFactory import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.DeliveryManagerTests.DMUtils.generateNextBlock +import encry.network.NodeViewSynchronizer.ReceivableMessages.{RollbackSucceed, SemanticallySuccessfulModifier} import encry.settings.Settings import encry.storage.VersionalStorage import encry.utils.ChainUtils.{coinbaseTransaction, defaultPaymentTransactionScratch, privKey, _} import encry.utils.{ChainUtils, FileHelper, TestHelper} import encry.view.actors.HistoryApplicator.StartModifiersApplicationOnStateApplicator +import encry.view.actors.NodeViewHolder.DownloadRequest import encry.view.actors.NodeViewHolder.ReceivableMessages.{LocallyGeneratedBlock, ModifierFromRemote} import encry.view.actors.{HistoryApplicator, StateApplicator} import encry.view.history.History @@ -56,7 +58,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) def blockToModifiers(block: Block): Seq[PersistentModifier] = Seq(block.header, block.payload) - def generateChain(blockQty: Int, genInvalid: Boolean = false): (UtxoState, UtxoState, List[Block]) = { + def generateChain(blockQty: Int, genInvalidBlockFrom: Option[Int] = None): (UtxoState, UtxoState, List[Block]) = { val numberOfInputsInOneTransaction = 1 //10 val transactionsNumberInEachBlock = 1 //1000 val initialboxCount = 1 //100000 @@ -73,7 +75,8 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val stateGenerationResults: (List[Block], Block, UtxoState, IndexedSeq[AssetBox]) = (0 until blockQty).foldLeft(List.empty[Block], genesisBlock, state, initialBoxes) { - case ((blocks, block, stateL, boxes), _) => + case ((blocks, block, stateL, boxes), height) => + val genInvalid = genInvalidBlockFrom.exists(height + 1 >= _) val nextBlock: Block = if (genInvalid) generateNextBlockForStateWithInvalidTrans(block, stateL, @@ -107,11 +110,11 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) } } + def checkFullBlockChainIsSynced(qty: Int): Unit = (0 until qty).foreach(_ => expectMsg(timeout, FullBlockChainIsSynced())) + val dir: File = FileHelper.getRandomTempDir val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) - val initialHistory: History = generateDummyHistory(settings) - val nodeViewHolder = TestProbe() val influx = TestProbe() @@ -122,6 +125,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val blockQty = 10 + val initialHistory: History = generateDummyHistory(settings) val (genesisState, state, chain) = generateChain(blockQty) val historyApplicator: TestActorRef[HistoryApplicator] = @@ -133,21 +137,22 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) chain.foreach(historyApplicator ! LocallyGeneratedBlock(_)) - (0 until (blockQty + 1) * 2).map(n => expectMsg(timeout, FullBlockChainIsSynced())) + checkFullBlockChainIsSynced((blockQty + 1) * 2) } } "HistoryApplicator" should { "check queues" in { - val (_, state, chain) = generateChain(1) + val initialHistory: History = generateDummyHistory(settings) + val (genesisState, state, chain) = generateChain(1) val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) val modifierIds = modifiers.map(_.id) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( - HistoryApplicator.props(initialHistory, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) + HistoryApplicator.props(initialHistory, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) @@ -176,51 +181,55 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) "HistoryApplicator" should { "check queue for rollback limit" in { - val (initState, state, chain) = generateChain(settings.levelDB.maxVersions + 30) + val overQty = 30 + + val history: History = generateDummyHistory(settings) + val (genesisState, state, chain) = generateChain(settings.levelDB.maxVersions + overQty) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( - HistoryApplicator.props(initialHistory, settings, initState, wallet, nodeViewHolder.ref, Some(influx.ref)) + HistoryApplicator.props(history, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) - system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) + system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) - val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) - - modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) + chain + .flatMap(blockToModifiers) + .foreach(historyApplicator ! ModifierFromRemote(_)) + println(s"modifiersQueue.size: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") historyApplicator.underlyingActor.modifiersQueue.size should be <= settings.levelDB.maxVersions - //println(s"modifiersQueue: ${historyApplicator.underlyingActor.modifiersQueue.size}") - expectMsg(timeout, FullBlockChainIsSynced()) + receiveN((settings.levelDB.maxVersions + overQty) * 2, 30 seconds) + + history.getBestBlockHeight shouldBe settings.levelDB.maxVersions + overQty } } "HistoryApplicator" should { "check rollback for invalid state" in { - val (genesisState, state, chain) = generateChain(3, genInvalid = true) - - val history = applyBlocksToHistory(initialHistory, chain) - println(s"history.height: ${history.getBestBlockHeight}") + val history: History = generateDummyHistory(settings) - // println(history.testApplicable(chain(1).header).right.get) - // val invalidState = state.applyModifier(chain(0)).right.get + //generate invalid starting from 2 blocks + val (genesisState, state, chain) = generateChain(3, Some(2)) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( - HistoryApplicator.props(history, settings, state, wallet, nodeViewHolder.ref, Some(influx.ref)) + HistoryApplicator.props(history, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) chain.foreach(b => println(Algos.encode(b.header.id))) - //blockToModifiers(chain(1)).foreach(historyApplicator ! ModifierFromRemote(_)) - chain.flatMap(blockToModifiers).foreach(historyApplicator ! ModifierFromRemote(_)) + chain + .flatMap(blockToModifiers) + .foreach(historyApplicator ! ModifierFromRemote(_)) - //historyApplicator.underlyingActor.modifiersQueue.size should be <= settings.levelDB.maxVersions - //println(s"modifiersQueue: ${historyApplicator.underlyingActor.modifiersQueue.size}") + checkFullBlockChainIsSynced(3 * 2) - expectMsg(timeout, FullBlockChainIsSynced())//Thread.sleep(60000) + history.getBestBlockHeight shouldBe 1 + history.getBestBlock.map(b => Algos.encode(b.id)) shouldBe Some(Algos.encode(chain(1).id)) + history.getBestBlockHeightDB shouldBe 1 } } } From 80fcbbe865f5fe8aac67aecdb2137db5fe692059 Mon Sep 17 00:00:00 2001 From: capdev Date: Tue, 24 Sep 2019 17:05:33 +0500 Subject: [PATCH 13/39] some refactoring --- .../encry/view/actors/HistoryApplicator.scala | 4 +- .../history/HistoryApplicatorTest.scala | 98 +++++++++---------- 2 files changed, 50 insertions(+), 52 deletions(-) diff --git a/src/main/scala/encry/view/actors/HistoryApplicator.scala b/src/main/scala/encry/view/actors/HistoryApplicator.scala index 68191cf612..d08eeef480 100644 --- a/src/main/scala/encry/view/actors/HistoryApplicator.scala +++ b/src/main/scala/encry/view/actors/HistoryApplicator.scala @@ -79,7 +79,7 @@ class HistoryApplicator(val history: History, case Right(progressInfo) if progressInfo.toApply.nonEmpty => logger.info(s"Modifier ${modifier.encodedId} successfully applied to history.") modifiersQueue = modifiersQueue.enqueue(modifier.encodedId -> progressInfo) - //println(s"modifiersQueue.enqueue ${modifiersQueue.size}") + println(s"modifiersQueue.enqueue ${modifiersQueue.size}") logger.info(s"New element put into queue. Current queue size is ${modifiersQueue.length}." + s"Current number of applied modifiers is $currentNumberOfAppliedModifiers.") influxRef.foreach(ref => @@ -111,7 +111,7 @@ class HistoryApplicator(val history: History, sender() ! StartModifiersApplicationOnStateApplicator(pi, IndexedSeq.empty[PersistentModifier]) modifiersQueue = newQueue } - //println(s"modifiersQueue.dequeue ${modifiersQueue.size}") + println(s"modifiersQueue.dequeue ${modifiersQueue.size}") case NotificationAboutSuccessfullyAppliedModifier => if (history.isFullChainSynced) { diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index b0f493090d..155581d320 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -6,32 +6,25 @@ import akka.actor.ActorSystem import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} import encry.modifiers.InstanceFactory import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.DeliveryManagerTests.DMUtils.generateNextBlock -import encry.network.NodeViewSynchronizer.ReceivableMessages.{RollbackSucceed, SemanticallySuccessfulModifier} +import encry.network.NodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier import encry.settings.Settings import encry.storage.VersionalStorage -import encry.utils.ChainUtils.{coinbaseTransaction, defaultPaymentTransactionScratch, privKey, _} -import encry.utils.{ChainUtils, FileHelper, TestHelper} -import encry.view.actors.HistoryApplicator.StartModifiersApplicationOnStateApplicator -import encry.view.actors.NodeViewHolder.DownloadRequest +import encry.utils.ChainUtils._ +import encry.utils.{ChainUtils, FileHelper} +import encry.view.actors.HistoryApplicator import encry.view.actors.NodeViewHolder.ReceivableMessages.{LocallyGeneratedBlock, ModifierFromRemote} -import encry.view.actors.{HistoryApplicator, StateApplicator} import encry.view.history.History import encry.view.state.{BoxHolder, UtxoState} import encry.view.wallet.EncryWallet -import org.encryfoundation.common.crypto.equihash.EquihashSolution import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.state.box.AssetBox import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height, ModifierId} +import org.encryfoundation.common.utils.TaggedTypes.Height import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -import scorex.crypto.hash.Digest32 import scala.collection.Seq import scala.concurrent.duration._ -import scala.util.Random class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike @@ -89,6 +82,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) nextBlock, stateN, boxes.drop(transactionsNumberInEachBlock * numberOfInputsInOneTransaction) ) } + //(genesisBlock +: stateGenerationResults._1).foreach(b => println(s"block.header : ${Algos.encode(b.header.id)}")) (state, stateGenerationResults._3, genesisBlock +: stateGenerationResults._1) } @@ -120,66 +114,74 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val timeout: FiniteDuration = 10 seconds - "HistoryApplicator add locall generated block" should { - "chain sync" in { + "HistoryApplicator" should { - val blockQty = 10 + "apply locall blocks and chain sync" in { - val initialHistory: History = generateDummyHistory(settings) - val (genesisState, state, chain) = generateChain(blockQty) + val blockQty = 10 - val historyApplicator: TestActorRef[HistoryApplicator] = - TestActorRef[HistoryApplicator]( - HistoryApplicator.props(initialHistory, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) - ) + val history: History = generateDummyHistory(settings) + val (genesisState, state, chain) = generateChain(blockQty) + + val historyApplicator: TestActorRef[HistoryApplicator] = + TestActorRef[HistoryApplicator]( + HistoryApplicator.props(history, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) + ) - system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) + system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) - chain.foreach(historyApplicator ! LocallyGeneratedBlock(_)) + chain.foreach(historyApplicator ! LocallyGeneratedBlock(_)) - checkFullBlockChainIsSynced((blockQty + 1) * 2) - } + checkFullBlockChainIsSynced((blockQty + 1) * 2) + history.getBestBlockHeight shouldBe blockQty } - "HistoryApplicator" should { "check queues" in { val initialHistory: History = generateDummyHistory(settings) val (genesisState, state, chain) = generateChain(1) val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) - val modifierIds = modifiers.map(_.id) + //val modifierIds = modifiers.map(_.id) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( HistoryApplicator.props(initialHistory, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) + system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) + modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) println(s"modifiersQueue: ${historyApplicator.underlyingActor.modifiersQueue.size}") println(s"currentNumberOfAppliedModifiers: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") - val modifiersQueue = historyApplicator.underlyingActor.modifiersQueue - modifiersQueue.size shouldBe 2 - historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 - - historyApplicator ! StateApplicator.RequestNextModifier - - historyApplicator.underlyingActor.modifiersQueue shouldBe 0 - historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 - - val history = historyApplicator.underlyingActor.history - - expectMsgType[StartModifiersApplicationOnStateApplicator](timeout) - - assert(modifierIds.forall(history.isModifierDefined)) - assert(modifierIds.map(Algos.encode) == modifiersQueue.map(_._1)) +// val modifiersQueue = historyApplicator.underlyingActor.modifiersQueue +// modifiersQueue.size shouldBe 2 +// historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 + + //Thread.sleep(15000) + receiveN(4, 10 seconds).forall { case _: SemanticallySuccessfulModifier => true } + //historyApplicator ! StateApplicator.RequestNextModifier +// +// val msg = receiveWhile(timeout) { +// case msg: StartModifiersApplicationOnStateApplicator => msg.progressInfo.toApply.size +// case msg: Any => println(msg) +// } +// println(s"msg: ${msg}") + + // historyApplicator.underlyingActor.modifiersQueue shouldBe 0 + // historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 + // + // val history = historyApplicator.underlyingActor.history + // + // expectMsgType[StartModifiersApplicationOnStateApplicator](timeout) + // + // assert(modifierIds.forall(history.isModifierDefined)) + // assert(modifierIds.map(Algos.encode) == modifiersQueue.map(_._1)) } - } - "HistoryApplicator" should { - "check queue for rollback limit" in { + "check queue for rollback height" in { val overQty = 30 @@ -203,9 +205,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) history.getBestBlockHeight shouldBe settings.levelDB.maxVersions + overQty } - } - "HistoryApplicator" should { "check rollback for invalid state" in { val history: History = generateDummyHistory(settings) @@ -219,8 +219,6 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) ) system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) - chain.foreach(b => println(Algos.encode(b.header.id))) - chain .flatMap(blockToModifiers) .foreach(historyApplicator ! ModifierFromRemote(_)) From 16132867b453a5083963fff9ba56260c4295be51 Mon Sep 17 00:00:00 2001 From: capdev Date: Tue, 24 Sep 2019 17:05:33 +0500 Subject: [PATCH 14/39] some refactoring --- .../encry/view/actors/HistoryApplicator.scala | 4 +- .../history/HistoryApplicatorTest.scala | 105 +++++++++--------- 2 files changed, 54 insertions(+), 55 deletions(-) diff --git a/src/main/scala/encry/view/actors/HistoryApplicator.scala b/src/main/scala/encry/view/actors/HistoryApplicator.scala index 68191cf612..d08eeef480 100644 --- a/src/main/scala/encry/view/actors/HistoryApplicator.scala +++ b/src/main/scala/encry/view/actors/HistoryApplicator.scala @@ -79,7 +79,7 @@ class HistoryApplicator(val history: History, case Right(progressInfo) if progressInfo.toApply.nonEmpty => logger.info(s"Modifier ${modifier.encodedId} successfully applied to history.") modifiersQueue = modifiersQueue.enqueue(modifier.encodedId -> progressInfo) - //println(s"modifiersQueue.enqueue ${modifiersQueue.size}") + println(s"modifiersQueue.enqueue ${modifiersQueue.size}") logger.info(s"New element put into queue. Current queue size is ${modifiersQueue.length}." + s"Current number of applied modifiers is $currentNumberOfAppliedModifiers.") influxRef.foreach(ref => @@ -111,7 +111,7 @@ class HistoryApplicator(val history: History, sender() ! StartModifiersApplicationOnStateApplicator(pi, IndexedSeq.empty[PersistentModifier]) modifiersQueue = newQueue } - //println(s"modifiersQueue.dequeue ${modifiersQueue.size}") + println(s"modifiersQueue.dequeue ${modifiersQueue.size}") case NotificationAboutSuccessfullyAppliedModifier => if (history.isFullChainSynced) { diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index b0f493090d..cb9124e375 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -6,32 +6,25 @@ import akka.actor.ActorSystem import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} import encry.modifiers.InstanceFactory import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.DeliveryManagerTests.DMUtils.generateNextBlock -import encry.network.NodeViewSynchronizer.ReceivableMessages.{RollbackSucceed, SemanticallySuccessfulModifier} +import encry.network.NodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier import encry.settings.Settings import encry.storage.VersionalStorage -import encry.utils.ChainUtils.{coinbaseTransaction, defaultPaymentTransactionScratch, privKey, _} -import encry.utils.{ChainUtils, FileHelper, TestHelper} -import encry.view.actors.HistoryApplicator.StartModifiersApplicationOnStateApplicator -import encry.view.actors.NodeViewHolder.DownloadRequest +import encry.utils.ChainUtils._ +import encry.utils.{ChainUtils, FileHelper} +import encry.view.actors.HistoryApplicator import encry.view.actors.NodeViewHolder.ReceivableMessages.{LocallyGeneratedBlock, ModifierFromRemote} -import encry.view.actors.{HistoryApplicator, StateApplicator} import encry.view.history.History import encry.view.state.{BoxHolder, UtxoState} import encry.view.wallet.EncryWallet -import org.encryfoundation.common.crypto.equihash.EquihashSolution import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.state.box.AssetBox import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height, ModifierId} +import org.encryfoundation.common.utils.TaggedTypes.Height import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -import scorex.crypto.hash.Digest32 import scala.collection.Seq import scala.concurrent.duration._ -import scala.util.Random class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike @@ -89,6 +82,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) nextBlock, stateN, boxes.drop(transactionsNumberInEachBlock * numberOfInputsInOneTransaction) ) } + //(genesisBlock +: stateGenerationResults._1).foreach(b => println(s"block.header : ${Algos.encode(b.header.id)}")) (state, stateGenerationResults._3, genesisBlock +: stateGenerationResults._1) } @@ -120,66 +114,75 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val timeout: FiniteDuration = 10 seconds - "HistoryApplicator add locall generated block" should { - "chain sync" in { + "HistoryApplicator" should { - val blockQty = 10 + "apply locall blocks and chain sync" in { - val initialHistory: History = generateDummyHistory(settings) - val (genesisState, state, chain) = generateChain(blockQty) + val blockQty = 10 - val historyApplicator: TestActorRef[HistoryApplicator] = - TestActorRef[HistoryApplicator]( - HistoryApplicator.props(initialHistory, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) - ) + val history: History = generateDummyHistory(settings) + val (genesisState, state, chain) = generateChain(blockQty) + + val historyApplicator: TestActorRef[HistoryApplicator] = + TestActorRef[HistoryApplicator]( + HistoryApplicator.props(history, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) + ) - system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) + system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) - chain.foreach(historyApplicator ! LocallyGeneratedBlock(_)) + chain.foreach(historyApplicator ! LocallyGeneratedBlock(_)) - checkFullBlockChainIsSynced((blockQty + 1) * 2) - } + checkFullBlockChainIsSynced((blockQty + 1) * 2) + history.getBestBlockHeight shouldBe blockQty } - "HistoryApplicator" should { "check queues" in { val initialHistory: History = generateDummyHistory(settings) val (genesisState, state, chain) = generateChain(1) val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) - val modifierIds = modifiers.map(_.id) + //val modifierIds = modifiers.map(_.id) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( HistoryApplicator.props(initialHistory, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) - modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) - - println(s"modifiersQueue: ${historyApplicator.underlyingActor.modifiersQueue.size}") - println(s"currentNumberOfAppliedModifiers: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") - - val modifiersQueue = historyApplicator.underlyingActor.modifiersQueue - modifiersQueue.size shouldBe 2 - historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 - - historyApplicator ! StateApplicator.RequestNextModifier - - historyApplicator.underlyingActor.modifiersQueue shouldBe 0 - historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 - - val history = historyApplicator.underlyingActor.history + system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) - expectMsgType[StartModifiersApplicationOnStateApplicator](timeout) + modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) - assert(modifierIds.forall(history.isModifierDefined)) - assert(modifierIds.map(Algos.encode) == modifiersQueue.map(_._1)) + println(s"test.modifiersQueue: ${historyApplicator.underlyingActor.modifiersQueue.size}") + println(s"test.currentNumberOfAppliedModifiers: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") + + receiveN(4, timeout).forall { case _: SemanticallySuccessfulModifier => true } + +// val modifiersQueue = historyApplicator.underlyingActor.modifiersQueue +// modifiersQueue.size shouldBe 2 +// historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 + + //Thread.sleep(15000) + //historyApplicator ! StateApplicator.RequestNextModifier +// +// val msg = receiveWhile(timeout) { +// case msg: StartModifiersApplicationOnStateApplicator => msg.progressInfo.toApply.size +// case msg: Any => println(msg) +// } +// println(s"msg: ${msg}") + + // historyApplicator.underlyingActor.modifiersQueue shouldBe 0 + // historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 + // + // val history = historyApplicator.underlyingActor.history + // + // expectMsgType[StartModifiersApplicationOnStateApplicator](timeout) + // + // assert(modifierIds.forall(history.isModifierDefined)) + // assert(modifierIds.map(Algos.encode) == modifiersQueue.map(_._1)) } - } - "HistoryApplicator" should { - "check queue for rollback limit" in { + "check queue for rollback height" in { val overQty = 30 @@ -203,9 +206,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) history.getBestBlockHeight shouldBe settings.levelDB.maxVersions + overQty } - } - "HistoryApplicator" should { "check rollback for invalid state" in { val history: History = generateDummyHistory(settings) @@ -219,8 +220,6 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) ) system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) - chain.foreach(b => println(Algos.encode(b.header.id))) - chain .flatMap(blockToModifiers) .foreach(historyApplicator ! ModifierFromRemote(_)) From 0c27d2d8cd61d7fc9ea7b66d94119a3586eb58fb Mon Sep 17 00:00:00 2001 From: capdev Date: Wed, 25 Sep 2019 12:14:51 +0500 Subject: [PATCH 15/39] draft check rollback on fat blocks test --- src/main/scala/encry/utils/ChainUtils.scala | 7 +- .../history/HistoryApplicatorTest.scala | 98 +++++++++++++++++-- 2 files changed, 92 insertions(+), 13 deletions(-) diff --git a/src/main/scala/encry/utils/ChainUtils.scala b/src/main/scala/encry/utils/ChainUtils.scala index 4b13b6ab21..beec64b427 100644 --- a/src/main/scala/encry/utils/ChainUtils.scala +++ b/src/main/scala/encry/utils/ChainUtils.scala @@ -73,12 +73,10 @@ object ChainUtils extends Settings with StrictLogging { } def generateGenesisBlock(genesisHeight: Height): Block = { - val supplyTotal: Amount = settings.constants.InitialEmissionAmount - val coinbaseTrans = TransactionFactory.coinbaseTransactionScratch( privKey.publicImage, System.currentTimeMillis(), - supplyTotal, + settings.constants.InitialEmissionAmount, amount = 0L, height = genesisHeight ) @@ -154,6 +152,9 @@ object ChainUtils extends Settings with StrictLogging { (boxes.tail, transactionsL :+ tx) }._2.filter(tx => state.validate(tx).isRight) ++ Seq(coinbaseTransaction(prevBlock.header.height + 1)) logger.info(s"Number of generated transactions: ${transactions.size}.") + + println(s"transactions.size: ${transactions.size}") + val header = Header( 1.toByte, prevBlock.id, diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index cb9124e375..c5474780cb 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -5,24 +5,31 @@ import java.io.File import akka.actor.ActorSystem import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} import encry.modifiers.InstanceFactory +import encry.modifiers.mempool.TransactionFactory import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier import encry.settings.Settings import encry.storage.VersionalStorage -import encry.utils.ChainUtils._ +import encry.utils.ChainUtils.{coinbaseTransaction, defaultPaymentTransactionScratch, privKey, _} import encry.utils.{ChainUtils, FileHelper} import encry.view.actors.HistoryApplicator import encry.view.actors.NodeViewHolder.ReceivableMessages.{LocallyGeneratedBlock, ModifierFromRemote} import encry.view.history.History import encry.view.state.{BoxHolder, UtxoState} import encry.view.wallet.EncryWallet +import org.encryfoundation.common.crypto.equihash.EquihashSolution import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.Block -import org.encryfoundation.common.modifiers.state.box.AssetBox +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.mempool.transaction.EncryAddress.Address +import org.encryfoundation.common.modifiers.mempool.transaction.{Pay2PubKeyAddress, Transaction} +import org.encryfoundation.common.modifiers.state.box.{AssetBox, EncryProposition} import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.Height +import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height} import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +import scorex.crypto.signatures.PublicKey +import scorex.utils.{Random => ScorexRandom} +import scala.util.Random import scala.collection.Seq import scala.concurrent.duration._ @@ -51,10 +58,39 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) def blockToModifiers(block: Block): Seq[PersistentModifier] = Seq(block.header, block.payload) + def rndAddress: Address = Pay2PubKeyAddress(PublicKey @@ ScorexRandom.randomBytes()).address + + def generateNextBlockForState(prevBlock: Block, state: UtxoState, box: Seq[AssetBox], splitCoef: Int = 2, + addDiff: Difficulty = Difficulty @@ BigInt(0)): Block = { + val timestamp = System.currentTimeMillis() + + val transactions: Seq[Transaction] = box.indices.foldLeft(box, Seq.empty[Transaction]) { + case ((boxes, transactionsL), _) => + val payTx: Transaction = TransactionFactory.defaultPaymentTransactionScratch(privKey, 0L, timestamp, IndexedSeq(boxes.head), + rndAddress, 1) + + val backTx: Transaction = TransactionFactory.defaultPaymentTransactionScratch(privKey, 0L, timestamp, IndexedSeq(boxes.head), + privKey.publicImage.address.address, boxes.head.amount - 1) + + (boxes.tail, transactionsL :+ payTx :+ backTx) + }._2.filter(tx => state.validate(tx).isRight) ++ Seq(ChainUtils.coinbaseTransaction(prevBlock.header.height + 1)) + + println(s"transactions.size: ${transactions.size}") + + val header = Header(1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), timestamp, + prevBlock.header.height + 1, Random.nextLong(), Difficulty @@ (BigInt(1) + addDiff), EquihashSolution(Seq(1, 3))) + + Block(header, Payload(header.id, transactions)) + } + + def genHardcodedBox(address: Address, nonce: Long): AssetBox = + AssetBox(EncryProposition.addressLocked(address), nonce, 10L) + def generateChain(blockQty: Int, genInvalidBlockFrom: Option[Int] = None): (UtxoState, UtxoState, List[Block]) = { val numberOfInputsInOneTransaction = 1 //10 - val transactionsNumberInEachBlock = 1 //1000 - val initialboxCount = 1 //100000 + val numberOfOutputsInOneTransaction = 1 //10 + val transactionsNumberInEachBlock = 1000 //1000 + val initialboxCount = 100000 //100000 val initialBoxes: IndexedSeq[AssetBox] = (0 until initialboxCount).map(nonce => genHardcodedBox(privKey.publicImage.address.address, nonce) @@ -66,6 +102,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val state: UtxoState = utxoFromBoxHolder(boxesHolder, dir, None, settings, VersionalStorage.LevelDB) .applyModifier(genesisBlock).right.get + //1000000000 val stateGenerationResults: (List[Block], Block, UtxoState, IndexedSeq[AssetBox]) = (0 until blockQty).foldLeft(List.empty[Block], genesisBlock, state, initialBoxes) { case ((blocks, block, stateL, boxes), height) => @@ -75,15 +112,30 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) generateNextBlockForStateWithInvalidTrans(block, stateL, block.payload.txs.flatMap(_.newBoxes.map(_.asInstanceOf[AssetBox])).toIndexedSeq) else - generateNextBlockForStateWithSpendingAllPreviousBoxes(block, stateL, - block.payload.txs.flatMap(_.newBoxes.map(_.asInstanceOf[AssetBox])).toIndexedSeq) + generateNextBlockForState(block, stateL, + //block.payload.txs.flatMap(_.newBoxes.map(_.asInstanceOf[AssetBox])).toIndexedSeq) + boxes.take(transactionsNumberInEachBlock * numberOfInputsInOneTransaction)) + val stateN: UtxoState = if (genInvalid) stateL else stateL.applyModifier(nextBlock).right.get (blocks :+ nextBlock, nextBlock, stateN, boxes.drop(transactionsNumberInEachBlock * numberOfInputsInOneTransaction) ) } +// val stateGenerationResults: (Vector[Block], Block, UtxoState, IndexedSeq[AssetBox]) = +// (0 until blockQty).foldLeft(Vector[Block](), genesisBlock, state, initialBoxes) { +// case ((blocks, block, stateL, boxes), _) => +// val nextBlock: Block = generateNextBlockValidForState( +// block, stateL, boxes.take(transactionsNumberInEachBlock * numberOfInputsInOneTransaction), +// transactionsNumberInEachBlock, numberOfInputsInOneTransaction, numberOfOutputsInOneTransaction +// ) +// val stateN: UtxoState = stateL.applyModifier(nextBlock).right.get +// (blocks :+ nextBlock, nextBlock, stateN, boxes.drop(transactionsNumberInEachBlock * numberOfInputsInOneTransaction) +// ) +// } + //(genesisBlock +: stateGenerationResults._1).foreach(b => println(s"block.header : ${Algos.encode(b.header.id)}")) - (state, stateGenerationResults._3, genesisBlock +: stateGenerationResults._1) + + (state, stateGenerationResults._3, genesisBlock +: stateGenerationResults._1.toList) } def applyBlocksToHistory(history: History, blocks: Seq[Block]): History = @@ -211,7 +263,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val history: History = generateDummyHistory(settings) - //generate invalid starting from 2 blocks + //generate invalid blocks begining from 2 blocks val (genesisState, state, chain) = generateChain(3, Some(2)) val historyApplicator: TestActorRef[HistoryApplicator] = @@ -230,5 +282,31 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) history.getBestBlock.map(b => Algos.encode(b.id)) shouldBe Some(Algos.encode(chain(1).id)) history.getBestBlockHeightDB shouldBe 1 } + + "check rollback on fat blocks" in { + + val history: History = generateDummyHistory(settings) + + //generate invalid blocks begining from 6 blocks + val (genesisState, state, chain) = generateChain(1) + + val historyApplicator: TestActorRef[HistoryApplicator] = + TestActorRef[HistoryApplicator]( + HistoryApplicator.props(history, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) + ) + system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) + + chain + .flatMap(blockToModifiers) + .foreach(historyApplicator ! ModifierFromRemote(_)) + + Thread.sleep(60000) +// checkFullBlockChainIsSynced(3 * 2) + +// history.getBestBlockHeight shouldBe 1 +// history.getBestBlock.map(b => Algos.encode(b.id)) shouldBe Some(Algos.encode(chain(1).id)) +// history.getBestBlockHeightDB shouldBe 1 + } + } } From ee2d6f79d5ebc6652f126d3db467fb3190403a94 Mon Sep 17 00:00:00 2001 From: capdev Date: Wed, 25 Sep 2019 16:55:51 +0500 Subject: [PATCH 16/39] add gen new chain with transQty --- .../history/HistoryApplicatorTest.scala | 150 ++++++++---------- 1 file changed, 63 insertions(+), 87 deletions(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index c5474780cb..af3d113e8f 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -46,96 +46,71 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) TestKit.shutdownActorSystem(system) } - def genHistoryBlocks(initialHistory: History, count: Int): (History, Seq[Block]) = { - (0 until count).foldLeft(initialHistory, Seq.empty[Block]) { - case ((prevHistory, blocks), _) => - val block: Block = generateNextBlock(prevHistory) - prevHistory.append(block.header) - prevHistory.append(block.payload) - (prevHistory.reportModifierIsValid(block), blocks :+ block) - } - } + // def genHistoryBlocks(initialHistory: History, count: Int): (History, Seq[Block]) = { + // (0 until count).foldLeft(initialHistory, Seq.empty[Block]) { + // case ((prevHistory, blocks), _) => + // val block: Block = generateNextBlock(prevHistory) + // prevHistory.append(block.header) + // prevHistory.append(block.payload) + // (prevHistory.reportModifierIsValid(block), blocks :+ block) + // } + // } def blockToModifiers(block: Block): Seq[PersistentModifier] = Seq(block.header, block.payload) def rndAddress: Address = Pay2PubKeyAddress(PublicKey @@ ScorexRandom.randomBytes()).address - def generateNextBlockForState(prevBlock: Block, state: UtxoState, box: Seq[AssetBox], splitCoef: Int = 2, - addDiff: Difficulty = Difficulty @@ BigInt(0)): Block = { + def genNextBlockForState(prevBlock: Block, state: UtxoState, boxes: Seq[AssetBox], invalid: Boolean = false, + addDiff: Difficulty = Difficulty @@ BigInt(0)): (Block, Seq[AssetBox]) = { val timestamp = System.currentTimeMillis() - val transactions: Seq[Transaction] = box.indices.foldLeft(box, Seq.empty[Transaction]) { - case ((boxes, transactionsL), _) => - val payTx: Transaction = TransactionFactory.defaultPaymentTransactionScratch(privKey, 0L, timestamp, IndexedSeq(boxes.head), - rndAddress, 1) - - val backTx: Transaction = TransactionFactory.defaultPaymentTransactionScratch(privKey, 0L, timestamp, IndexedSeq(boxes.head), - privKey.publicImage.address.address, boxes.head.amount - 1) + val amount = if(invalid) boxes.map(_.amount).sum + 1L else 1L - (boxes.tail, transactionsL :+ payTx :+ backTx) - }._2.filter(tx => state.validate(tx).isRight) ++ Seq(ChainUtils.coinbaseTransaction(prevBlock.header.height + 1)) - - println(s"transactions.size: ${transactions.size}") + val transactions: Seq[Transaction] = + boxes.map(b => TransactionFactory.defaultPaymentTransactionScratch(privKey, 1L, timestamp, IndexedSeq(b), rndAddress, amount)) :+ + ChainUtils.coinbaseTransaction(prevBlock.header.height + 1) val header = Header(1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), timestamp, prevBlock.header.height + 1, Random.nextLong(), Difficulty @@ (BigInt(1) + addDiff), EquihashSolution(Seq(1, 3))) - Block(header, Payload(header.id, transactions)) + val newBoxes = transactions.filter(_.newBoxes.size > 1).map(_.newBoxes.toSeq(1).asInstanceOf[AssetBox]) + (Block(header, Payload(header.id, transactions)), newBoxes) } - def genHardcodedBox(address: Address, nonce: Long): AssetBox = - AssetBox(EncryProposition.addressLocked(address), nonce, 10L) - - def generateChain(blockQty: Int, genInvalidBlockFrom: Option[Int] = None): (UtxoState, UtxoState, List[Block]) = { - val numberOfInputsInOneTransaction = 1 //10 - val numberOfOutputsInOneTransaction = 1 //10 - val transactionsNumberInEachBlock = 1000 //1000 - val initialboxCount = 100000 //100000 + def genForkBlock(boxQty: Int, prevBlock: Block, box: AssetBox, addDiff: Difficulty = Difficulty @@ BigInt(0)): (Block, Seq[AssetBox]) = { + val timestamp = System.currentTimeMillis() - val initialBoxes: IndexedSeq[AssetBox] = (0 until initialboxCount).map(nonce => - genHardcodedBox(privKey.publicImage.address.address, nonce) + val transactions: Seq[Transaction] = Seq( + ChainUtils.defaultPaymentTransactionScratch(privKey, 1L, timestamp, IndexedSeq(box), rndAddress, 1000L, None, boxQty), + ChainUtils.coinbaseTransaction(prevBlock.header.height + 1) ) - val boxesHolder: BoxHolder = BoxHolder(initialBoxes) + + val header = Header(1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), timestamp, + prevBlock.header.height + 1, Random.nextLong(), Difficulty @@ (BigInt(1) + addDiff), EquihashSolution(Seq(1, 3))) + + (Block(header, Payload(header.id, transactions)), transactions.head.newBoxes.tail.map(_.asInstanceOf[AssetBox]).toSeq) + } + + def genChain(blockQty: Int, transPerBlock: Int = 1, genInvalidBlockFrom: Option[Int] = None): (UtxoState, UtxoState, List[Block]) = { + assert(blockQty >= 2, "chain at least 2 blocks") val genesisBlock: Block = ChainUtils.generateGenesisBlock(Height @@ 0) - val state: UtxoState = utxoFromBoxHolder(boxesHolder, dir, None, settings, VersionalStorage.LevelDB) + val state: UtxoState = utxoFromBoxHolder(BoxHolder(Seq.empty), dir, None, settings, VersionalStorage.LevelDB) .applyModifier(genesisBlock).right.get - //1000000000 - val stateGenerationResults: (List[Block], Block, UtxoState, IndexedSeq[AssetBox]) = - (0 until blockQty).foldLeft(List.empty[Block], genesisBlock, state, initialBoxes) { - case ((blocks, block, stateL, boxes), height) => - val genInvalid = genInvalidBlockFrom.exists(height + 1 >= _) - val nextBlock: Block = - if (genInvalid) - generateNextBlockForStateWithInvalidTrans(block, stateL, - block.payload.txs.flatMap(_.newBoxes.map(_.asInstanceOf[AssetBox])).toIndexedSeq) - else - generateNextBlockForState(block, stateL, - //block.payload.txs.flatMap(_.newBoxes.map(_.asInstanceOf[AssetBox])).toIndexedSeq) - boxes.take(transactionsNumberInEachBlock * numberOfInputsInOneTransaction)) - - val stateN: UtxoState = if (genInvalid) stateL else stateL.applyModifier(nextBlock).right.get - (blocks :+ nextBlock, - nextBlock, stateN, boxes.drop(transactionsNumberInEachBlock * numberOfInputsInOneTransaction) - ) + val (forkBlock, forkBoxes) = genForkBlock(transPerBlock, genesisBlock, genesisBlock.payload.txs.head.newBoxes.map(_.asInstanceOf[AssetBox]).head) + val forkState = state.applyModifier(forkBlock).right.get + + val (chain, _, newState, _) = + (2 until blockQty).foldLeft(List(genesisBlock, forkBlock), forkBlock, forkState, forkBoxes) { + case ((blocks, blockL, stateL, boxes), height) => + val invalid = genInvalidBlockFrom.exists(height + 1 >= _) + val (newBlock, newBoxes) = genNextBlockForState(blockL, stateL, boxes, invalid) + val newState = if(invalid) stateL else stateL.applyModifier(newBlock).right.get + (blocks :+ newBlock, newBlock, newState, newBoxes) } -// val stateGenerationResults: (Vector[Block], Block, UtxoState, IndexedSeq[AssetBox]) = -// (0 until blockQty).foldLeft(Vector[Block](), genesisBlock, state, initialBoxes) { -// case ((blocks, block, stateL, boxes), _) => -// val nextBlock: Block = generateNextBlockValidForState( -// block, stateL, boxes.take(transactionsNumberInEachBlock * numberOfInputsInOneTransaction), -// transactionsNumberInEachBlock, numberOfInputsInOneTransaction, numberOfOutputsInOneTransaction -// ) -// val stateN: UtxoState = stateL.applyModifier(nextBlock).right.get -// (blocks :+ nextBlock, nextBlock, stateN, boxes.drop(transactionsNumberInEachBlock * numberOfInputsInOneTransaction) -// ) -// } - - //(genesisBlock +: stateGenerationResults._1).foreach(b => println(s"block.header : ${Algos.encode(b.header.id)}")) - - (state, stateGenerationResults._3, genesisBlock +: stateGenerationResults._1.toList) + (state, newState, chain) } def applyBlocksToHistory(history: History, blocks: Seq[Block]): History = @@ -173,7 +148,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val blockQty = 10 val history: History = generateDummyHistory(settings) - val (genesisState, state, chain) = generateChain(blockQty) + val (genesisState, state, chain) = genChain(blockQty) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( @@ -191,7 +166,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) "check queues" in { val initialHistory: History = generateDummyHistory(settings) - val (genesisState, state, chain) = generateChain(1) + val (genesisState, state, chain) = genChain(1) val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) //val modifierIds = modifiers.map(_.id) @@ -210,18 +185,18 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) receiveN(4, timeout).forall { case _: SemanticallySuccessfulModifier => true } -// val modifiersQueue = historyApplicator.underlyingActor.modifiersQueue -// modifiersQueue.size shouldBe 2 -// historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 + // val modifiersQueue = historyApplicator.underlyingActor.modifiersQueue + // modifiersQueue.size shouldBe 2 + // historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 //Thread.sleep(15000) //historyApplicator ! StateApplicator.RequestNextModifier -// -// val msg = receiveWhile(timeout) { -// case msg: StartModifiersApplicationOnStateApplicator => msg.progressInfo.toApply.size -// case msg: Any => println(msg) -// } -// println(s"msg: ${msg}") + // + // val msg = receiveWhile(timeout) { + // case msg: StartModifiersApplicationOnStateApplicator => msg.progressInfo.toApply.size + // case msg: Any => println(msg) + // } + // println(s"msg: ${msg}") // historyApplicator.underlyingActor.modifiersQueue shouldBe 0 // historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 @@ -239,7 +214,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val overQty = 30 val history: History = generateDummyHistory(settings) - val (genesisState, state, chain) = generateChain(settings.levelDB.maxVersions + overQty) + val (genesisState, state, chain) = genChain(settings.levelDB.maxVersions + overQty) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( @@ -264,7 +239,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val history: History = generateDummyHistory(settings) //generate invalid blocks begining from 2 blocks - val (genesisState, state, chain) = generateChain(3, Some(2)) + val (genesisState, state, chain) = genChain(3, 1, Some(2)) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( @@ -288,7 +263,9 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val history: History = generateDummyHistory(settings) //generate invalid blocks begining from 6 blocks - val (genesisState, state, chain) = generateChain(1) + val (genesisState, state, chain) = genChain(10, 5, Some(6)) + + chain.foreach(b => println(s"block.header : ${Algos.encode(b.header.id)}")) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( @@ -300,12 +277,11 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) .flatMap(blockToModifiers) .foreach(historyApplicator ! ModifierFromRemote(_)) - Thread.sleep(60000) -// checkFullBlockChainIsSynced(3 * 2) + checkFullBlockChainIsSynced(10) -// history.getBestBlockHeight shouldBe 1 -// history.getBestBlock.map(b => Algos.encode(b.id)) shouldBe Some(Algos.encode(chain(1).id)) -// history.getBestBlockHeightDB shouldBe 1 + history.getBestBlockHeight shouldBe Height @@ 4 + history.getBestBlockHeightDB shouldBe Height @@ 4 + history.getBestBlock.map(b => Algos.encode(b.id)) shouldBe Some(Algos.encode(chain(4).id)) } } From 3f81a7df2ab2b2a7a78de8d1ed2a94993efb6d00 Mon Sep 17 00:00:00 2001 From: capdev Date: Wed, 25 Sep 2019 17:40:46 +0500 Subject: [PATCH 17/39] adapted test for new chain some refactoring --- .../history/HistoryApplicatorTest.scala | 210 +++++++----------- 1 file changed, 79 insertions(+), 131 deletions(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index af3d113e8f..4aa7d441c4 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -46,52 +46,43 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) TestKit.shutdownActorSystem(system) } - // def genHistoryBlocks(initialHistory: History, count: Int): (History, Seq[Block]) = { - // (0 until count).foldLeft(initialHistory, Seq.empty[Block]) { - // case ((prevHistory, blocks), _) => - // val block: Block = generateNextBlock(prevHistory) - // prevHistory.append(block.header) - // prevHistory.append(block.payload) - // (prevHistory.reportModifierIsValid(block), blocks :+ block) - // } - // } - def blockToModifiers(block: Block): Seq[PersistentModifier] = Seq(block.header, block.payload) def rndAddress: Address = Pay2PubKeyAddress(PublicKey @@ ScorexRandom.randomBytes()).address - def genNextBlockForState(prevBlock: Block, state: UtxoState, boxes: Seq[AssetBox], invalid: Boolean = false, - addDiff: Difficulty = Difficulty @@ BigInt(0)): (Block, Seq[AssetBox]) = { - val timestamp = System.currentTimeMillis() + def genChain(blockQty: Int, transPerBlock: Int = 1, genInvalidBlockFrom: Option[Int] = None): (UtxoState, UtxoState, List[Block]) = { - val amount = if(invalid) boxes.map(_.amount).sum + 1L else 1L + def genForkBlock(boxQty: Int, prevBlock: Block, box: AssetBox, addDiff: Difficulty = Difficulty @@ BigInt(0)): (Block, Seq[AssetBox]) = { + val timestamp = System.currentTimeMillis() - val transactions: Seq[Transaction] = - boxes.map(b => TransactionFactory.defaultPaymentTransactionScratch(privKey, 1L, timestamp, IndexedSeq(b), rndAddress, amount)) :+ + val transactions: Seq[Transaction] = Seq( + ChainUtils.defaultPaymentTransactionScratch(privKey, 1L, timestamp, IndexedSeq(box), rndAddress, 1000L, None, boxQty), ChainUtils.coinbaseTransaction(prevBlock.header.height + 1) + ) - val header = Header(1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), timestamp, - prevBlock.header.height + 1, Random.nextLong(), Difficulty @@ (BigInt(1) + addDiff), EquihashSolution(Seq(1, 3))) + val header = Header(1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), timestamp, + prevBlock.header.height + 1, Random.nextLong(), Difficulty @@ (BigInt(1) + addDiff), EquihashSolution(Seq(1, 3))) - val newBoxes = transactions.filter(_.newBoxes.size > 1).map(_.newBoxes.toSeq(1).asInstanceOf[AssetBox]) - (Block(header, Payload(header.id, transactions)), newBoxes) - } + (Block(header, Payload(header.id, transactions)), transactions.head.newBoxes.tail.map(_.asInstanceOf[AssetBox]).toSeq) + } - def genForkBlock(boxQty: Int, prevBlock: Block, box: AssetBox, addDiff: Difficulty = Difficulty @@ BigInt(0)): (Block, Seq[AssetBox]) = { - val timestamp = System.currentTimeMillis() + def genNextBlockForState(prevBlock: Block, state: UtxoState, boxes: Seq[AssetBox], invalid: Boolean = false, + addDiff: Difficulty = Difficulty @@ BigInt(0)): (Block, Seq[AssetBox]) = { + val timestamp = System.currentTimeMillis() - val transactions: Seq[Transaction] = Seq( - ChainUtils.defaultPaymentTransactionScratch(privKey, 1L, timestamp, IndexedSeq(box), rndAddress, 1000L, None, boxQty), - ChainUtils.coinbaseTransaction(prevBlock.header.height + 1) - ) + val amount = if(invalid) boxes.map(_.amount).sum + 1L else 1L - val header = Header(1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), timestamp, - prevBlock.header.height + 1, Random.nextLong(), Difficulty @@ (BigInt(1) + addDiff), EquihashSolution(Seq(1, 3))) + val transactions: Seq[Transaction] = + boxes.map(b => TransactionFactory.defaultPaymentTransactionScratch(privKey, 1L, timestamp, IndexedSeq(b), rndAddress, amount)) :+ + ChainUtils.coinbaseTransaction(prevBlock.header.height + 1) - (Block(header, Payload(header.id, transactions)), transactions.head.newBoxes.tail.map(_.asInstanceOf[AssetBox]).toSeq) - } + val header = Header(1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), timestamp, + prevBlock.header.height + 1, Random.nextLong(), Difficulty @@ (BigInt(1) + addDiff), EquihashSolution(Seq(1, 3))) + + val newBoxes = transactions.filter(_.newBoxes.size > 1).map(_.newBoxes.toSeq(1).asInstanceOf[AssetBox]) + (Block(header, Payload(header.id, transactions)), newBoxes) + } - def genChain(blockQty: Int, transPerBlock: Int = 1, genInvalidBlockFrom: Option[Int] = None): (UtxoState, UtxoState, List[Block]) = { assert(blockQty >= 2, "chain at least 2 blocks") val genesisBlock: Block = ChainUtils.generateGenesisBlock(Height @@ 0) @@ -113,25 +104,48 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) (state, newState, chain) } - def applyBlocksToHistory(history: History, blocks: Seq[Block]): History = - blocks.foldLeft(history) { - case (prevHistory, block) => - prevHistory.append(block.header) - prevHistory.append(block.payload) - prevHistory.reportModifierIsValid(block) - } + def rollbackTest(transQty: Int) { - def printIds(blocks: List[Block]) = { - blocks.foreach { b => - println(s"header.timestamp: ${b.header.timestamp}") - println(s"header.id: ${Algos.encode(b.header.id)}") - println(s"header.payloadId: ${Algos.encode(b.header.payloadId)}") - println(s"payload.id: ${Algos.encode(b.payload.id)}") - println(s"payload.headerId: ${Algos.encode(b.payload.headerId)}") - } + val history: History = generateDummyHistory(settings) + //generate invalid blocks begining from 6 blocks + val (genesisState, state, chain) = genChain(10, transQty, Some(6)) + + val historyApplicator: TestActorRef[HistoryApplicator] = + TestActorRef[HistoryApplicator]( + HistoryApplicator.props(history, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) + ) + system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) + + chain + .flatMap(blockToModifiers) + .foreach(historyApplicator ! ModifierFromRemote(_)) + + receiveN(6 * 2, 30 seconds) + + history.getBestBlockHeight shouldBe 4 + history.getBestBlock.map(b => Algos.encode(b.id)) shouldBe Some(Algos.encode(chain(4).id)) + history.getBestBlockHeightDB shouldBe 4 } - def checkFullBlockChainIsSynced(qty: Int): Unit = (0 until qty).foreach(_ => expectMsg(timeout, FullBlockChainIsSynced())) +// def applyBlocksToHistory(history: History, blocks: Seq[Block]): History = +// blocks.foldLeft(history) { +// case (prevHistory, block) => +// prevHistory.append(block.header) +// prevHistory.append(block.payload) +// prevHistory.reportModifierIsValid(block) +// } + +// def printIds(blocks: List[Block]) = { +// blocks.foreach { b => +// println(s"header.timestamp: ${b.header.timestamp}") +// println(s"header.id: ${Algos.encode(b.header.id)}") +// println(s"header.payloadId: ${Algos.encode(b.header.payloadId)}") +// println(s"payload.id: ${Algos.encode(b.payload.id)}") +// println(s"payload.headerId: ${Algos.encode(b.payload.headerId)}") +// } +// } + + //def checkFullBlockChainIsSynced(qty: Int): Unit = (0 until qty).foreach(_ => expectMsg(timeout, FullBlockChainIsSynced())) val dir: File = FileHelper.getRandomTempDir val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) @@ -146,7 +160,6 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) "apply locall blocks and chain sync" in { val blockQty = 10 - val history: History = generateDummyHistory(settings) val (genesisState, state, chain) = genChain(blockQty) @@ -155,21 +168,21 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) HistoryApplicator.props(history, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) - system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) + system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) chain.foreach(historyApplicator ! LocallyGeneratedBlock(_)) - checkFullBlockChainIsSynced((blockQty + 1) * 2) - history.getBestBlockHeight shouldBe blockQty + receiveN(blockQty * 2, 30 seconds) + history.getBestBlockHeight shouldBe blockQty - 1 } - "check queues" in { + "check SemanticallySuccessfulModifier per block" in { + val blockQty = 2 val initialHistory: History = generateDummyHistory(settings) - val (genesisState, state, chain) = genChain(1) + val (genesisState, state, chain) = genChain(blockQty) val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) - //val modifierIds = modifiers.map(_.id) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( @@ -180,33 +193,10 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) - println(s"test.modifiersQueue: ${historyApplicator.underlyingActor.modifiersQueue.size}") - println(s"test.currentNumberOfAppliedModifiers: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") - - receiveN(4, timeout).forall { case _: SemanticallySuccessfulModifier => true } - - // val modifiersQueue = historyApplicator.underlyingActor.modifiersQueue - // modifiersQueue.size shouldBe 2 - // historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 - - //Thread.sleep(15000) - //historyApplicator ! StateApplicator.RequestNextModifier - // - // val msg = receiveWhile(timeout) { - // case msg: StartModifiersApplicationOnStateApplicator => msg.progressInfo.toApply.size - // case msg: Any => println(msg) - // } - // println(s"msg: ${msg}") - - // historyApplicator.underlyingActor.modifiersQueue shouldBe 0 - // historyApplicator.underlyingActor.currentNumberOfAppliedModifiers shouldBe 2 - // - // val history = historyApplicator.underlyingActor.history - // - // expectMsgType[StartModifiersApplicationOnStateApplicator](timeout) - // - // assert(modifierIds.forall(history.isModifierDefined)) - // assert(modifierIds.map(Algos.encode) == modifiersQueue.map(_._1)) + //println(s"test.modifiersQueue: ${historyApplicator.underlyingActor.modifiersQueue.size}") + //println(s"test.currentNumberOfAppliedModifiers: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") + + receiveN(blockQty * 2, timeout).forall { case _: SemanticallySuccessfulModifier => true } } "check queue for rollback height" in { @@ -214,7 +204,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val overQty = 30 val history: History = generateDummyHistory(settings) - val (genesisState, state, chain) = genChain(settings.levelDB.maxVersions + overQty) + val (genesisState, state, chain) = genChain(settings.levelDB.maxVersions + overQty, 10) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( @@ -226,62 +216,20 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) .flatMap(blockToModifiers) .foreach(historyApplicator ! ModifierFromRemote(_)) - println(s"modifiersQueue.size: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") + //println(s"modifiersQueue.size: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") historyApplicator.underlyingActor.modifiersQueue.size should be <= settings.levelDB.maxVersions receiveN((settings.levelDB.maxVersions + overQty) * 2, 30 seconds) - history.getBestBlockHeight shouldBe settings.levelDB.maxVersions + overQty + history.getBestBlockHeight shouldBe settings.levelDB.maxVersions + overQty - 1 } - "check rollback for invalid state" in { - - val history: History = generateDummyHistory(settings) - - //generate invalid blocks begining from 2 blocks - val (genesisState, state, chain) = genChain(3, 1, Some(2)) - - val historyApplicator: TestActorRef[HistoryApplicator] = - TestActorRef[HistoryApplicator]( - HistoryApplicator.props(history, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) - ) - system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) - - chain - .flatMap(blockToModifiers) - .foreach(historyApplicator ! ModifierFromRemote(_)) - - checkFullBlockChainIsSynced(3 * 2) - - history.getBestBlockHeight shouldBe 1 - history.getBestBlock.map(b => Algos.encode(b.id)) shouldBe Some(Algos.encode(chain(1).id)) - history.getBestBlockHeightDB shouldBe 1 + "check history rollback for invalid state" in { + rollbackTest(100) } - "check rollback on fat blocks" in { - - val history: History = generateDummyHistory(settings) - - //generate invalid blocks begining from 6 blocks - val (genesisState, state, chain) = genChain(10, 5, Some(6)) - - chain.foreach(b => println(s"block.header : ${Algos.encode(b.header.id)}")) - - val historyApplicator: TestActorRef[HistoryApplicator] = - TestActorRef[HistoryApplicator]( - HistoryApplicator.props(history, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) - ) - system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) - - chain - .flatMap(blockToModifiers) - .foreach(historyApplicator ! ModifierFromRemote(_)) - - checkFullBlockChainIsSynced(10) - - history.getBestBlockHeight shouldBe Height @@ 4 - history.getBestBlockHeightDB shouldBe Height @@ 4 - history.getBestBlock.map(b => Algos.encode(b.id)) shouldBe Some(Algos.encode(chain(4).id)) + "check history rollback on fat blocks" in { + rollbackTest(5000) } } From db12820f4756306619742e9b05e1a45b219a94e3 Mon Sep 17 00:00:00 2001 From: capdev Date: Wed, 25 Sep 2019 18:11:00 +0500 Subject: [PATCH 18/39] extract chain to ChainGenerator --- .../history/HistoryApplicatorTest.scala | 102 +---------- .../scala/encry/utils/ChainGenerator.scala | 165 ++++++++++++++++++ 2 files changed, 174 insertions(+), 93 deletions(-) create mode 100644 src/test/scala/encry/utils/ChainGenerator.scala diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 4aa7d441c4..053ddfaa53 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -5,42 +5,24 @@ import java.io.File import akka.actor.ActorSystem import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} import encry.modifiers.InstanceFactory -import encry.modifiers.mempool.TransactionFactory -import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier import encry.settings.Settings -import encry.storage.VersionalStorage -import encry.utils.ChainUtils.{coinbaseTransaction, defaultPaymentTransactionScratch, privKey, _} -import encry.utils.{ChainUtils, FileHelper} +import encry.utils.ChainGenerator._ +import encry.utils.FileHelper import encry.view.actors.HistoryApplicator import encry.view.actors.NodeViewHolder.ReceivableMessages.{LocallyGeneratedBlock, ModifierFromRemote} import encry.view.history.History -import encry.view.state.{BoxHolder, UtxoState} import encry.view.wallet.EncryWallet -import org.encryfoundation.common.crypto.equihash.EquihashSolution import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} -import org.encryfoundation.common.modifiers.mempool.transaction.EncryAddress.Address -import org.encryfoundation.common.modifiers.mempool.transaction.{Pay2PubKeyAddress, Transaction} -import org.encryfoundation.common.modifiers.state.box.{AssetBox, EncryProposition} +import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height} import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -import scorex.crypto.signatures.PublicKey -import scorex.utils.{Random => ScorexRandom} -import scala.util.Random import scala.collection.Seq import scala.concurrent.duration._ -class HistoryApplicatorTest extends TestKit(ActorSystem()) - with WordSpecLike - with ImplicitSender - with BeforeAndAfterAll - with Matchers - with InstanceFactory - with OneInstancePerTest - with Settings { +class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike with ImplicitSender with BeforeAndAfterAll with Matchers + with InstanceFactory with OneInstancePerTest with Settings { override def afterAll: Unit = { TestKit.shutdownActorSystem(system) @@ -48,67 +30,11 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) def blockToModifiers(block: Block): Seq[PersistentModifier] = Seq(block.header, block.payload) - def rndAddress: Address = Pay2PubKeyAddress(PublicKey @@ ScorexRandom.randomBytes()).address - - def genChain(blockQty: Int, transPerBlock: Int = 1, genInvalidBlockFrom: Option[Int] = None): (UtxoState, UtxoState, List[Block]) = { - - def genForkBlock(boxQty: Int, prevBlock: Block, box: AssetBox, addDiff: Difficulty = Difficulty @@ BigInt(0)): (Block, Seq[AssetBox]) = { - val timestamp = System.currentTimeMillis() - - val transactions: Seq[Transaction] = Seq( - ChainUtils.defaultPaymentTransactionScratch(privKey, 1L, timestamp, IndexedSeq(box), rndAddress, 1000L, None, boxQty), - ChainUtils.coinbaseTransaction(prevBlock.header.height + 1) - ) - - val header = Header(1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), timestamp, - prevBlock.header.height + 1, Random.nextLong(), Difficulty @@ (BigInt(1) + addDiff), EquihashSolution(Seq(1, 3))) - - (Block(header, Payload(header.id, transactions)), transactions.head.newBoxes.tail.map(_.asInstanceOf[AssetBox]).toSeq) - } - - def genNextBlockForState(prevBlock: Block, state: UtxoState, boxes: Seq[AssetBox], invalid: Boolean = false, - addDiff: Difficulty = Difficulty @@ BigInt(0)): (Block, Seq[AssetBox]) = { - val timestamp = System.currentTimeMillis() - - val amount = if(invalid) boxes.map(_.amount).sum + 1L else 1L - - val transactions: Seq[Transaction] = - boxes.map(b => TransactionFactory.defaultPaymentTransactionScratch(privKey, 1L, timestamp, IndexedSeq(b), rndAddress, amount)) :+ - ChainUtils.coinbaseTransaction(prevBlock.header.height + 1) - - val header = Header(1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), timestamp, - prevBlock.header.height + 1, Random.nextLong(), Difficulty @@ (BigInt(1) + addDiff), EquihashSolution(Seq(1, 3))) - - val newBoxes = transactions.filter(_.newBoxes.size > 1).map(_.newBoxes.toSeq(1).asInstanceOf[AssetBox]) - (Block(header, Payload(header.id, transactions)), newBoxes) - } - - assert(blockQty >= 2, "chain at least 2 blocks") - - val genesisBlock: Block = ChainUtils.generateGenesisBlock(Height @@ 0) - - val state: UtxoState = utxoFromBoxHolder(BoxHolder(Seq.empty), dir, None, settings, VersionalStorage.LevelDB) - .applyModifier(genesisBlock).right.get - - val (forkBlock, forkBoxes) = genForkBlock(transPerBlock, genesisBlock, genesisBlock.payload.txs.head.newBoxes.map(_.asInstanceOf[AssetBox]).head) - val forkState = state.applyModifier(forkBlock).right.get - - val (chain, _, newState, _) = - (2 until blockQty).foldLeft(List(genesisBlock, forkBlock), forkBlock, forkState, forkBoxes) { - case ((blocks, blockL, stateL, boxes), height) => - val invalid = genInvalidBlockFrom.exists(height + 1 >= _) - val (newBlock, newBoxes) = genNextBlockForState(blockL, stateL, boxes, invalid) - val newState = if(invalid) stateL else stateL.applyModifier(newBlock).right.get - (blocks :+ newBlock, newBlock, newState, newBoxes) - } - (state, newState, chain) - } - def rollbackTest(transQty: Int) { val history: History = generateDummyHistory(settings) //generate invalid blocks begining from 6 blocks - val (genesisState, state, chain) = genChain(10, transQty, Some(6)) + val (genesisState, state, chain) = genChain(privKey, dir, settings, 10, transQty, Some(6)) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( @@ -135,16 +61,6 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) // prevHistory.reportModifierIsValid(block) // } -// def printIds(blocks: List[Block]) = { -// blocks.foreach { b => -// println(s"header.timestamp: ${b.header.timestamp}") -// println(s"header.id: ${Algos.encode(b.header.id)}") -// println(s"header.payloadId: ${Algos.encode(b.header.payloadId)}") -// println(s"payload.id: ${Algos.encode(b.payload.id)}") -// println(s"payload.headerId: ${Algos.encode(b.payload.headerId)}") -// } -// } - //def checkFullBlockChainIsSynced(qty: Int): Unit = (0 until qty).foreach(_ => expectMsg(timeout, FullBlockChainIsSynced())) val dir: File = FileHelper.getRandomTempDir @@ -161,7 +77,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val blockQty = 10 val history: History = generateDummyHistory(settings) - val (genesisState, state, chain) = genChain(blockQty) + val (genesisState, state, chain) = genChain(privKey, dir, settings, blockQty) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( @@ -180,7 +96,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val blockQty = 2 val initialHistory: History = generateDummyHistory(settings) - val (genesisState, state, chain) = genChain(blockQty) + val (genesisState, state, chain) = genChain(privKey, dir, settings, blockQty) val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) @@ -204,7 +120,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) val overQty = 30 val history: History = generateDummyHistory(settings) - val (genesisState, state, chain) = genChain(settings.levelDB.maxVersions + overQty, 10) + val (genesisState, state, chain) = genChain(privKey, dir, settings, settings.levelDB.maxVersions + overQty, 10) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( diff --git a/src/test/scala/encry/utils/ChainGenerator.scala b/src/test/scala/encry/utils/ChainGenerator.scala new file mode 100644 index 0000000000..0df84d3230 --- /dev/null +++ b/src/test/scala/encry/utils/ChainGenerator.scala @@ -0,0 +1,165 @@ +package encry.utils + +import java.io.File + +import akka.actor.ActorRef +import encry.modifiers.mempool.TransactionFactory +import encry.settings.EncryAppSettings +import encry.storage.VersionalStorage +import encry.storage.VersionalStorage.{StorageKey, StorageType, StorageValue, StorageVersion} +import encry.storage.iodb.versionalIODB.IODBWrapper +import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} +import encry.view.state.{BoxHolder, UtxoState} +import io.iohk.iodb.LSMStore +import org.encryfoundation.common.crypto.{PrivateKey25519, PublicKey25519, Signature25519} +import org.encryfoundation.common.crypto.equihash.EquihashSolution +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.mempool.directive.TransferDirective +import org.encryfoundation.common.modifiers.mempool.transaction.EncryAddress.Address +import org.encryfoundation.common.modifiers.mempool.transaction.{Input, Pay2PubKeyAddress, Proof, PubKeyLockedContract, Transaction, UnsignedTransaction} +import org.encryfoundation.common.modifiers.state.box.Box.Amount +import org.encryfoundation.common.modifiers.state.box.{AssetBox, MonetaryBox} +import org.encryfoundation.common.utils.TaggedTypes.{ADKey, Difficulty, Height} +import org.encryfoundation.prismlang.core.wrapped.BoxedValue +import org.iq80.leveldb.Options +import scorex.crypto.hash.Digest32 +import scorex.crypto.signatures.PublicKey + +import scala.collection.Seq +import scorex.utils.{Random => ScorexRandom} + +import scala.util.Random + +object ChainGenerator { + + def genChain(privKey: PrivateKey25519, dir: File, settings: EncryAppSettings, blockQty: Int, transPerBlock: Int = 1, + genInvalidBlockFrom: Option[Int] = None): (UtxoState, UtxoState, List[Block]) = { + + def utxoFromBoxHolder(bh: BoxHolder, dir: File, nodeViewHolderRef: Option[ActorRef], settings: EncryAppSettings, + storageType: StorageType): UtxoState = { + val storage = settings.storage.state match { + case VersionalStorage.IODB => + IODBWrapper(new LSMStore(dir, keepVersions = settings.constants.DefaultKeepVersions)) + case VersionalStorage.LevelDB => + val levelDBInit = LevelDbFactory.factory.open(dir, new Options) + VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settings.levelDB, keySize = 32)) + } + + storage.insert( + StorageVersion @@ Array.fill(32)(0: Byte), + bh.boxes.values.map(bx => (StorageKey !@@ bx.id, StorageValue @@ bx.bytes)).toList + ) + + new UtxoState(storage, settings.constants) + } + + def paymentTransactionWithMultipleOutputs(privKey: PrivateKey25519, fee: Amount, timestamp: Long, useBoxes: IndexedSeq[MonetaryBox], + recipient: Address, amount: Amount, tokenIdOpt: Option[ADKey] = None, + numOfOutputs: Int): Transaction = { + + val pubKey: PublicKey25519 = privKey.publicImage + val uInputs: IndexedSeq[Input] = useBoxes + .map(bx => Input.unsigned(bx.id, Right(PubKeyLockedContract(pubKey.pubKeyBytes)))) + .toIndexedSeq + + val change: Amount = useBoxes.map(_.amount).sum - (amount + fee) + val directives: IndexedSeq[TransferDirective] = + if (change > 0) TransferDirective(recipient, amount, tokenIdOpt) +: (0 until numOfOutputs).map(_ => + TransferDirective(pubKey.address.address, change / numOfOutputs, tokenIdOpt)) + else IndexedSeq(TransferDirective(recipient, amount, tokenIdOpt)) + + val uTransaction: UnsignedTransaction = UnsignedTransaction(fee, timestamp, uInputs, directives) + val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) + uTransaction.toSigned(IndexedSeq.empty, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) + } + + def genForkBlock(boxQty: Int, prevBlock: Block, box: AssetBox, recipient: Address, + addDiff: Difficulty = Difficulty @@ BigInt(0)): (Block, Seq[AssetBox]) = { + val timestamp = System.currentTimeMillis() + + val transactions: Seq[Transaction] = Seq( + paymentTransactionWithMultipleOutputs(privKey, 1L, timestamp, IndexedSeq(box), recipient, 1000L, None, boxQty), + coinbaseTransaction(prevBlock.header.height + 1) + ) + + val header = Header(1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), timestamp, + prevBlock.header.height + 1, Random.nextLong(), Difficulty @@ (BigInt(1) + addDiff), EquihashSolution(Seq(1, 3))) + + (Block(header, Payload(header.id, transactions)), transactions.head.newBoxes.tail.map(_.asInstanceOf[AssetBox]).toSeq) + } + + def generateGenesisBlock(genesisHeight: Height): Block = { + val coinbaseTrans = TransactionFactory.coinbaseTransactionScratch( + privKey.publicImage, + System.currentTimeMillis(), + settings.constants.InitialEmissionAmount, + amount = 0L, + height = genesisHeight + ) + + val txs: Seq[Transaction] = Seq(coinbaseTrans) + val txsRoot: Digest32 = Payload.rootHash(txs.map(_.id)) + val header = + Header( + 1.toByte, + Header.GenesisParentId, + txsRoot, + System.currentTimeMillis(), + genesisHeight, + Random.nextLong(), + settings.constants.InitialDifficulty, + EquihashSolution(Seq(1, 3)) + ) + Block(header, Payload(header.id, txs)) + } + + def coinbaseTransaction(height: Int): Transaction = TransactionFactory.coinbaseTransactionScratch( + privKey.publicImage, + System.currentTimeMillis(), + supply = settings.constants.InitialEmissionAmount, + amount = 1L, + height = Height @@ height + ) + + def genNextBlockForState(prevBlock: Block, state: UtxoState, boxes: Seq[AssetBox], recipient: Address, invalid: Boolean = false, + addDiff: Difficulty = Difficulty @@ BigInt(0)): (Block, Seq[AssetBox]) = { + val timestamp = System.currentTimeMillis() + + val amount = if (invalid) boxes.map(_.amount).sum + 1L else 1L + + val transactions: Seq[Transaction] = + boxes.map(b => TransactionFactory.defaultPaymentTransactionScratch(privKey, 1L, timestamp, IndexedSeq(b), recipient, amount)) :+ + coinbaseTransaction(prevBlock.header.height + 1) + + val header = Header(1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), timestamp, + prevBlock.header.height + 1, Random.nextLong(), Difficulty @@ (BigInt(1) + addDiff), EquihashSolution(Seq(1, 3))) + + val newBoxes = transactions.filter(_.newBoxes.size > 1).map(_.newBoxes.toSeq(1).asInstanceOf[AssetBox]) + (Block(header, Payload(header.id, transactions)), newBoxes) + } + + def randomAddress: Address = Pay2PubKeyAddress(PublicKey @@ ScorexRandom.randomBytes()).address + + assert(blockQty >= 2, "chain at least 2 blocks") + + val genesisBlock: Block = generateGenesisBlock(Height @@ 0) + + val state: UtxoState = utxoFromBoxHolder(BoxHolder(Seq.empty), dir, None, settings, VersionalStorage.LevelDB) + .applyModifier(genesisBlock).right.get + + val (forkBlock, forkBoxes) = + genForkBlock(transPerBlock, genesisBlock, genesisBlock.payload.txs.head.newBoxes.map(_.asInstanceOf[AssetBox]).head, randomAddress) + val forkState = state.applyModifier(forkBlock).right.get + + val (chain, _, newState, _) = + (2 until blockQty).foldLeft(List(genesisBlock, forkBlock), forkBlock, forkState, forkBoxes) { + case ((blocks, blockL, stateL, boxes), height) => + val invalid = genInvalidBlockFrom.exists(height + 1 >= _) + val (newBlock, newBoxes) = genNextBlockForState(blockL, stateL, boxes, randomAddress, invalid) + val newState = if (invalid) stateL else stateL.applyModifier(newBlock).right.get + (blocks :+ newBlock, newBlock, newState, newBoxes) + } + (state, newState, chain) + } + +} \ No newline at end of file From f62f07f24cb829be5589538ba4eb7a05a7f40575 Mon Sep 17 00:00:00 2001 From: capdev Date: Thu, 26 Sep 2019 10:53:54 +0500 Subject: [PATCH 19/39] some refactoring --- .../history/HistoryApplicatorTest.scala | 36 +++++++++---------- .../scala/encry/utils/ChainGenerator.scala | 4 +-- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 053ddfaa53..c82a80b5c9 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -5,6 +5,7 @@ import java.io.File import akka.actor.ActorSystem import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} import encry.modifiers.InstanceFactory +import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier import encry.settings.Settings import encry.utils.ChainGenerator._ @@ -31,14 +32,13 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w def blockToModifiers(block: Block): Seq[PersistentModifier] = Seq(block.header, block.payload) def rollbackTest(transQty: Int) { - val history: History = generateDummyHistory(settings) //generate invalid blocks begining from 6 blocks - val (genesisState, state, chain) = genChain(privKey, dir, settings, 10, transQty, Some(6)) + val (initialState, state, chain) = genChain(privKey, dir, settings, 10, transQty, Some(6)) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( - HistoryApplicator.props(history, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) + HistoryApplicator.props(history, settings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) @@ -61,7 +61,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w // prevHistory.reportModifierIsValid(block) // } - //def checkFullBlockChainIsSynced(qty: Int): Unit = (0 until qty).foreach(_ => expectMsg(timeout, FullBlockChainIsSynced())) + def checkFullBlockChainIsSynced(qty: Int): Unit = (0 until qty).foreach(_ => expectMsg(timeout, FullBlockChainIsSynced())) val dir: File = FileHelper.getRandomTempDir val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) @@ -77,54 +77,52 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w val blockQty = 10 val history: History = generateDummyHistory(settings) - val (genesisState, state, chain) = genChain(privKey, dir, settings, blockQty) + val (initialState, state, chain) = genChain(privKey, dir, settings, blockQty) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( - HistoryApplicator.props(history, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) + HistoryApplicator.props(history, settings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) - system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) + system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) chain.foreach(historyApplicator ! LocallyGeneratedBlock(_)) - receiveN(blockQty * 2, 30 seconds) + checkFullBlockChainIsSynced(blockQty * 2) + //receiveN(blockQty * 2, 30 seconds) history.getBestBlockHeight shouldBe blockQty - 1 } - "check SemanticallySuccessfulModifier per block" in { + "apply remote blocks and check SemanticallySuccessfulModifier" in { val blockQty = 2 val initialHistory: History = generateDummyHistory(settings) - val (genesisState, state, chain) = genChain(privKey, dir, settings, blockQty) + val (initialState, state, chain) = genChain(privKey, dir, settings, blockQty) val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( - HistoryApplicator.props(initialHistory, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) + HistoryApplicator.props(initialHistory, settings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) - //println(s"test.modifiersQueue: ${historyApplicator.underlyingActor.modifiersQueue.size}") - //println(s"test.currentNumberOfAppliedModifiers: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") - receiveN(blockQty * 2, timeout).forall { case _: SemanticallySuccessfulModifier => true } } - "check queue for rollback height" in { + "apply remote blocks and check queue for rollback height" in { val overQty = 30 val history: History = generateDummyHistory(settings) - val (genesisState, state, chain) = genChain(privKey, dir, settings, settings.levelDB.maxVersions + overQty, 10) + val (initialState, state, chain) = genChain(privKey, dir, settings, settings.levelDB.maxVersions + overQty, 10) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( - HistoryApplicator.props(history, settings, genesisState, wallet, nodeViewHolder.ref, Some(influx.ref)) + HistoryApplicator.props(history, settings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) @@ -140,11 +138,11 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w history.getBestBlockHeight shouldBe settings.levelDB.maxVersions + overQty - 1 } - "check history rollback for invalid state" in { + "apply remote blocks and check history rollback for invalid state" in { rollbackTest(100) } - "check history rollback on fat blocks" in { + "apply remote blocks and check history rollback on fat blocks" in { rollbackTest(5000) } diff --git a/src/test/scala/encry/utils/ChainGenerator.scala b/src/test/scala/encry/utils/ChainGenerator.scala index 0df84d3230..6c1a48f0e4 100644 --- a/src/test/scala/encry/utils/ChainGenerator.scala +++ b/src/test/scala/encry/utils/ChainGenerator.scala @@ -145,11 +145,11 @@ object ChainGenerator { val genesisBlock: Block = generateGenesisBlock(Height @@ 0) val state: UtxoState = utxoFromBoxHolder(BoxHolder(Seq.empty), dir, None, settings, VersionalStorage.LevelDB) - .applyModifier(genesisBlock).right.get + val genesisState = state.applyModifier(genesisBlock).right.get val (forkBlock, forkBoxes) = genForkBlock(transPerBlock, genesisBlock, genesisBlock.payload.txs.head.newBoxes.map(_.asInstanceOf[AssetBox]).head, randomAddress) - val forkState = state.applyModifier(forkBlock).right.get + val forkState = genesisState.applyModifier(forkBlock).right.get val (chain, _, newState, _) = (2 until blockQty).foldLeft(List(genesisBlock, forkBlock), forkBlock, forkState, forkBoxes) { From c3f8aeb2e1e78a2981a35878960f4bc35adbf5e0 Mon Sep 17 00:00:00 2001 From: capdev Date: Thu, 26 Sep 2019 11:57:03 +0500 Subject: [PATCH 20/39] cleaned up unnecessary changes --- .../test/scala/benches/HistoryBenches.scala | 2 +- .../SerializedAssetTransactionBenchmark.scala | 2 +- .../benches/SerializedDataTxBenchmark.scala | 2 +- .../SerializedMonetaryTxBenchmark.scala | 2 +- .../src/test/scala/benches/StateBenches.scala | 4 +- .../scala/benches/StateRollbackBench.scala | 4 +- .../src/test/scala/benches/Utils.scala | 81 +++---------------- .../benches/VersionalLevelDBBanches.scala | 4 +- .../encry/view/actors/HistoryApplicator.scala | 9 ++- .../encry/modifiers/InstanceFactory.scala | 5 +- .../history/HistoryApplicatorTest.scala | 3 +- 11 files changed, 30 insertions(+), 88 deletions(-) rename src/main/scala/encry/utils/ChainUtils.scala => benchmarks/src/test/scala/benches/Utils.scala (86%) diff --git a/benchmarks/src/test/scala/benches/HistoryBenches.scala b/benchmarks/src/test/scala/benches/HistoryBenches.scala index 02e6e23462..99e5e52ddf 100644 --- a/benchmarks/src/test/scala/benches/HistoryBenches.scala +++ b/benchmarks/src/test/scala/benches/HistoryBenches.scala @@ -4,7 +4,7 @@ import java.io.File import java.util.concurrent.TimeUnit import benches.HistoryBenches.HistoryBenchState -import encry.utils.ChainUtils.{generateHistory, generateNextBlockValidForHistory, getRandomTempDir} +import benches.Utils.{generateHistory, generateNextBlockValidForHistory, getRandomTempDir} import encry.EncryApp import encry.settings.EncryAppSettings import encry.view.history.History diff --git a/benchmarks/src/test/scala/benches/SerializedAssetTransactionBenchmark.scala b/benchmarks/src/test/scala/benches/SerializedAssetTransactionBenchmark.scala index 31f51f179d..df3e1047db 100644 --- a/benchmarks/src/test/scala/benches/SerializedAssetTransactionBenchmark.scala +++ b/benchmarks/src/test/scala/benches/SerializedAssetTransactionBenchmark.scala @@ -3,7 +3,7 @@ package benches import java.util.concurrent.TimeUnit import benches.SerializedAssetTransactionBenchmark.SerializedAssetBenchState -import encry.utils.ChainUtils._ +import benches.Utils._ import encryBenchmark.{BenchSettings, Settings} import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionSerializer} import org.encryfoundation.common.modifiers.state.box.AssetBox diff --git a/benchmarks/src/test/scala/benches/SerializedDataTxBenchmark.scala b/benchmarks/src/test/scala/benches/SerializedDataTxBenchmark.scala index 425700c263..8ab3ef0621 100644 --- a/benchmarks/src/test/scala/benches/SerializedDataTxBenchmark.scala +++ b/benchmarks/src/test/scala/benches/SerializedDataTxBenchmark.scala @@ -3,7 +3,7 @@ package benches import java.util.concurrent.TimeUnit import benches.SerializedDataTxBenchmark.SerializedDataBenchState -import encry.utils.ChainUtils._ +import benches.Utils._ import encryBenchmark.{BenchSettings, Settings} import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionSerializer} import org.encryfoundation.common.modifiers.state.box.AssetBox diff --git a/benchmarks/src/test/scala/benches/SerializedMonetaryTxBenchmark.scala b/benchmarks/src/test/scala/benches/SerializedMonetaryTxBenchmark.scala index 2525bd4507..404c5a9f8c 100644 --- a/benchmarks/src/test/scala/benches/SerializedMonetaryTxBenchmark.scala +++ b/benchmarks/src/test/scala/benches/SerializedMonetaryTxBenchmark.scala @@ -3,7 +3,7 @@ package benches import java.util.concurrent.TimeUnit import benches.SerializedMonetaryTxBenchmark.SerializedMonetaryBenchState -import encry.utils.ChainUtils._ +import benches.Utils._ import encryBenchmark.{BenchSettings, Settings} import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionSerializer} import org.encryfoundation.common.modifiers.state.box.AssetBox diff --git a/benchmarks/src/test/scala/benches/StateBenches.scala b/benchmarks/src/test/scala/benches/StateBenches.scala index e076e9ed41..9aa45760b2 100644 --- a/benchmarks/src/test/scala/benches/StateBenches.scala +++ b/benchmarks/src/test/scala/benches/StateBenches.scala @@ -5,7 +5,7 @@ import java.util.concurrent.TimeUnit import benches.StateBenches.StateBenchState import org.openjdk.jmh.annotations._ -import encry.utils.ChainUtils._ +import benches.Utils._ import encry.EncryApp import encry.settings.EncryAppSettings import encry.storage.VersionalStorage @@ -73,7 +73,7 @@ object StateBenches extends BenchSettings { ) val boxesHolder: BoxHolder = BoxHolder(initialBoxes) var state: UtxoState = utxoFromBoxHolder(boxesHolder, tmpDir, None, settings, VersionalStorage.LevelDB) - val genesisBlock: Block = generateGenesisBlockValidForState + val genesisBlock: Block = generateGenesisBlockValidForState(state) state = state.applyModifier(genesisBlock).right.get diff --git a/benchmarks/src/test/scala/benches/StateRollbackBench.scala b/benchmarks/src/test/scala/benches/StateRollbackBench.scala index 26eea4c59c..812fc160f9 100644 --- a/benchmarks/src/test/scala/benches/StateRollbackBench.scala +++ b/benchmarks/src/test/scala/benches/StateRollbackBench.scala @@ -4,7 +4,7 @@ import java.io.File import java.util.concurrent.TimeUnit import benches.StateRollbackBench.StateRollbackState -import encry.utils.ChainUtils._ +import benches.Utils._ import encry.storage.VersionalStorage import encry.utils.CoreTaggedTypes.VersionTag import encry.view.state.{BoxHolder, UtxoState} @@ -66,7 +66,7 @@ object StateRollbackBench extends BenchSettings { ) val boxesHolder: BoxHolder = BoxHolder(initialBoxes) var state: UtxoState = utxoFromBoxHolder(boxesHolder, tmpDir, None, settings, VersionalStorage.LevelDB) - val genesisBlock: Block = generateGenesisBlockValidForState + val genesisBlock: Block = generateGenesisBlockValidForState(state) state = state.applyModifier(genesisBlock).right.get diff --git a/src/main/scala/encry/utils/ChainUtils.scala b/benchmarks/src/test/scala/benches/Utils.scala similarity index 86% rename from src/main/scala/encry/utils/ChainUtils.scala rename to benchmarks/src/test/scala/benches/Utils.scala index beec64b427..0d8e54d3be 100644 --- a/src/main/scala/encry/utils/ChainUtils.scala +++ b/benchmarks/src/test/scala/benches/Utils.scala @@ -1,19 +1,17 @@ -package encry.utils +package benches import java.io.File import akka.actor.ActorRef import com.typesafe.scalalogging.StrictLogging -import encry.EncryApp.timeProvider -import encry.consensus.EncrySupplyController import encry.modifiers.mempool.TransactionFactory -import encry.settings.{EncryAppSettings, Settings} +import encry.settings.{Settings, EncryAppSettings} import encry.storage.VersionalStorage import encry.storage.VersionalStorage.{StorageKey, StorageType, StorageValue, StorageVersion} import encry.storage.iodb.versionalIODB.IODBWrapper import encry.storage.levelDb.versionalLevelDB.VersionalLevelDBCompanion.{LevelDBVersion, VersionalLevelDbKey, VersionalLevelDbValue} import encry.storage.levelDb.versionalLevelDB._ -import encry.utils.NetworkTime.Time +import encry.utils.{FileHelper, Mnemonic, NetworkTimeProvider} import encry.view.history.History import encry.view.history.storage.HistoryStorage import encry.view.state.{BoxHolder, UtxoState} @@ -36,7 +34,7 @@ import scorex.utils.Random import scala.collection.immutable import scala.util.{Random => R} -object ChainUtils extends Settings with StrictLogging { +object Utils extends Settings with StrictLogging { val mnemonicKey: String = "index another island accuse valid aerobic little absurd bunker keep insect scissors" val privKey: PrivateKey25519 = createPrivKey(Some(mnemonicKey)) @@ -63,7 +61,7 @@ object ChainUtils extends Settings with StrictLogging { ) :: acc } - def generateGenesisBlockValidForState: Block = { + def generateGenesisBlockValidForState(state: UtxoState): Block = { val txs = Seq(coinbaseTransaction(0)) val header = genHeader.copy( parentId = Header.GenesisParentId, @@ -72,26 +70,6 @@ object ChainUtils extends Settings with StrictLogging { Block(header, Payload(header.id, txs)) } - def generateGenesisBlock(genesisHeight: Height): Block = { - val coinbaseTrans = TransactionFactory.coinbaseTransactionScratch( - privKey.publicImage, - System.currentTimeMillis(), - settings.constants.InitialEmissionAmount, - amount = 0L, - height = genesisHeight - ) - - val txs: Seq[Transaction] = Seq(coinbaseTrans) - val txsRoot: Digest32 = Payload.rootHash(txs.map(_.id)) - val header = genHeader.copy( - timestamp = System.currentTimeMillis(), - parentId = Header.GenesisParentId, - height = genesisHeight, - transactionsRoot = txsRoot - ) - Block(header, Payload(header.id, txs)) - } - def generateGenesisBlockValidForHistory: Block = { val header = genHeader.copy(parentId = Header.GenesisParentId, height = settings.constants.GenesisHeight) Block(header, Payload(header.id, Seq(coinbaseTransaction))) @@ -136,14 +114,13 @@ object ChainUtils extends Settings with StrictLogging { box: Seq[AssetBox], splitCoef: Int = 2, addDiff: Difficulty = Difficulty @@ BigInt(0)): Block = { - val timestamp = System.currentTimeMillis() val transactions: Seq[Transaction] = box.indices.foldLeft(box, Seq.empty[Transaction]) { case ((boxes, transactionsL), _) => val tx: Transaction = defaultPaymentTransactionScratch( privKey, fee = 1, - timestamp, + timestamp = 11L, useBoxes = IndexedSeq(boxes.head), recipient = privKey.publicImage.address.address, amount = boxes.head.amount - 1, @@ -152,48 +129,11 @@ object ChainUtils extends Settings with StrictLogging { (boxes.tail, transactionsL :+ tx) }._2.filter(tx => state.validate(tx).isRight) ++ Seq(coinbaseTransaction(prevBlock.header.height + 1)) logger.info(s"Number of generated transactions: ${transactions.size}.") - - println(s"transactions.size: ${transactions.size}") - - val header = Header( - 1.toByte, - prevBlock.id, - Payload.rootHash(transactions.map(_.id)), - timestamp, - prevBlock.header.height + 1, - R.nextLong(), - Difficulty @@ (BigInt(1) + addDiff), - EquihashSolution(Seq(1, 3)) - ) - Block(header, Payload(header.id, transactions)) - } - - def generateNextBlockForStateWithInvalidTrans(prevBlock: Block, - state: UtxoState, - box: Seq[AssetBox], - splitCoef: Int = 2, - addDiff: Difficulty = Difficulty @@ BigInt(0)): Block = { - val timestamp = System.currentTimeMillis() - - val transactions: Seq[Transaction] = box.indices.foldLeft(box, Seq.empty[Transaction]) { - case ((boxes, transactionsL), _) => - val tx: Transaction = defaultPaymentTransactionScratch( - privKey, - fee = 1, - timestamp, - useBoxes = IndexedSeq(boxes.head), - recipient = privKey.publicImage.address.address, - amount = boxes.head.amount * 2, //invalid - numOfOutputs = splitCoef - ) - (boxes.tail, transactionsL :+ tx) - }._2.filter(tx => state.validate(tx).isRight) ++ Seq(coinbaseTransaction(prevBlock.header.height + 1)) - logger.info(s"Number of generated transactions: ${transactions.size}.") val header = Header( 1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), - timestamp, + System.currentTimeMillis(), prevBlock.header.height + 1, R.nextLong(), Difficulty @@ (BigInt(1) + addDiff), @@ -282,7 +222,7 @@ object ChainUtils extends Settings with StrictLogging { def coinbaseTransaction(height: Int): Transaction = TransactionFactory.coinbaseTransactionScratch( privKey.publicImage, System.currentTimeMillis(), - supply = settings.constants.InitialEmissionAmount, + supply = 10000000L, amount = 1L, height = Height @@ height ) @@ -459,6 +399,7 @@ object ChainUtils extends Settings with StrictLogging { } def generateHistory(settings: EncryAppSettings, file: File): History = { + val indexStore: LSMStore = new LSMStore(FileHelper.getRandomTempDir, keepVersions = 0) val objectsStore: LSMStore = new LSMStore(FileHelper.getRandomTempDir, keepVersions = 0) val levelDBInit = LevelDbFactory.factory.open(FileHelper.getRandomTempDir, new Options) @@ -468,8 +409,8 @@ object ChainUtils extends Settings with StrictLogging { val ntp: NetworkTimeProvider = new NetworkTimeProvider(settings.ntp) new History { - override val historyStorage: HistoryStorage = storage - override val timeProvider: NetworkTimeProvider = ntp + override val historyStorage: HistoryStorage = storage + override val timeProvider: NetworkTimeProvider = ntp } } diff --git a/benchmarks/src/test/scala/benches/VersionalLevelDBBanches.scala b/benchmarks/src/test/scala/benches/VersionalLevelDBBanches.scala index 896957536f..e8f8bebdec 100644 --- a/benchmarks/src/test/scala/benches/VersionalLevelDBBanches.scala +++ b/benchmarks/src/test/scala/benches/VersionalLevelDBBanches.scala @@ -4,7 +4,7 @@ import java.util.concurrent.TimeUnit import benches.VersionalLevelDBBanches.VersionalLevelDBState import encry.settings.LevelDBSettings import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VersionalLevelDBCompanion} -import encry.utils.{ChainUtils, FileHelper} +import encry.utils.FileHelper import org.iq80.leveldb.Options import org.openjdk.jmh.annotations.{Benchmark, Mode, Scope, State} import org.openjdk.jmh.infra.Blackhole @@ -55,7 +55,7 @@ object VersionalLevelDBBanches { //val elems1k = Utils.generateRandomLevelDbElemsWithoutDeletions(1000, 100) //val elems5k = Utils.generateRandomLevelDbElemsWithoutDeletions(5000, 100) - val elems10k = ChainUtils.generateRandomLevelDbElemsWithoutDeletions(10000, 100) + val elems10k = Utils.generateRandomLevelDbElemsWithoutDeletions(10000, 100) //val elems30k = Utils.generateRandomLevelDbElemsWithoutDeletions(30000, 100) } } \ No newline at end of file diff --git a/src/main/scala/encry/view/actors/HistoryApplicator.scala b/src/main/scala/encry/view/actors/HistoryApplicator.scala index d08eeef480..7ccf1c1f78 100644 --- a/src/main/scala/encry/view/actors/HistoryApplicator.scala +++ b/src/main/scala/encry/view/actors/HistoryApplicator.scala @@ -28,7 +28,7 @@ import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.immutable.Queue import scala.collection.mutable -class HistoryApplicator(val history: History, +class HistoryApplicator(history: History, state: UtxoState, wallet: EncryWallet, setting: EncryAppSettings, @@ -172,6 +172,10 @@ class HistoryApplicator(val history: History, context.system.eventStream.publish(DownloadRequest(tid, id)) } + type ModifierIdAsKey = scala.collection.mutable.WrappedArray.ofByte + + def toKey(id: ModifierId): ModifierIdAsKey = new mutable.WrappedArray.ofByte(id) + } object HistoryApplicator { @@ -194,9 +198,6 @@ object HistoryApplicator { case otherwise => 4 }) - type ModifierIdAsKey = scala.collection.mutable.WrappedArray.ofByte - def toKey(id: ModifierId): ModifierIdAsKey = new mutable.WrappedArray.ofByte(id) - def props(history: History, setting: EncryAppSettings, state: UtxoState, diff --git a/src/test/scala/encry/modifiers/InstanceFactory.scala b/src/test/scala/encry/modifiers/InstanceFactory.scala index 0cd2d3fa6b..0574fc894e 100755 --- a/src/test/scala/encry/modifiers/InstanceFactory.scala +++ b/src/test/scala/encry/modifiers/InstanceFactory.scala @@ -4,7 +4,7 @@ import encry.modifiers.mempool._ import encry.modifiers.state.Keys import encry.settings.{EncryAppSettings, NodeSettings} import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} -import encry.utils.{ChainUtils, EncryGenerator, FileHelper, NetworkTimeProvider, TestHelper} +import encry.utils.{EncryGenerator, FileHelper, NetworkTimeProvider, TestHelper} import encry.view.history.History import encry.view.history.storage.HistoryStorage import io.iohk.iodb.LSMStore @@ -14,6 +14,7 @@ import org.encryfoundation.common.modifiers.state.box.{AssetBox, EncryPropositio import org.encryfoundation.common.modifiers.state.box.Box.Amount import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{Height, _} + import org.encryfoundation.prismlang.compiler.CompiledContract import org.encryfoundation.prismlang.core.Ast.Expr import org.encryfoundation.prismlang.core.{Ast, Types} @@ -46,7 +47,7 @@ trait InstanceFactory extends Keys with EncryGenerator { } def generateGenesisBlock(genesisHeight: Height): Block = { - val txs: Seq[Transaction] = Seq(ChainUtils.coinbaseTransaction(0)) + val txs: Seq[Transaction] = Seq(coinbaseTransaction) val txsRoot: Digest32 = Payload.rootHash(txs.map(_.id)) val header = genHeader.copy( parentId = Header.GenesisParentId, diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index c82a80b5c9..5570c25c4a 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -89,7 +89,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w chain.foreach(historyApplicator ! LocallyGeneratedBlock(_)) checkFullBlockChainIsSynced(blockQty * 2) - //receiveN(blockQty * 2, 30 seconds) + history.getBestBlockHeight shouldBe blockQty - 1 } @@ -130,7 +130,6 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w .flatMap(blockToModifiers) .foreach(historyApplicator ! ModifierFromRemote(_)) - //println(s"modifiersQueue.size: ${historyApplicator.underlyingActor.currentNumberOfAppliedModifiers}") historyApplicator.underlyingActor.modifiersQueue.size should be <= settings.levelDB.maxVersions receiveN((settings.levelDB.maxVersions + overQty) * 2, 30 seconds) From 06287190e5bd171243ca4605a0d6ed57a4297fbe Mon Sep 17 00:00:00 2001 From: capdev Date: Thu, 26 Sep 2019 12:29:16 +0500 Subject: [PATCH 21/39] change test params for travis build --- .../history/HistoryApplicatorTest.scala | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 5570c25c4a..53845b496b 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -46,7 +46,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w .flatMap(blockToModifiers) .foreach(historyApplicator ! ModifierFromRemote(_)) - receiveN(6 * 2, 30 seconds) + receiveN(6 * 2, 120 seconds) history.getBestBlockHeight shouldBe 4 history.getBestBlock.map(b => Algos.encode(b.id)) shouldBe Some(Algos.encode(chain(4).id)) @@ -73,7 +73,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w "HistoryApplicator" should { - "apply locall blocks and chain sync" in { + "apply locall blocks and check chain sync" in { val blockQty = 10 val history: History = generateDummyHistory(settings) @@ -88,29 +88,34 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w chain.foreach(historyApplicator ! LocallyGeneratedBlock(_)) - checkFullBlockChainIsSynced(blockQty * 2) + expectMsg(timeout, FullBlockChainIsSynced()) history.getBestBlockHeight shouldBe blockQty - 1 } - "apply remote blocks and check SemanticallySuccessfulModifier" in { + "apply remote blocks and check chain sync" in { - val blockQty = 2 - val initialHistory: History = generateDummyHistory(settings) + val blockQty = 10 + val history: History = generateDummyHistory(settings) val (initialState, state, chain) = genChain(privKey, dir, settings, blockQty) val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( - HistoryApplicator.props(initialHistory, settings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) + HistoryApplicator.props(history, settings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) - system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) + system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) - receiveN(blockQty * 2, timeout).forall { case _: SemanticallySuccessfulModifier => true } +// receiveN(blockQty * 2, timeout).forall { case _: SemanticallySuccessfulModifier => true } +// chain.foreach(historyApplicator ! LocallyGeneratedBlock(_)) + + expectMsg(timeout, FullBlockChainIsSynced()) + + history.getBestBlockHeight shouldBe blockQty - 1 } "apply remote blocks and check queue for rollback height" in { @@ -132,7 +137,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w historyApplicator.underlyingActor.modifiersQueue.size should be <= settings.levelDB.maxVersions - receiveN((settings.levelDB.maxVersions + overQty) * 2, 30 seconds) + receiveN((settings.levelDB.maxVersions + overQty) * 2, 120 seconds) history.getBestBlockHeight shouldBe settings.levelDB.maxVersions + overQty - 1 } From 352db2e533ae5a322adc9036460a8f8437d47109 Mon Sep 17 00:00:00 2001 From: capdev Date: Thu, 26 Sep 2019 12:52:15 +0500 Subject: [PATCH 22/39] change test params for travis build --- .../modifiers/history/HistoryApplicatorTest.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 53845b496b..b48f7df2cc 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -46,7 +46,8 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w .flatMap(blockToModifiers) .foreach(historyApplicator ! ModifierFromRemote(_)) - receiveN(6 * 2, 120 seconds) + Thread.sleep(5000 + transQty/30) + //receiveN(6 * 2, 120 seconds) history.getBestBlockHeight shouldBe 4 history.getBestBlock.map(b => Algos.encode(b.id)) shouldBe Some(Algos.encode(chain(4).id)) @@ -110,11 +111,9 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) -// receiveN(blockQty * 2, timeout).forall { case _: SemanticallySuccessfulModifier => true } -// chain.foreach(historyApplicator ! LocallyGeneratedBlock(_)) + Thread.sleep(5000) expectMsg(timeout, FullBlockChainIsSynced()) - history.getBestBlockHeight shouldBe blockQty - 1 } @@ -137,7 +136,8 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w historyApplicator.underlyingActor.modifiersQueue.size should be <= settings.levelDB.maxVersions - receiveN((settings.levelDB.maxVersions + overQty) * 2, 120 seconds) + Thread.sleep(10000) + //receiveN((settings.levelDB.maxVersions + overQty) * 2, 120 seconds) history.getBestBlockHeight shouldBe settings.levelDB.maxVersions + overQty - 1 } From eb39bef0cb4c50828c18367e395a17594dca477f Mon Sep 17 00:00:00 2001 From: capdev Date: Thu, 26 Sep 2019 14:08:06 +0500 Subject: [PATCH 23/39] HistoryDummy for PowScheme validate --- .../view/history/HistoryModifiersValidator.scala | 6 +++--- .../scala/encry/modifiers/InstanceFactory.scala | 16 +++++++++++++++- .../history/HistoryApplicatorTest.scala | 2 +- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala index 9d46b017c1..8951496ab6 100644 --- a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala +++ b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala @@ -72,9 +72,9 @@ trait HistoryModifiersValidator extends HistoryApi with Settings { _ <- Either.cond(heightOf(h.parentId).exists(h => getBestHeaderHeight - h < settings.constants.MaxRollbackDepth), (), HeaderFatalValidationError(s"Header ${h.encodedId} has height greater than max roll back depth")) powSchemeValidationResult = powScheme.verify(h) -// _ <- Either.cond(powSchemeValidationResult.isRight, (), -// HeaderFatalValidationError(s"Wrong proof-of-work solution in header ${h.encodedId}" + -// s" caused: $powSchemeValidationResult")) + _ <- Either.cond(powSchemeValidationResult.isRight, (), + HeaderFatalValidationError(s"Wrong proof-of-work solution in header ${h.encodedId}" + + s" caused: $powSchemeValidationResult")) _ <- Either.cond(isSemanticallyValid(h.parentId) != ModifierSemanticValidity.Invalid, (), HeaderFatalValidationError(s"Header ${h.encodedId} is semantically invalid")) _ <- Either.cond(h.timestamp - timeProvider.estimatedTime <= settings.constants.MaxTimeDrift, (), diff --git a/src/test/scala/encry/modifiers/InstanceFactory.scala b/src/test/scala/encry/modifiers/InstanceFactory.scala index 0574fc894e..0c29067280 100755 --- a/src/test/scala/encry/modifiers/InstanceFactory.scala +++ b/src/test/scala/encry/modifiers/InstanceFactory.scala @@ -1,5 +1,7 @@ package encry.modifiers +import encry.consensus.EquihashPowScheme +import encry.crypto.equihash.{Equihash, EquihashValidationErrors} import encry.modifiers.mempool._ import encry.modifiers.state.Keys import encry.settings.{EncryAppSettings, NodeSettings} @@ -14,7 +16,6 @@ import org.encryfoundation.common.modifiers.state.box.{AssetBox, EncryPropositio import org.encryfoundation.common.modifiers.state.box.Box.Amount import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{Height, _} - import org.encryfoundation.prismlang.compiler.CompiledContract import org.encryfoundation.prismlang.core.Ast.Expr import org.encryfoundation.prismlang.core.{Ast, Types} @@ -22,6 +23,7 @@ import org.iq80.leveldb.Options import scorex.crypto.hash.Digest32 import scorex.utils.Random +import scala.math.BigInt import scala.util.{Random => Scarand} trait InstanceFactory extends Keys with EncryGenerator { @@ -213,6 +215,8 @@ trait InstanceFactory extends Keys with EncryGenerator { }._2 } + + def generateDummyHistory(settings: EncryAppSettings): History = { val indexStore: LSMStore = new LSMStore(FileHelper.getRandomTempDir, keepVersions = 0) @@ -223,9 +227,19 @@ trait InstanceFactory extends Keys with EncryGenerator { val ntp: NetworkTimeProvider = new NetworkTimeProvider(settings.ntp) + class EquihashPowSchemeWithoutValidateSolution(n: Char, k: Char, version: Byte, preGenesisHeight: Height, maxTarget: BigInt) + extends EquihashPowScheme(n: Char, k: Char, version: Byte, preGenesisHeight: Height, maxTarget: BigInt) { + override def verify(header: Header): Either[EquihashValidationErrors, Boolean] = Right(true) + } + + val equihashPowSchemeWithoutValidateSolution: EquihashPowScheme = + new EquihashPowSchemeWithoutValidateSolution(settings.constants.n, settings.constants.k, + settings.constants.Version, settings.constants.PreGenesisHeight, settings.constants.MaxTarget) + new History { override val historyStorage: HistoryStorage = storage override val timeProvider: NetworkTimeProvider = ntp + override val powScheme: EquihashPowScheme = equihashPowSchemeWithoutValidateSolution } } } \ No newline at end of file diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index b48f7df2cc..24eea8eb0e 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -46,7 +46,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w .flatMap(blockToModifiers) .foreach(historyApplicator ! ModifierFromRemote(_)) - Thread.sleep(5000 + transQty/30) + Thread.sleep(3000 + transQty/30) //receiveN(6 * 2, 120 seconds) history.getBestBlockHeight shouldBe 4 From e6ec35c9f07c01090dfad74422264bc6e576c800 Mon Sep 17 00:00:00 2001 From: capdev Date: Thu, 26 Sep 2019 14:48:00 +0500 Subject: [PATCH 24/39] override settings state = LevelDB --- .../history/HistoryApplicatorTest.scala | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 24eea8eb0e..aaedb94f9a 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -7,7 +7,8 @@ import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} import encry.modifiers.InstanceFactory import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier -import encry.settings.Settings +import encry.settings.{EncryAppSettings, Settings} +import encry.storage.VersionalStorage import encry.utils.ChainGenerator._ import encry.utils.FileHelper import encry.view.actors.HistoryApplicator @@ -23,22 +24,23 @@ import scala.collection.Seq import scala.concurrent.duration._ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike with ImplicitSender with BeforeAndAfterAll with Matchers - with InstanceFactory with OneInstancePerTest with Settings { + with InstanceFactory with OneInstancePerTest with Settings { override def afterAll: Unit = { TestKit.shutdownActorSystem(system) } + val testSettings: EncryAppSettings = settings.copy(storage = settings.storage.copy(state = VersionalStorage.LevelDB)) def blockToModifiers(block: Block): Seq[PersistentModifier] = Seq(block.header, block.payload) def rollbackTest(transQty: Int) { - val history: History = generateDummyHistory(settings) + val history: History = generateDummyHistory(testSettings) //generate invalid blocks begining from 6 blocks - val (initialState, state, chain) = genChain(privKey, dir, settings, 10, transQty, Some(6)) + val (initialState, state, chain) = genChain(privKey, dir, testSettings, 10, transQty, Some(6)) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( - HistoryApplicator.props(history, settings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) + HistoryApplicator.props(history, testSettings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) @@ -65,7 +67,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w def checkFullBlockChainIsSynced(qty: Int): Unit = (0 until qty).foreach(_ => expectMsg(timeout, FullBlockChainIsSynced())) val dir: File = FileHelper.getRandomTempDir - val wallet: EncryWallet = EncryWallet.readOrGenerate(settings.copy(directory = dir.getAbsolutePath)) + val wallet: EncryWallet = EncryWallet.readOrGenerate(testSettings.copy(directory = dir.getAbsolutePath)) val nodeViewHolder = TestProbe() val influx = TestProbe() @@ -77,12 +79,12 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w "apply locall blocks and check chain sync" in { val blockQty = 10 - val history: History = generateDummyHistory(settings) - val (initialState, state, chain) = genChain(privKey, dir, settings, blockQty) + val history: History = generateDummyHistory(testSettings) + val (initialState, state, chain) = genChain(privKey, dir, testSettings, blockQty) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( - HistoryApplicator.props(history, settings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) + HistoryApplicator.props(history, testSettings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) @@ -97,21 +99,21 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w "apply remote blocks and check chain sync" in { val blockQty = 10 - val history: History = generateDummyHistory(settings) - val (initialState, state, chain) = genChain(privKey, dir, settings, blockQty) + val history: History = generateDummyHistory(testSettings) + val (initialState, state, chain) = genChain(privKey, dir, testSettings, blockQty) val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( - HistoryApplicator.props(history, settings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) + HistoryApplicator.props(history, testSettings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) - Thread.sleep(5000) + Thread.sleep(3000) expectMsg(timeout, FullBlockChainIsSynced()) history.getBestBlockHeight shouldBe blockQty - 1 @@ -121,12 +123,12 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w val overQty = 30 - val history: History = generateDummyHistory(settings) - val (initialState, state, chain) = genChain(privKey, dir, settings, settings.levelDB.maxVersions + overQty, 10) + val history: History = generateDummyHistory(testSettings) + val (initialState, state, chain) = genChain(privKey, dir, testSettings, testSettings.levelDB.maxVersions + overQty, 10) val historyApplicator: TestActorRef[HistoryApplicator] = TestActorRef[HistoryApplicator]( - HistoryApplicator.props(history, settings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) + HistoryApplicator.props(history, testSettings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) @@ -134,12 +136,12 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w .flatMap(blockToModifiers) .foreach(historyApplicator ! ModifierFromRemote(_)) - historyApplicator.underlyingActor.modifiersQueue.size should be <= settings.levelDB.maxVersions + historyApplicator.underlyingActor.modifiersQueue.size should be <= testSettings.levelDB.maxVersions - Thread.sleep(10000) - //receiveN((settings.levelDB.maxVersions + overQty) * 2, 120 seconds) + Thread.sleep(5000) + //receiveN((testSettings.levelDB.maxVersions + overQty) * 2, 120 seconds) - history.getBestBlockHeight shouldBe settings.levelDB.maxVersions + overQty - 1 + history.getBestBlockHeight shouldBe testSettings.levelDB.maxVersions + overQty - 1 } "apply remote blocks and check history rollback for invalid state" in { From 57ff59505821a126ff339c1d9cd7f8aa4b8d8838 Mon Sep 17 00:00:00 2001 From: capdev Date: Thu, 26 Sep 2019 15:03:17 +0500 Subject: [PATCH 25/39] change test params for travis build --- .../scala/encry/modifiers/history/HistoryApplicatorTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index aaedb94f9a..4adb4a2a73 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -48,7 +48,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w .flatMap(blockToModifiers) .foreach(historyApplicator ! ModifierFromRemote(_)) - Thread.sleep(3000 + transQty/30) + Thread.sleep(3000 + transQty) //receiveN(6 * 2, 120 seconds) history.getBestBlockHeight shouldBe 4 From 92281c6ddbde0b223958e88e190abd7a1e1b3a2b Mon Sep 17 00:00:00 2001 From: capdev Date: Thu, 26 Sep 2019 16:12:15 +0500 Subject: [PATCH 26/39] replace sleep to receiveN --- .../history/HistoryApplicatorTest.scala | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 4adb4a2a73..e04d27f559 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -48,8 +48,8 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w .flatMap(blockToModifiers) .foreach(historyApplicator ! ModifierFromRemote(_)) - Thread.sleep(3000 + transQty) - //receiveN(6 * 2, 120 seconds) + //Thread.sleep(3000 + transQty) + receiveN(6 * 2, timeout) history.getBestBlockHeight shouldBe 4 history.getBestBlock.map(b => Algos.encode(b.id)) shouldBe Some(Algos.encode(chain(4).id)) @@ -64,7 +64,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w // prevHistory.reportModifierIsValid(block) // } - def checkFullBlockChainIsSynced(qty: Int): Unit = (0 until qty).foreach(_ => expectMsg(timeout, FullBlockChainIsSynced())) + def checkFullBlockChainIsSynced(qty: Int): Unit = (0 until qty).foreach(_ => expectMsg(30 seconds, FullBlockChainIsSynced())) val dir: File = FileHelper.getRandomTempDir val wallet: EncryWallet = EncryWallet.readOrGenerate(testSettings.copy(directory = dir.getAbsolutePath)) @@ -72,7 +72,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w val nodeViewHolder = TestProbe() val influx = TestProbe() - val timeout: FiniteDuration = 10 seconds + val timeout: FiniteDuration = 30 seconds "HistoryApplicator" should { @@ -91,7 +91,9 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w chain.foreach(historyApplicator ! LocallyGeneratedBlock(_)) - expectMsg(timeout, FullBlockChainIsSynced()) + receiveN((blockQty - 1) * 2, timeout) + //checkFullBlockChainIsSynced((blockQty - 1) * 2) + //expectMsg(timeout, FullBlockChainIsSynced()) history.getBestBlockHeight shouldBe blockQty - 1 } @@ -113,9 +115,10 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) - Thread.sleep(3000) - - expectMsg(timeout, FullBlockChainIsSynced()) + //Thread.sleep(3000) + receiveN((blockQty - 1) * 2, timeout) + //checkFullBlockChainIsSynced((blockQty - 1) * 2) + //expectMsg(timeout, FullBlockChainIsSynced()) history.getBestBlockHeight shouldBe blockQty - 1 } @@ -138,8 +141,9 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w historyApplicator.underlyingActor.modifiersQueue.size should be <= testSettings.levelDB.maxVersions - Thread.sleep(5000) - //receiveN((testSettings.levelDB.maxVersions + overQty) * 2, 120 seconds) + //Thread.sleep(5000) + //checkFullBlockChainIsSynced((testSettings.levelDB.maxVersions + overQty) * 2) + receiveN((testSettings.levelDB.maxVersions + overQty) * 2, timeout) history.getBestBlockHeight shouldBe testSettings.levelDB.maxVersions + overQty - 1 } From 57f252e397dff09f1dd8b47690d9884977aa2079 Mon Sep 17 00:00:00 2001 From: capdev Date: Thu, 26 Sep 2019 16:24:33 +0500 Subject: [PATCH 27/39] change test params for travis build --- .../scala/encry/modifiers/history/HistoryApplicatorTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index e04d27f559..22e455a9e2 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -116,7 +116,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) //Thread.sleep(3000) - receiveN((blockQty - 1) * 2, timeout) + receiveN(8 * 2, timeout) //checkFullBlockChainIsSynced((blockQty - 1) * 2) //expectMsg(timeout, FullBlockChainIsSynced()) history.getBestBlockHeight shouldBe blockQty - 1 From 6a7a59dd25b057950bbab84e2c03fde4ded67f6a Mon Sep 17 00:00:00 2001 From: capdev Date: Thu, 26 Sep 2019 16:35:03 +0500 Subject: [PATCH 28/39] change test params for travis build --- .../scala/encry/modifiers/history/HistoryApplicatorTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 22e455a9e2..6097b74c7a 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -116,7 +116,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) //Thread.sleep(3000) - receiveN(8 * 2, timeout) + receiveN(6 * 2, timeout) //checkFullBlockChainIsSynced((blockQty - 1) * 2) //expectMsg(timeout, FullBlockChainIsSynced()) history.getBestBlockHeight shouldBe blockQty - 1 From 42a72b7b49694ff7ea863d605174ea25762ecdad Mon Sep 17 00:00:00 2001 From: capdev Date: Thu, 26 Sep 2019 18:40:40 +0500 Subject: [PATCH 29/39] cleanup --- .../encry/view/actors/HistoryApplicator.scala | 45 ++-- .../encry/view/actors/StateApplicator.scala | 6 - .../encry/modifiers/InstanceFactory.scala | 6 +- .../scala/encry/utils/ChainGenerator.scala | 197 +++++++++--------- 4 files changed, 114 insertions(+), 140 deletions(-) diff --git a/src/main/scala/encry/view/actors/HistoryApplicator.scala b/src/main/scala/encry/view/actors/HistoryApplicator.scala index 7ccf1c1f78..c1950e31cd 100644 --- a/src/main/scala/encry/view/actors/HistoryApplicator.scala +++ b/src/main/scala/encry/view/actors/HistoryApplicator.scala @@ -49,12 +49,10 @@ class HistoryApplicator(history: History, override def receive: Receive = { case ModifierFromRemote(mod) if !history.isModifierDefined(mod.id) && !ModifiersCache.contains(toKey(mod.id)) => - //println("ModifierFromRemote") ModifiersCache.put(toKey(mod.id), mod, history) getModifierForApplying() case ModifierFromRemote(modifier) => - //println(s"history.isModifierDefined(mod.id): ${history.isModifierDefined(modifier.id)} ModifiersCache.contains(toKey(mod.id): ${ModifiersCache.contains(toKey(modifier.id))}") logger.info(s"Modifier ${modifier.encodedId} contains in history or in modifiers cache. Reject it.") case LocallyGeneratedBlock(block) => @@ -74,12 +72,10 @@ class HistoryApplicator(history: History, currentNumberOfAppliedModifiers -= 1 logger.info(s"Modifier ${modifier.encodedId} unsuccessfully applied to history with exception ${ex.getMessage}." + s" Current currentNumberOfAppliedModifiers $currentNumberOfAppliedModifiers.") - println("SyntacticallyFailedModification") context.system.eventStream.publish(SyntacticallyFailedModification(modifier, List(HistoryApplyError(ex.getMessage)))) case Right(progressInfo) if progressInfo.toApply.nonEmpty => logger.info(s"Modifier ${modifier.encodedId} successfully applied to history.") modifiersQueue = modifiersQueue.enqueue(modifier.encodedId -> progressInfo) - println(s"modifiersQueue.enqueue ${modifiersQueue.size}") logger.info(s"New element put into queue. Current queue size is ${modifiersQueue.length}." + s"Current number of applied modifiers is $currentNumberOfAppliedModifiers.") influxRef.foreach(ref => @@ -92,13 +88,11 @@ class HistoryApplicator(history: History, walletApplicator ! WalletNeedRollbackTo(VersionTag !@@ progressInfo.branchPoint.get) if (progressInfo.toRemove.nonEmpty) nodeViewHolder ! TransactionsForWallet(progressInfo.toRemove) - //println("history.append success") stateApplicator ! NotificationAboutNewModifier getModifierForApplying() case Right(progressInfo) => logger.info(s"Progress info is empty after appending to the state.") if (!isLocallyGenerated) requestDownloads(progressInfo) - println("SemanticallySuccessfulModifier") context.system.eventStream.publish(SemanticallySuccessfulModifier(modifier)) currentNumberOfAppliedModifiers -= 1 getModifierForApplying() @@ -111,13 +105,11 @@ class HistoryApplicator(history: History, sender() ! StartModifiersApplicationOnStateApplicator(pi, IndexedSeq.empty[PersistentModifier]) modifiersQueue = newQueue } - println(s"modifiersQueue.dequeue ${modifiersQueue.size}") case NotificationAboutSuccessfullyAppliedModifier => if (history.isFullChainSynced) { logger.info(s"BlockChain is synced on state applicator at height ${history.getBestHeaderHeight}!") ModifiersCache.setChainSynced() - println("FullBlockChainIsSynced") context.system.eventStream.publish(FullBlockChainIsSynced()) } currentNumberOfAppliedModifiers -= 1 @@ -131,44 +123,37 @@ class HistoryApplicator(history: History, getModifierForApplying() case NeedToReportAsInValid(block) => - //println(s"NeedToReportAsInValid") logger.info(s"History got message NeedToReportAsInValid for block ${block.encodedId}.") currentNumberOfAppliedModifiers -= 1 val (_, newProgressInfo: ProgressInfo) = history.reportModifierIsInvalid(block) sender() ! NewProgressInfoAfterMarkingAsInValid(newProgressInfo) - //println(s"ha history.height: ${history.getBestBlockHeight}") case nonsense => logger.info(s"History applicator actor got from $sender message $nonsense.") } - def getModifierForApplying(): Unit = { - //println(s"currentNumberOfAppliedModifiers: $currentNumberOfAppliedModifiers") - //println(s"modifiersQueue.size: ${modifiersQueue.size}") - if (currentNumberOfAppliedModifiers < setting.levelDB.maxVersions) { - logger.debug(s"It's possible to append new modifier to history. Trying to get new one from the cache.") - if (locallyGeneratedModifiers.nonEmpty) locallyGeneratedModifiers.dequeueOption.foreach { - case (modifier, newQueue) => - locallyGeneratedModifiers = newQueue - currentNumberOfAppliedModifiers += 1 - logger.debug(s"Found new local modifier ${modifier.encodedId} with type ${modifier.modifierTypeId}." + - s"currentNumberOfAppliedModifiers is $currentNumberOfAppliedModifiers." + - s" Current locally generated modifiers size ${locallyGeneratedModifiers.size}.") - self ! ModifierToHistoryAppending(modifier, isLocallyGenerated = true) - } - else ModifiersCache.popCandidate(history).foreach { modifier => + def getModifierForApplying(): Unit = if (currentNumberOfAppliedModifiers < setting.levelDB.maxVersions) { + logger.debug(s"It's possible to append new modifier to history. Trying to get new one from the cache.") + if (locallyGeneratedModifiers.nonEmpty) locallyGeneratedModifiers.dequeueOption.foreach { + case (modifier, newQueue) => + locallyGeneratedModifiers = newQueue currentNumberOfAppliedModifiers += 1 - logger.debug(s"Found new modifier ${modifier.encodedId} with type ${modifier.modifierTypeId}." + + logger.debug(s"Found new local modifier ${modifier.encodedId} with type ${modifier.modifierTypeId}." + s"currentNumberOfAppliedModifiers is $currentNumberOfAppliedModifiers." + - s" Current mod cache size ${ModifiersCache.size}") - self ! ModifierToHistoryAppending(modifier) - } + s" Current locally generated modifiers size ${locallyGeneratedModifiers.size}.") + self ! ModifierToHistoryAppending(modifier, isLocallyGenerated = true) + } + else ModifiersCache.popCandidate(history).foreach { modifier => + currentNumberOfAppliedModifiers += 1 + logger.debug(s"Found new modifier ${modifier.encodedId} with type ${modifier.modifierTypeId}." + + s"currentNumberOfAppliedModifiers is $currentNumberOfAppliedModifiers." + + s" Current mod cache size ${ModifiersCache.size}") + self ! ModifierToHistoryAppending(modifier) } } def requestDownloads(pi: ProgressInfo): Unit = pi.toDownload.foreach { case (tid, id) => if (tid != Transaction.modifierTypeId) logger.debug(s"HistoryApplicator call requestDownloads for modifier ${Algos.encode(id)} of type $tid") - println("DownloadRequest") context.system.eventStream.publish(DownloadRequest(tid, id)) } diff --git a/src/main/scala/encry/view/actors/StateApplicator.scala b/src/main/scala/encry/view/actors/StateApplicator.scala index ccc48d313d..eb450cdf62 100644 --- a/src/main/scala/encry/view/actors/StateApplicator.scala +++ b/src/main/scala/encry/view/actors/StateApplicator.scala @@ -47,7 +47,6 @@ class StateApplicator(settings: EncryAppSettings, case header: Header => state.lastBlockTimestamp = header.timestamp historyApplicator ! NeedToReportAsValid(header) - println("stateApplicator.SemanticallySuccessfulModifier") context.system.eventStream.publish(SemanticallySuccessfulModifier(header)) val newToApply: List[PersistentModifier] = toApply.drop(1) if (newToApply.nonEmpty) { @@ -135,7 +134,6 @@ class StateApplicator(settings: EncryAppSettings, combinedStateChange.outputsToDb.toList, combinedStateChange.inputsToDb.toList ) - println("stateApplicator.SemanticallySuccessfulModifier") context.system.eventStream.publish(SemanticallySuccessfulModifier(block)) historyApplicator ! NeedToReportAsValid(block) influxRef.foreach { ref => @@ -158,7 +156,6 @@ class StateApplicator(settings: EncryAppSettings, case TransactionValidatedFailure(tx, ex) => logger.info(s"Transaction ${tx.encodedId} failed in validation by state.") context.children.foreach(_ ! Kill) - println("stateApplicator.SemanticallyFailedModification") context.system.eventStream.publish(SemanticallyFailedModification(block, List(StateModifierApplyError(s"$ex")))) historyApplicator ! NeedToReportAsInValid(block) context.become(awaitingNewProgressInfo(block, ui, toApply)) @@ -201,12 +198,10 @@ class StateApplicator(settings: EncryAppSettings, else Success(state) -> suffixApplied stateToApplyTry match { case Failure(exception) => - println("stateApplicator.RollbackFailed") context.system.eventStream.publish(RollbackFailed(branchPointOpt)) EncryApp.forceStopApplication(500, s"Rollback failed: $exception.") case Success(stateToApply) => logger.info(s"Successfully applied to the state. Starting modifiers applying.") - println("stateApplicator.RollbackSucceed") context.system.eventStream.publish(RollbackSucceed(branchPointOpt)) self ! StartModifiersApplying context.become(modifierApplication( @@ -228,7 +223,6 @@ class StateApplicator(settings: EncryAppSettings, def requestDownloads(pi: ProgressInfo): Unit = pi.toDownload.foreach { case (tid, id) => if (tid != Transaction.modifierTypeId) logger.debug(s"StateApplicator call requestDownloads for modifier ${Algos.encode(id)} of type $tid") - println("stateApplicator.DownloadRequest") context.system.eventStream.publish(DownloadRequest(tid, id)) } } diff --git a/src/test/scala/encry/modifiers/InstanceFactory.scala b/src/test/scala/encry/modifiers/InstanceFactory.scala index 0c29067280..2610aea5ec 100755 --- a/src/test/scala/encry/modifiers/InstanceFactory.scala +++ b/src/test/scala/encry/modifiers/InstanceFactory.scala @@ -1,7 +1,7 @@ package encry.modifiers import encry.consensus.EquihashPowScheme -import encry.crypto.equihash.{Equihash, EquihashValidationErrors} +import encry.crypto.equihash.EquihashValidationErrors import encry.modifiers.mempool._ import encry.modifiers.state.Keys import encry.settings.{EncryAppSettings, NodeSettings} @@ -16,6 +16,7 @@ import org.encryfoundation.common.modifiers.state.box.{AssetBox, EncryPropositio import org.encryfoundation.common.modifiers.state.box.Box.Amount import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{Height, _} + import org.encryfoundation.prismlang.compiler.CompiledContract import org.encryfoundation.prismlang.core.Ast.Expr import org.encryfoundation.prismlang.core.{Ast, Types} @@ -23,7 +24,6 @@ import org.iq80.leveldb.Options import scorex.crypto.hash.Digest32 import scorex.utils.Random -import scala.math.BigInt import scala.util.{Random => Scarand} trait InstanceFactory extends Keys with EncryGenerator { @@ -215,8 +215,6 @@ trait InstanceFactory extends Keys with EncryGenerator { }._2 } - - def generateDummyHistory(settings: EncryAppSettings): History = { val indexStore: LSMStore = new LSMStore(FileHelper.getRandomTempDir, keepVersions = 0) diff --git a/src/test/scala/encry/utils/ChainGenerator.scala b/src/test/scala/encry/utils/ChainGenerator.scala index 6c1a48f0e4..c884f0e9be 100644 --- a/src/test/scala/encry/utils/ChainGenerator.scala +++ b/src/test/scala/encry/utils/ChainGenerator.scala @@ -27,7 +27,6 @@ import scorex.crypto.signatures.PublicKey import scala.collection.Seq import scorex.utils.{Random => ScorexRandom} - import scala.util.Random object ChainGenerator { @@ -35,131 +34,129 @@ object ChainGenerator { def genChain(privKey: PrivateKey25519, dir: File, settings: EncryAppSettings, blockQty: Int, transPerBlock: Int = 1, genInvalidBlockFrom: Option[Int] = None): (UtxoState, UtxoState, List[Block]) = { - def utxoFromBoxHolder(bh: BoxHolder, dir: File, nodeViewHolderRef: Option[ActorRef], settings: EncryAppSettings, - storageType: StorageType): UtxoState = { - val storage = settings.storage.state match { - case VersionalStorage.IODB => - IODBWrapper(new LSMStore(dir, keepVersions = settings.constants.DefaultKeepVersions)) - case VersionalStorage.LevelDB => - val levelDBInit = LevelDbFactory.factory.open(dir, new Options) - VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settings.levelDB, keySize = 32)) - } + def randomAddress: Address = Pay2PubKeyAddress(PublicKey @@ ScorexRandom.randomBytes()).address - storage.insert( - StorageVersion @@ Array.fill(32)(0: Byte), - bh.boxes.values.map(bx => (StorageKey !@@ bx.id, StorageValue @@ bx.bytes)).toList - ) + assert(blockQty >= 2, "chain at least 2 blocks") - new UtxoState(storage, settings.constants) - } + val genesisBlock: Block = genGenesisBlock(privKey.publicImage, settings.constants.InitialEmissionAmount, + settings.constants.InitialDifficulty, Height @@ 0) - def paymentTransactionWithMultipleOutputs(privKey: PrivateKey25519, fee: Amount, timestamp: Long, useBoxes: IndexedSeq[MonetaryBox], - recipient: Address, amount: Amount, tokenIdOpt: Option[ADKey] = None, - numOfOutputs: Int): Transaction = { + val state: UtxoState = utxoFromBoxHolder(BoxHolder(Seq.empty), dir, None, settings, VersionalStorage.LevelDB) + val genesisState = state.applyModifier(genesisBlock).right.get - val pubKey: PublicKey25519 = privKey.publicImage - val uInputs: IndexedSeq[Input] = useBoxes - .map(bx => Input.unsigned(bx.id, Right(PubKeyLockedContract(pubKey.pubKeyBytes)))) - .toIndexedSeq + val (forkBlock, forkBoxes) = + genForkBlock(privKey, settings.constants.InitialEmissionAmount, transPerBlock, genesisBlock, + genesisBlock.payload.txs.head.newBoxes.map(_.asInstanceOf[AssetBox]).head, randomAddress) + val forkState = genesisState.applyModifier(forkBlock).right.get - val change: Amount = useBoxes.map(_.amount).sum - (amount + fee) - val directives: IndexedSeq[TransferDirective] = - if (change > 0) TransferDirective(recipient, amount, tokenIdOpt) +: (0 until numOfOutputs).map(_ => - TransferDirective(pubKey.address.address, change / numOfOutputs, tokenIdOpt)) - else IndexedSeq(TransferDirective(recipient, amount, tokenIdOpt)) + val (chain, _, newState, _) = + (2 until blockQty).foldLeft(List(genesisBlock, forkBlock), forkBlock, forkState, forkBoxes) { + case ((blocks, blockL, stateL, boxes), height) => + val invalid = genInvalidBlockFrom.exists(height + 1 >= _) + val (newBlock, newBoxes) = genNextBlockForState(privKey, blockL, settings.constants.InitialEmissionAmount, stateL, boxes, + randomAddress, invalid) + val newState = if (invalid) stateL else stateL.applyModifier(newBlock).right.get + (blocks :+ newBlock, newBlock, newState, newBoxes) + } + (state, newState, chain) + } - val uTransaction: UnsignedTransaction = UnsignedTransaction(fee, timestamp, uInputs, directives) - val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) - uTransaction.toSigned(IndexedSeq.empty, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) - } + def paymentTransactionWithMultipleOutputs(privKey: PrivateKey25519, fee: Amount, timestamp: Long, useBoxes: IndexedSeq[MonetaryBox], + recipient: Address, amount: Amount, tokenIdOpt: Option[ADKey] = None, + numOfOutputs: Int): Transaction = { - def genForkBlock(boxQty: Int, prevBlock: Block, box: AssetBox, recipient: Address, - addDiff: Difficulty = Difficulty @@ BigInt(0)): (Block, Seq[AssetBox]) = { - val timestamp = System.currentTimeMillis() + val pubKey: PublicKey25519 = privKey.publicImage + val uInputs: IndexedSeq[Input] = useBoxes + .map(bx => Input.unsigned(bx.id, Right(PubKeyLockedContract(pubKey.pubKeyBytes)))) + .toIndexedSeq - val transactions: Seq[Transaction] = Seq( - paymentTransactionWithMultipleOutputs(privKey, 1L, timestamp, IndexedSeq(box), recipient, 1000L, None, boxQty), - coinbaseTransaction(prevBlock.header.height + 1) - ) + val change: Amount = useBoxes.map(_.amount).sum - (amount + fee) + val directives: IndexedSeq[TransferDirective] = + if (change > 0) TransferDirective(recipient, amount, tokenIdOpt) +: (0 until numOfOutputs).map(_ => + TransferDirective(pubKey.address.address, change / numOfOutputs, tokenIdOpt)) + else IndexedSeq(TransferDirective(recipient, amount, tokenIdOpt)) - val header = Header(1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), timestamp, - prevBlock.header.height + 1, Random.nextLong(), Difficulty @@ (BigInt(1) + addDiff), EquihashSolution(Seq(1, 3))) + val uTransaction: UnsignedTransaction = UnsignedTransaction(fee, timestamp, uInputs, directives) + val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) + uTransaction.toSigned(IndexedSeq.empty, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) + } - (Block(header, Payload(header.id, transactions)), transactions.head.newBoxes.tail.map(_.asInstanceOf[AssetBox]).toSeq) + def utxoFromBoxHolder(bh: BoxHolder, dir: File, nodeViewHolderRef: Option[ActorRef], settings: EncryAppSettings, + storageType: StorageType): UtxoState = { + val storage = settings.storage.state match { + case VersionalStorage.IODB => + IODBWrapper(new LSMStore(dir, keepVersions = settings.constants.DefaultKeepVersions)) + case VersionalStorage.LevelDB => + val levelDBInit = LevelDbFactory.factory.open(dir, new Options) + VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settings.levelDB, keySize = 32)) } - def generateGenesisBlock(genesisHeight: Height): Block = { - val coinbaseTrans = TransactionFactory.coinbaseTransactionScratch( - privKey.publicImage, - System.currentTimeMillis(), - settings.constants.InitialEmissionAmount, - amount = 0L, - height = genesisHeight - ) + storage.insert( + StorageVersion @@ Array.fill(32)(0: Byte), + bh.boxes.values.map(bx => (StorageKey !@@ bx.id, StorageValue @@ bx.bytes)).toList + ) - val txs: Seq[Transaction] = Seq(coinbaseTrans) - val txsRoot: Digest32 = Payload.rootHash(txs.map(_.id)) - val header = - Header( - 1.toByte, - Header.GenesisParentId, - txsRoot, - System.currentTimeMillis(), - genesisHeight, - Random.nextLong(), - settings.constants.InitialDifficulty, - EquihashSolution(Seq(1, 3)) - ) - Block(header, Payload(header.id, txs)) - } + new UtxoState(storage, settings.constants) + } - def coinbaseTransaction(height: Int): Transaction = TransactionFactory.coinbaseTransactionScratch( - privKey.publicImage, + def genGenesisBlock(pubKey: PublicKey25519, initialEmissionAmount: Int, initialDifficulty: Difficulty, genesisHeight: Height): Block = { + val coinbaseTrans = TransactionFactory.coinbaseTransactionScratch( + pubKey, System.currentTimeMillis(), - supply = settings.constants.InitialEmissionAmount, - amount = 1L, - height = Height @@ height + initialEmissionAmount, + amount = 0L, + height = genesisHeight ) - def genNextBlockForState(prevBlock: Block, state: UtxoState, boxes: Seq[AssetBox], recipient: Address, invalid: Boolean = false, - addDiff: Difficulty = Difficulty @@ BigInt(0)): (Block, Seq[AssetBox]) = { - val timestamp = System.currentTimeMillis() - - val amount = if (invalid) boxes.map(_.amount).sum + 1L else 1L + val txs: Seq[Transaction] = Seq(coinbaseTrans) + val txsRoot: Digest32 = Payload.rootHash(txs.map(_.id)) + val header = + Header( + 1.toByte, + Header.GenesisParentId, + txsRoot, + System.currentTimeMillis(), + genesisHeight, + Random.nextLong(), + initialDifficulty, + EquihashSolution(Seq(1, 3)) + ) + Block(header, Payload(header.id, txs)) + } - val transactions: Seq[Transaction] = - boxes.map(b => TransactionFactory.defaultPaymentTransactionScratch(privKey, 1L, timestamp, IndexedSeq(b), recipient, amount)) :+ - coinbaseTransaction(prevBlock.header.height + 1) + def genForkBlock(privKey: PrivateKey25519, initialEmissionAmount: Int, boxQty: Int, prevBlock: Block, box: AssetBox, recipient: Address, + addDiff: Difficulty = Difficulty @@ BigInt(0)): (Block, Seq[AssetBox]) = { + val timestamp = System.currentTimeMillis() - val header = Header(1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), timestamp, - prevBlock.header.height + 1, Random.nextLong(), Difficulty @@ (BigInt(1) + addDiff), EquihashSolution(Seq(1, 3))) + val transactions: Seq[Transaction] = Seq( + paymentTransactionWithMultipleOutputs(privKey, 1L, timestamp, IndexedSeq(box), recipient, 1L, None, boxQty), + TransactionFactory.coinbaseTransactionScratch(privKey.publicImage, System.currentTimeMillis(), initialEmissionAmount, + 1L, Height @@ (prevBlock.header.height + 1)) + ) - val newBoxes = transactions.filter(_.newBoxes.size > 1).map(_.newBoxes.toSeq(1).asInstanceOf[AssetBox]) - (Block(header, Payload(header.id, transactions)), newBoxes) - } + val header = Header(1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), timestamp, + prevBlock.header.height + 1, Random.nextLong(), Difficulty @@ (BigInt(1) + addDiff), EquihashSolution(Seq(1, 3))) - def randomAddress: Address = Pay2PubKeyAddress(PublicKey @@ ScorexRandom.randomBytes()).address + (Block(header, Payload(header.id, transactions)), transactions.head.newBoxes.tail.map(_.asInstanceOf[AssetBox]).toSeq) + } - assert(blockQty >= 2, "chain at least 2 blocks") + def genNextBlockForState(privKey: PrivateKey25519, prevBlock: Block, initialEmissionAmount: Int, state: UtxoState, boxes: Seq[AssetBox], + recipient: Address, invalid: Boolean = false, + addDiff: Difficulty = Difficulty @@ BigInt(0)): (Block, Seq[AssetBox]) = { + val timestamp = System.currentTimeMillis() - val genesisBlock: Block = generateGenesisBlock(Height @@ 0) + val amount = if (invalid) boxes.map(_.amount).sum + 1L else 1L - val state: UtxoState = utxoFromBoxHolder(BoxHolder(Seq.empty), dir, None, settings, VersionalStorage.LevelDB) - val genesisState = state.applyModifier(genesisBlock).right.get + val transactions: Seq[Transaction] = + boxes.map(b => TransactionFactory.defaultPaymentTransactionScratch(privKey, 1L, timestamp, IndexedSeq(b), recipient, amount)) :+ + TransactionFactory.coinbaseTransactionScratch(privKey.publicImage, System.currentTimeMillis(), initialEmissionAmount, + 1L, Height @@ (prevBlock.header.height + 1)) - val (forkBlock, forkBoxes) = - genForkBlock(transPerBlock, genesisBlock, genesisBlock.payload.txs.head.newBoxes.map(_.asInstanceOf[AssetBox]).head, randomAddress) - val forkState = genesisState.applyModifier(forkBlock).right.get + val header = Header(1.toByte, prevBlock.id, Payload.rootHash(transactions.map(_.id)), timestamp, + prevBlock.header.height + 1, Random.nextLong(), Difficulty @@ (BigInt(1) + addDiff), EquihashSolution(Seq(1, 3))) - val (chain, _, newState, _) = - (2 until blockQty).foldLeft(List(genesisBlock, forkBlock), forkBlock, forkState, forkBoxes) { - case ((blocks, blockL, stateL, boxes), height) => - val invalid = genInvalidBlockFrom.exists(height + 1 >= _) - val (newBlock, newBoxes) = genNextBlockForState(blockL, stateL, boxes, randomAddress, invalid) - val newState = if (invalid) stateL else stateL.applyModifier(newBlock).right.get - (blocks :+ newBlock, newBlock, newState, newBoxes) - } - (state, newState, chain) + val newBoxes = transactions.filter(_.newBoxes.size > 1).map(_.newBoxes.toSeq(1).asInstanceOf[AssetBox]) + (Block(header, Payload(header.id, transactions)), newBoxes) } } \ No newline at end of file From 9835e90bb437e7e71de392f689d5020045c7c578 Mon Sep 17 00:00:00 2001 From: capdev Date: Fri, 27 Sep 2019 10:59:16 +0500 Subject: [PATCH 30/39] wait condition cleanup --- .../history/HistoryApplicatorTest.scala | 43 ++++++------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 6097b74c7a..603c0c6139 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -6,7 +6,6 @@ import akka.actor.ActorSystem import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} import encry.modifiers.InstanceFactory import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.NodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier import encry.settings.{EncryAppSettings, Settings} import encry.storage.VersionalStorage import encry.utils.ChainGenerator._ @@ -42,30 +41,21 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w TestActorRef[HistoryApplicator]( HistoryApplicator.props(history, testSettings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) - system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) + system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) chain .flatMap(blockToModifiers) .foreach(historyApplicator ! ModifierFromRemote(_)) - //Thread.sleep(3000 + transQty) - receiveN(6 * 2, timeout) + expectMsg(timeout, FullBlockChainIsSynced()) - history.getBestBlockHeight shouldBe 4 + Thread.sleep(1000 + transQty/2) + + awaitCond(history.getBestBlockHeight == 4, timeout) history.getBestBlock.map(b => Algos.encode(b.id)) shouldBe Some(Algos.encode(chain(4).id)) history.getBestBlockHeightDB shouldBe 4 } -// def applyBlocksToHistory(history: History, blocks: Seq[Block]): History = -// blocks.foldLeft(history) { -// case (prevHistory, block) => -// prevHistory.append(block.header) -// prevHistory.append(block.payload) -// prevHistory.reportModifierIsValid(block) -// } - - def checkFullBlockChainIsSynced(qty: Int): Unit = (0 until qty).foreach(_ => expectMsg(30 seconds, FullBlockChainIsSynced())) - val dir: File = FileHelper.getRandomTempDir val wallet: EncryWallet = EncryWallet.readOrGenerate(testSettings.copy(directory = dir.getAbsolutePath)) @@ -91,11 +81,8 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w chain.foreach(historyApplicator ! LocallyGeneratedBlock(_)) - receiveN((blockQty - 1) * 2, timeout) - //checkFullBlockChainIsSynced((blockQty - 1) * 2) - //expectMsg(timeout, FullBlockChainIsSynced()) - - history.getBestBlockHeight shouldBe blockQty - 1 + expectMsg(timeout, FullBlockChainIsSynced()) + awaitCond(history.getBestBlockHeight == blockQty - 1, timeout) } "apply remote blocks and check chain sync" in { @@ -115,11 +102,8 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w modifiers.foreach(historyApplicator ! ModifierFromRemote(_)) - //Thread.sleep(3000) - receiveN(6 * 2, timeout) - //checkFullBlockChainIsSynced((blockQty - 1) * 2) - //expectMsg(timeout, FullBlockChainIsSynced()) - history.getBestBlockHeight shouldBe blockQty - 1 + expectMsg(timeout, FullBlockChainIsSynced()) + awaitCond(history.getBestBlockHeight == blockQty - 1, timeout) } "apply remote blocks and check queue for rollback height" in { @@ -133,7 +117,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w TestActorRef[HistoryApplicator]( HistoryApplicator.props(history, testSettings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) ) - system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) + system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) chain .flatMap(blockToModifiers) @@ -141,11 +125,8 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w historyApplicator.underlyingActor.modifiersQueue.size should be <= testSettings.levelDB.maxVersions - //Thread.sleep(5000) - //checkFullBlockChainIsSynced((testSettings.levelDB.maxVersions + overQty) * 2) - receiveN((testSettings.levelDB.maxVersions + overQty) * 2, timeout) - - history.getBestBlockHeight shouldBe testSettings.levelDB.maxVersions + overQty - 1 + expectMsg(timeout, FullBlockChainIsSynced()) + awaitCond(history.getBestBlockHeight == testSettings.levelDB.maxVersions + overQty - 1) } "apply remote blocks and check history rollback for invalid state" in { From c34eb688df6b151f2d4bd1656b96b3ddb9a513fe Mon Sep 17 00:00:00 2001 From: capdev Date: Fri, 27 Sep 2019 17:34:47 +0500 Subject: [PATCH 31/39] add sync when apply headers and payloads separately test add sync and rollback headers for invalid payload test add checkBestBlock some refactoring --- .../test/scala/benches/HistoryBenches.scala | 7 +- .../src/test/scala/benches/StateBenches.scala | 5 +- .../scala/benches/StateRollbackBench.scala | 6 +- benchmarks/src/test/scala/benches/Utils.scala | 36 +-------- .../mempool/TransactionFactory.scala | 56 ++++++++++++- .../history/HistoryApplicatorTest.scala | 81 ++++++++++++++++++- .../scala/encry/utils/ChainGenerator.scala | 24 +----- .../scala/encry/utils/EncryGenerator.scala | 65 +-------------- .../encry/view/state/UtxoStateSpec.scala | 21 ----- 9 files changed, 150 insertions(+), 151 deletions(-) diff --git a/benchmarks/src/test/scala/benches/HistoryBenches.scala b/benchmarks/src/test/scala/benches/HistoryBenches.scala index 99e5e52ddf..fb9ea64587 100644 --- a/benchmarks/src/test/scala/benches/HistoryBenches.scala +++ b/benchmarks/src/test/scala/benches/HistoryBenches.scala @@ -4,9 +4,10 @@ import java.io.File import java.util.concurrent.TimeUnit import benches.HistoryBenches.HistoryBenchState -import benches.Utils.{generateHistory, generateNextBlockValidForHistory, getRandomTempDir} +import benches.Utils.{generateHistory, generateNextBlockValidForHistory} import encry.EncryApp import encry.settings.EncryAppSettings +import encry.utils.FileHelper import encry.view.history.History import encryBenchmark.{BenchSettings, Settings} import org.encryfoundation.common.modifiers.history.Block @@ -22,7 +23,7 @@ class HistoryBenches { @Benchmark def appendBlocksToHistoryBench(benchStateHistory: HistoryBenchState, bh: Blackhole): Unit = { bh.consume { - val history: History = generateHistory(benchStateHistory.settings, getRandomTempDir) + val history: History = generateHistory(benchStateHistory.settings, FileHelper.getRandomTempDir) benchStateHistory.blocks.foldLeft(history) { case (historyL, block) => historyL.append(block.header) historyL.append(block.payload) @@ -64,7 +65,7 @@ object HistoryBenches extends BenchSettings { @State(Scope.Benchmark) class HistoryBenchState extends encry.settings.Settings { - val tmpDir: File = getRandomTempDir + val tmpDir: File = FileHelper.getRandomTempDir val initialHistory: History = generateHistory(settings, tmpDir) val resultedHistory: (History, Option[Block], Vector[Block]) = diff --git a/benchmarks/src/test/scala/benches/StateBenches.scala b/benchmarks/src/test/scala/benches/StateBenches.scala index 9aa45760b2..d541356f4a 100644 --- a/benchmarks/src/test/scala/benches/StateBenches.scala +++ b/benchmarks/src/test/scala/benches/StateBenches.scala @@ -10,6 +10,9 @@ import encry.EncryApp import encry.settings.EncryAppSettings import encry.storage.VersionalStorage import encry.storage.VersionalStorage.IODB +import encry.utils.{ChainGenerator, FileHelper} +import encry.utils.ChainGenerator._ +import encry.utils.FileHelper.getRandomTempDir import encry.view.state.{BoxHolder, UtxoState} import encryBenchmark.{BenchSettings, Settings} import org.encryfoundation.common.modifiers.history.Block @@ -25,7 +28,7 @@ class StateBenches { def applyBlocksToTheState(stateBench: StateBenchState, bh: Blackhole): Unit = { bh.consume { val innerState: UtxoState = - utxoFromBoxHolder(stateBench.boxesHolder, getRandomTempDir, None, stateBench.settings, VersionalStorage.LevelDB) + utxoFromBoxHolder(stateBench.boxesHolder, getRandomTempDir, None, stateBench.settings, VersionalStorage.LevelDB) stateBench.chain.foldLeft(innerState) { case (state, block) => state.applyModifier(block).right.get } diff --git a/benchmarks/src/test/scala/benches/StateRollbackBench.scala b/benchmarks/src/test/scala/benches/StateRollbackBench.scala index 812fc160f9..14f9989e20 100644 --- a/benchmarks/src/test/scala/benches/StateRollbackBench.scala +++ b/benchmarks/src/test/scala/benches/StateRollbackBench.scala @@ -6,6 +6,7 @@ import java.util.concurrent.TimeUnit import benches.StateRollbackBench.StateRollbackState import benches.Utils._ import encry.storage.VersionalStorage +import encry.utils.ChainGenerator import encry.utils.CoreTaggedTypes.VersionTag import encry.view.state.{BoxHolder, UtxoState} import encryBenchmark.{BenchSettings, Settings} @@ -17,6 +18,7 @@ import org.openjdk.jmh.infra.Blackhole import org.openjdk.jmh.profile.GCProfiler import org.openjdk.jmh.runner.{Runner, RunnerException} import org.openjdk.jmh.runner.options.{OptionsBuilder, TimeValue, VerboseMode} +import encry.utils.FileHelper.getRandomTempDir class StateRollbackBench { @@ -24,7 +26,7 @@ class StateRollbackBench { def applyBlocksToTheState(stateBench: StateRollbackState, bh: Blackhole): Unit = { bh.consume { val innerState: UtxoState = - utxoFromBoxHolder(stateBench.boxesHolder, getRandomTempDir, None, stateBench.settings, VersionalStorage.IODB) + ChainGenerator.utxoFromBoxHolder(stateBench.boxesHolder, getRandomTempDir, None, stateBench.settings, VersionalStorage.IODB) val newState = stateBench.chain.foldLeft(innerState -> List.empty[VersionTag]) { case ((state, rootHashes), block) => val newState = state.applyModifier(block).right.get newState -> (rootHashes :+ newState.version) @@ -65,7 +67,7 @@ object StateRollbackBench extends BenchSettings { genHardcodedBox(privKey.publicImage.address.address, nonce) ) val boxesHolder: BoxHolder = BoxHolder(initialBoxes) - var state: UtxoState = utxoFromBoxHolder(boxesHolder, tmpDir, None, settings, VersionalStorage.LevelDB) + var state: UtxoState = ChainGenerator.utxoFromBoxHolder(boxesHolder, tmpDir, None, settings, VersionalStorage.LevelDB) val genesisBlock: Block = generateGenesisBlockValidForState(state) state = state.applyModifier(genesisBlock).right.get diff --git a/benchmarks/src/test/scala/benches/Utils.scala b/benchmarks/src/test/scala/benches/Utils.scala index 0d8e54d3be..256bc04687 100644 --- a/benchmarks/src/test/scala/benches/Utils.scala +++ b/benchmarks/src/test/scala/benches/Utils.scala @@ -34,6 +34,8 @@ import scorex.utils.Random import scala.collection.immutable import scala.util.{Random => R} +import encry.utils.ChainGenerator._ + object Utils extends Settings with StrictLogging { val mnemonicKey: String = "index another island accuse valid aerobic little absurd bunker keep insect scissors" @@ -70,11 +72,6 @@ object Utils extends Settings with StrictLogging { Block(header, Payload(header.id, txs)) } - def generateGenesisBlockValidForHistory: Block = { - val header = genHeader.copy(parentId = Header.GenesisParentId, height = settings.constants.GenesisHeight) - Block(header, Payload(header.id, Seq(coinbaseTransaction))) - } - def generateNextBlockValidForState(prevBlock: Block, state: UtxoState, box: Seq[AssetBox], @@ -171,35 +168,6 @@ object Utils extends Settings with StrictLogging { def genAssetBox(address: Address, amount: Amount = 100000L, tokenIdOpt: Option[ADKey] = None): AssetBox = AssetBox(EncryProposition.addressLocked(address), R.nextLong(), amount, tokenIdOpt) - def utxoFromBoxHolder(bh: BoxHolder, - dir: File, - nodeViewHolderRef: Option[ActorRef], - settings: EncryAppSettings, - storageType: StorageType): UtxoState = { - val storage = settings.storage.state match { - case VersionalStorage.IODB => - logger.info("Init state with iodb storage") - IODBWrapper(new LSMStore(dir, keepVersions = settings.constants.DefaultKeepVersions)) - case VersionalStorage.LevelDB => - logger.info("Init state with levelDB storage") - val levelDBInit = LevelDbFactory.factory.open(dir, new Options) - VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settings.levelDB, keySize = 32)) - } - - storage.insert( - StorageVersion @@ Array.fill(32)(0: Byte), - bh.boxes.values.map(bx => (StorageKey !@@ bx.id, StorageValue @@ bx.bytes)).toList - ) - - new UtxoState(storage, settings.constants) - } - - def getRandomTempDir: File = { - val dir = java.nio.file.Files.createTempDirectory("encry_test_" + R.alphanumeric.take(15).mkString).toFile - dir.deleteOnExit() - dir - } - def genHeader: Header = { val random = new scala.util.Random Header( diff --git a/src/main/scala/encry/modifiers/mempool/TransactionFactory.scala b/src/main/scala/encry/modifiers/mempool/TransactionFactory.scala index 07e81d778a..999f209300 100644 --- a/src/main/scala/encry/modifiers/mempool/TransactionFactory.scala +++ b/src/main/scala/encry/modifiers/mempool/TransactionFactory.scala @@ -2,7 +2,7 @@ package encry.modifiers.mempool import com.typesafe.scalalogging.StrictLogging import org.encryfoundation.common.crypto.{PrivateKey25519, PublicKey25519, Signature25519} -import org.encryfoundation.common.modifiers.mempool.directive.{Directive, TransferDirective} +import org.encryfoundation.common.modifiers.mempool.directive.{DataDirective, Directive, TransferDirective} import org.encryfoundation.common.modifiers.mempool.transaction.EncryAddress.Address import org.encryfoundation.common.modifiers.mempool.transaction._ import org.encryfoundation.common.modifiers.state.box.Box.Amount @@ -102,4 +102,58 @@ object TransactionFactory extends StrictLogging { IndexedSeq(TransferDirective(recipient, howMuchWillTransfer, tokenIdOpt)) prepareTransaction(privKey, fee, timestamp, useOutputs, directives, change, tokenIdOpt) } + + def paymentTransactionWithMultipleOutputs(privKey: PrivateKey25519, fee: Amount, timestamp: Long, useBoxes: IndexedSeq[MonetaryBox], + recipient: Address, amount: Amount, tokenIdOpt: Option[ADKey] = None, + numOfOutputs: Int): Transaction = { + + val pubKey: PublicKey25519 = privKey.publicImage + val uInputs: IndexedSeq[Input] = useBoxes + .map(bx => Input.unsigned(bx.id, Right(PubKeyLockedContract(pubKey.pubKeyBytes)))) + .toIndexedSeq + + val change: Amount = useBoxes.map(_.amount).sum - (amount + fee) + val directives: IndexedSeq[TransferDirective] = + if (change > 0) TransferDirective(recipient, amount, tokenIdOpt) +: (0 until numOfOutputs).map(_ => + TransferDirective(pubKey.address.address, change / numOfOutputs, tokenIdOpt)) + else IndexedSeq(TransferDirective(recipient, amount, tokenIdOpt)) + + val uTransaction: UnsignedTransaction = UnsignedTransaction(fee, timestamp, uInputs, directives) + val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) + uTransaction.toSigned(IndexedSeq.empty, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) + } + + def dataTransactionScratch(privKey: PrivateKey25519, + fee: Long, + timestamp: Long, + useOutputs: IndexedSeq[MonetaryBox], + amount: Long, + data: Array[Byte], + numOfOutputs: Int = 5): Transaction = { + + val pubKey: PublicKey25519 = privKey.publicImage + + val uInputs: IndexedSeq[Input] = useOutputs + .map(bx => Input.unsigned(bx.id, Right(PubKeyLockedContract(pubKey.pubKeyBytes)))) + .toIndexedSeq + + val change: Amount = useOutputs.map(_.amount).sum - (amount + fee) + + val directives: IndexedSeq[DataDirective] = + (0 until numOfOutputs).foldLeft(IndexedSeq.empty[DataDirective]) { case (directivesAll, _) => + directivesAll :+ DataDirective(PubKeyLockedContract(privKey.publicImage.pubKeyBytes).contract.hash, data) + } + + val newDirectives: IndexedSeq[Directive] = + if (change > 0) TransferDirective(pubKey.address.address, amount, None) +: (0 until numOfOutputs).map(_ => + TransferDirective(pubKey.address.address, change / numOfOutputs, None)) ++: directives + else directives + + val uTransaction: UnsignedTransaction = UnsignedTransaction(fee, timestamp, uInputs, newDirectives) + + val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) + + uTransaction.toSigned(IndexedSeq.empty, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) + } + } \ No newline at end of file diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 603c0c6139..2bad2fb691 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -52,8 +52,15 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w Thread.sleep(1000 + transQty/2) awaitCond(history.getBestBlockHeight == 4, timeout) - history.getBestBlock.map(b => Algos.encode(b.id)) shouldBe Some(Algos.encode(chain(4).id)) - history.getBestBlockHeightDB shouldBe 4 + + checkBestBlock(history, chain(4)) + } + + def checkBestBlock(history: History, expectBlock: Block) { + history.getBestHeaderHeight shouldBe expectBlock.header.height + history.getBestHeader.map(h => Algos.encode(h.id)) shouldBe Some(Algos.encode(expectBlock.header.id)) + history.getBestBlock.map(b => Algos.encode(b.id)) shouldBe Some(Algos.encode(expectBlock.id)) + history.getBestBlockHeightDB shouldBe expectBlock.header.height } val dir: File = FileHelper.getRandomTempDir @@ -83,6 +90,8 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w expectMsg(timeout, FullBlockChainIsSynced()) awaitCond(history.getBestBlockHeight == blockQty - 1, timeout) + + checkBestBlock(history, chain(blockQty - 1)) } "apply remote blocks and check chain sync" in { @@ -104,6 +113,8 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w expectMsg(timeout, FullBlockChainIsSynced()) awaitCond(history.getBestBlockHeight == blockQty - 1, timeout) + + checkBestBlock(history, chain(blockQty - 1)) } "apply remote blocks and check queue for rollback height" in { @@ -127,15 +138,77 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w expectMsg(timeout, FullBlockChainIsSynced()) awaitCond(history.getBestBlockHeight == testSettings.levelDB.maxVersions + overQty - 1) + + checkBestBlock(history, chain(testSettings.levelDB.maxVersions + overQty - 1)) } - "apply remote blocks and check history rollback for invalid state" in { + "should sync and check history rollback on invalid block " in { rollbackTest(100) } - "apply remote blocks and check history rollback on fat blocks" in { + "should sync and check history rollback on invalid block for fat blocks" in { rollbackTest(5000) } + "sync when apply headers and payloads separately" in { + val blockQty = 10 + val history: History = generateDummyHistory(testSettings) + val (initialState, state, chain) = genChain(privKey, dir, testSettings, blockQty) + + val headers: Seq[PersistentModifier] = chain.map(_.header) + val payloads: Seq[PersistentModifier] = chain.map(_.payload) + + val historyApplicator: TestActorRef[HistoryApplicator] = + TestActorRef[HistoryApplicator]( + HistoryApplicator.props(history, testSettings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) + ) + + system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) + + headers.foreach(historyApplicator ! ModifierFromRemote(_)) + + awaitCond(history.getBestHeaderHeight == blockQty - 1, timeout) + history.getBestHeaderHeight shouldBe blockQty - 1 + history.getBestHeader.map(h => Algos.encode(h.id)) shouldBe Some(Algos.encode(chain(blockQty - 1).header.id)) + + payloads.foreach(historyApplicator ! ModifierFromRemote(_)) + + expectMsg(timeout, FullBlockChainIsSynced()) + awaitCond(history.getBestBlockHeight == blockQty - 1, timeout) + checkBestBlock(history, chain(blockQty - 1)) + } + + "sync and rollback headers for invalid payload" in { + val blockQty = 10 + val history: History = generateDummyHistory(testSettings) + //generate invalid blocks begining from 6 blocks + val (initialState, state, chain) = genChain(privKey, dir, testSettings, blockQty, 100, Some(6)) + + val headers: Seq[PersistentModifier] = chain.map(_.header) + val payloads: Seq[PersistentModifier] = chain.map(_.payload) + + val historyApplicator: TestActorRef[HistoryApplicator] = + TestActorRef[HistoryApplicator]( + HistoryApplicator.props(history, testSettings, initialState, wallet, nodeViewHolder.ref, Some(influx.ref)) + ) + + system.eventStream.subscribe(self, classOf[FullBlockChainIsSynced]) + + headers.foreach(historyApplicator ! ModifierFromRemote(_)) + + awaitCond(history.getBestHeaderHeight == blockQty - 1, timeout, 100 millis, + s"history.getBestBlockHeight ${history.getBestHeaderHeight} expected ${blockQty - 1}") + history.getBestHeaderHeight shouldBe blockQty - 1 + history.getBestHeader.map(h => Algos.encode(h.id)) shouldBe Some(Algos.encode(chain(blockQty - 1).header.id)) + + payloads.foreach(historyApplicator ! ModifierFromRemote(_)) + + expectMsg(timeout, FullBlockChainIsSynced()) + awaitCond(history.getBestBlockHeight == 4, timeout, 100 millis, + s"history.getBestBlockHeight ${history.getBestBlockHeight} expected 4") + checkBestBlock(history, chain(4)) + } + + } } diff --git a/src/test/scala/encry/utils/ChainGenerator.scala b/src/test/scala/encry/utils/ChainGenerator.scala index c884f0e9be..18dde6d537 100644 --- a/src/test/scala/encry/utils/ChainGenerator.scala +++ b/src/test/scala/encry/utils/ChainGenerator.scala @@ -31,7 +31,7 @@ import scala.util.Random object ChainGenerator { - def genChain(privKey: PrivateKey25519, dir: File, settings: EncryAppSettings, blockQty: Int, transPerBlock: Int = 1, + def genChain(privKey: PrivateKey25519, dir: File, settings: EncryAppSettings, blockQty: Int, transPerBlock: Int = 100, genInvalidBlockFrom: Option[Int] = None): (UtxoState, UtxoState, List[Block]) = { def randomAddress: Address = Pay2PubKeyAddress(PublicKey @@ ScorexRandom.randomBytes()).address @@ -61,26 +61,6 @@ object ChainGenerator { (state, newState, chain) } - def paymentTransactionWithMultipleOutputs(privKey: PrivateKey25519, fee: Amount, timestamp: Long, useBoxes: IndexedSeq[MonetaryBox], - recipient: Address, amount: Amount, tokenIdOpt: Option[ADKey] = None, - numOfOutputs: Int): Transaction = { - - val pubKey: PublicKey25519 = privKey.publicImage - val uInputs: IndexedSeq[Input] = useBoxes - .map(bx => Input.unsigned(bx.id, Right(PubKeyLockedContract(pubKey.pubKeyBytes)))) - .toIndexedSeq - - val change: Amount = useBoxes.map(_.amount).sum - (amount + fee) - val directives: IndexedSeq[TransferDirective] = - if (change > 0) TransferDirective(recipient, amount, tokenIdOpt) +: (0 until numOfOutputs).map(_ => - TransferDirective(pubKey.address.address, change / numOfOutputs, tokenIdOpt)) - else IndexedSeq(TransferDirective(recipient, amount, tokenIdOpt)) - - val uTransaction: UnsignedTransaction = UnsignedTransaction(fee, timestamp, uInputs, directives) - val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) - uTransaction.toSigned(IndexedSeq.empty, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) - } - def utxoFromBoxHolder(bh: BoxHolder, dir: File, nodeViewHolderRef: Option[ActorRef], settings: EncryAppSettings, storageType: StorageType): UtxoState = { val storage = settings.storage.state match { @@ -129,7 +109,7 @@ object ChainGenerator { val timestamp = System.currentTimeMillis() val transactions: Seq[Transaction] = Seq( - paymentTransactionWithMultipleOutputs(privKey, 1L, timestamp, IndexedSeq(box), recipient, 1L, None, boxQty), + TransactionFactory.paymentTransactionWithMultipleOutputs(privKey, 1L, timestamp, IndexedSeq(box), recipient, 1L, None, boxQty), TransactionFactory.coinbaseTransactionScratch(privKey.publicImage, System.currentTimeMillis(), initialEmissionAmount, 1L, Height @@ (prevBlock.header.height + 1)) ) diff --git a/src/test/scala/encry/utils/EncryGenerator.scala b/src/test/scala/encry/utils/EncryGenerator.scala index d102e45619..19c6a6aa31 100644 --- a/src/test/scala/encry/utils/EncryGenerator.scala +++ b/src/test/scala/encry/utils/EncryGenerator.scala @@ -1,6 +1,7 @@ package encry.utils import encry.modifiers.mempool.TransactionFactory +import encry.modifiers.mempool.TransactionFactory.{dataTransactionScratch, paymentTransactionWithMultipleOutputs} import encry.settings.Settings import encry.utils.TestHelper.Props import org.encryfoundation.common.crypto.equihash.EquihashSolution @@ -162,7 +163,7 @@ trait EncryGenerator extends Settings { numberOfOutputs: Int): Vector[Transaction] = (0 until boxes.size / numberOfInputs).foldLeft(boxes, Vector.empty[Transaction]) { case ((boxesLocal, transactions), _) => - val tx: Transaction = defaultPaymentTransactionScratch( + val tx: Transaction = paymentTransactionWithMultipleOutputs( privKey, fee = 111, timestamp = 11L, @@ -208,68 +209,6 @@ trait EncryGenerator extends Settings { (boxesLocal.drop(numberOfInputs), tx +: transactions) }._2 - def defaultPaymentTransactionScratch(privKey: PrivateKey25519, - fee: Amount, - timestamp: Long, - useBoxes: IndexedSeq[MonetaryBox], - recipient: Address, - amount: Amount, - tokenIdOpt: Option[ADKey] = None, - numOfOutputs: Int = 5): Transaction = { - - val pubKey: PublicKey25519 = privKey.publicImage - - val uInputs: IndexedSeq[Input] = useBoxes - .map(bx => Input.unsigned(bx.id, Right(PubKeyLockedContract(pubKey.pubKeyBytes)))) - .toIndexedSeq - - val change: Amount = useBoxes.map(_.amount).sum - (amount + fee) - - val directives: IndexedSeq[TransferDirective] = - if (change > 0) TransferDirective(recipient, amount, tokenIdOpt) +: (0 until numOfOutputs).map(_ => - TransferDirective(pubKey.address.address, change / numOfOutputs, tokenIdOpt)) - else IndexedSeq(TransferDirective(recipient, amount, tokenIdOpt)) - - val uTransaction: UnsignedTransaction = UnsignedTransaction(fee, timestamp, uInputs, directives) - - val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) - - uTransaction.toSigned(IndexedSeq.empty, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) - } - - def dataTransactionScratch(privKey: PrivateKey25519, - fee: Long, - timestamp: Long, - useOutputs: IndexedSeq[MonetaryBox], - amount: Long, - data: Array[Byte], - numOfOutputs: Int = 5): Transaction = { - - val pubKey: PublicKey25519 = privKey.publicImage - - val uInputs: IndexedSeq[Input] = useOutputs - .map(bx => Input.unsigned(bx.id, Right(PubKeyLockedContract(pubKey.pubKeyBytes)))) - .toIndexedSeq - - val change: Amount = useOutputs.map(_.amount).sum - (amount + fee) - - val directives: IndexedSeq[DataDirective] = - (0 until numOfOutputs).foldLeft(IndexedSeq.empty[DataDirective]) { case (directivesAll, _) => - directivesAll :+ DataDirective(PubKeyLockedContract(privKey.publicImage.pubKeyBytes).contract.hash, data) - } - - val newDirectives: IndexedSeq[Directive] = - if (change > 0) TransferDirective(pubKey.address.address, amount, None) +: (0 until numOfOutputs).map(_ => - TransferDirective(pubKey.address.address, change / numOfOutputs, None)) ++: directives - else directives - - val uTransaction: UnsignedTransaction = UnsignedTransaction(fee, timestamp, uInputs, newDirectives) - - val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) - - uTransaction.toSigned(IndexedSeq.empty, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) - } - def assetIssuingTransactionScratch(privKey: PrivateKey25519, fee: Long, timestamp: Long, diff --git a/src/test/scala/encry/view/state/UtxoStateSpec.scala b/src/test/scala/encry/view/state/UtxoStateSpec.scala index 91ef368a25..aabb14de53 100755 --- a/src/test/scala/encry/view/state/UtxoStateSpec.scala +++ b/src/test/scala/encry/view/state/UtxoStateSpec.scala @@ -45,27 +45,6 @@ class UtxoStateSpec extends PropSpec with Matchers with EncryGenerator { // ) // } - def utxoFromBoxHolder(bh: BoxHolder, - dir: File, - nodeViewHolderRef: Option[ActorRef], - settings: EncryAppSettings): UtxoState = { - val storage = settings.storage.state match { - case VersionalStorage.IODB => - IODBWrapper(new LSMStore(dir, keepVersions = settings.constants.DefaultKeepVersions)) - case VersionalStorage.LevelDB => - val levelDBInit = LevelDbFactory.factory.open(dir, new Options) - VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settings.levelDB, keySize = 32)) - } - - storage.insert( - StorageVersion @@ Array.fill(32)(0: Byte), - bh.boxes.values.map(bx => (StorageKey !@@ bx.id, StorageValue @@ bx.bytes)).toList - ) - - new UtxoState(storage, settings.constants) - } - - // property("Proofs for transaction") { // // val (privKey: PrivateKey, pubKey: PublicKey) = Curve25519.createKeyPair(Random.randomBytes()) From 44081a58a8a3961c94c0951ac2357be8b59a5362 Mon Sep 17 00:00:00 2001 From: capdev Date: Mon, 30 Sep 2019 13:23:30 +0500 Subject: [PATCH 32/39] big refactoring InstanceFactory, EncryGenerator, KeyHelper -> TestEntityGenerator, HistoryGenerator --- .../src/test/scala/benches/StateBenches.scala | 8 +- .../scala/benches/StateRollbackBench.scala | 13 +- benchmarks/src/test/scala/benches/Utils.scala | 58 +--- it/src/main/scala/encry/it/docker/Node.scala | 2 +- .../main/scala/encry/it/util/KeyHelper.scala | 21 -- .../AssetTokenTransactionTest.scala | 7 +- .../it/transactions/DataTransactionTest.scala | 8 +- ...ransferTransactionWithEncryCoinsTest.scala | 7 +- .../mempool/TransactionFactory.scala | 35 ++- .../encry/network/PeerConnectionHandler.scala | 4 +- src/main/scala/encry/utils/Utils.scala | 23 ++ .../routes/TransactionsApiRouteSpec.scala | 4 +- .../emission/EncrySupplyControllerTest.scala | 3 +- .../encry/modifiers/InstanceFactory.scala | 243 --------------- .../history/BlockSerializerTest.scala | 5 +- .../history/HeaderSerializerTest.scala | 4 +- .../encry/modifiers/history/HeaderSpec.scala | 6 +- .../history/HistoryApplicatorTest.scala | 37 ++- .../history/ModifiersProtoTest.scala | 60 +++- .../mempool/TransactionSerializerTest.scala | 4 +- .../modifiers/mempool/TransactionSpec.scala | 15 +- .../state/AssetBoxSerializerTest.scala | 13 +- .../scala/encry/modifiers/state/Keys.scala | 12 - .../scala/encry/network/BlackListTests.scala | 7 +- .../network/ConnectWithNewPeerTests.scala | 3 +- .../ConnectedPeersCollectionsTests.scala | 3 +- .../DeliveryManagerPriorityTests.scala | 7 +- ...DeliveryManagerReRequestModifiesSpec.scala | 9 +- .../DeliveryManagerRequestModifiesSpec.scala | 12 +- ...Utils.scala => DeliveryManagerUtils.scala} | 8 +- .../DownloadedModifiersValidatorTests.scala | 9 +- .../BasicNetworkMessagesProtoTest.scala | 13 +- .../scala/encry/utils/ChainGenerator.scala | 24 +- .../scala/encry/utils/EncryGenerator.scala | 279 ----------------- .../scala/encry/utils/HistoryGenerator.scala | 47 +++ src/test/scala/encry/utils/Keys.scala | 9 + .../encry/utils/TestEntityGenerator.scala | 292 ++++++++++++++++++ .../encry/view/history/EncryHistoryTest.scala | 4 +- .../HistoryComparisionResultTest.scala | 35 ++- .../history/ModifiersValidationTest.scala | 19 +- .../encry/view/mempool/MemoryPoolTests.scala | 5 +- .../encry/view/state/EncryStateSpec.scala | 3 +- .../encry/view/state/UtxoStateSpec.scala | 4 +- .../scala/encry/view/wallet/WalletSpec.scala | 13 +- 44 files changed, 621 insertions(+), 776 deletions(-) delete mode 100644 it/src/main/scala/encry/it/util/KeyHelper.scala delete mode 100755 src/test/scala/encry/modifiers/InstanceFactory.scala delete mode 100644 src/test/scala/encry/modifiers/state/Keys.scala rename src/test/scala/encry/network/DeliveryManagerTests/{DMUtils.scala => DeliveryManagerUtils.scala} (92%) delete mode 100644 src/test/scala/encry/utils/EncryGenerator.scala create mode 100644 src/test/scala/encry/utils/HistoryGenerator.scala create mode 100644 src/test/scala/encry/utils/Keys.scala create mode 100644 src/test/scala/encry/utils/TestEntityGenerator.scala diff --git a/benchmarks/src/test/scala/benches/StateBenches.scala b/benchmarks/src/test/scala/benches/StateBenches.scala index d541356f4a..6a82660aca 100644 --- a/benchmarks/src/test/scala/benches/StateBenches.scala +++ b/benchmarks/src/test/scala/benches/StateBenches.scala @@ -6,15 +6,12 @@ import java.util.concurrent.TimeUnit import benches.StateBenches.StateBenchState import org.openjdk.jmh.annotations._ import benches.Utils._ -import encry.EncryApp -import encry.settings.EncryAppSettings import encry.storage.VersionalStorage import encry.storage.VersionalStorage.IODB -import encry.utils.{ChainGenerator, FileHelper} import encry.utils.ChainGenerator._ import encry.utils.FileHelper.getRandomTempDir import encry.view.state.{BoxHolder, UtxoState} -import encryBenchmark.{BenchSettings, Settings} +import encryBenchmark.BenchSettings import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.state.box.AssetBox import org.openjdk.jmh.infra.Blackhole @@ -76,7 +73,8 @@ object StateBenches extends BenchSettings { ) val boxesHolder: BoxHolder = BoxHolder(initialBoxes) var state: UtxoState = utxoFromBoxHolder(boxesHolder, tmpDir, None, settings, VersionalStorage.LevelDB) - val genesisBlock: Block = generateGenesisBlockValidForState(state) + val genesisBlock: Block = genGenesisBlock(privKey.publicImage, settings.constants.InitialEmissionAmount, + settings.constants.InitialDifficulty, settings.constants.GenesisHeight) state = state.applyModifier(genesisBlock).right.get diff --git a/benchmarks/src/test/scala/benches/StateRollbackBench.scala b/benchmarks/src/test/scala/benches/StateRollbackBench.scala index 14f9989e20..c02296d9f7 100644 --- a/benchmarks/src/test/scala/benches/StateRollbackBench.scala +++ b/benchmarks/src/test/scala/benches/StateRollbackBench.scala @@ -6,19 +6,19 @@ import java.util.concurrent.TimeUnit import benches.StateRollbackBench.StateRollbackState import benches.Utils._ import encry.storage.VersionalStorage -import encry.utils.ChainGenerator import encry.utils.CoreTaggedTypes.VersionTag import encry.view.state.{BoxHolder, UtxoState} -import encryBenchmark.{BenchSettings, Settings} +import encryBenchmark.BenchSettings import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.state.box.AssetBox -import org.encryfoundation.common.utils.TaggedTypes.{ADKey, Difficulty} +import org.encryfoundation.common.utils.TaggedTypes.Difficulty import org.openjdk.jmh.annotations.{Benchmark, Mode, Scope, State} import org.openjdk.jmh.infra.Blackhole import org.openjdk.jmh.profile.GCProfiler import org.openjdk.jmh.runner.{Runner, RunnerException} import org.openjdk.jmh.runner.options.{OptionsBuilder, TimeValue, VerboseMode} import encry.utils.FileHelper.getRandomTempDir +import encry.utils.ChainGenerator.{utxoFromBoxHolder, genGenesisBlock} class StateRollbackBench { @@ -26,7 +26,7 @@ class StateRollbackBench { def applyBlocksToTheState(stateBench: StateRollbackState, bh: Blackhole): Unit = { bh.consume { val innerState: UtxoState = - ChainGenerator.utxoFromBoxHolder(stateBench.boxesHolder, getRandomTempDir, None, stateBench.settings, VersionalStorage.IODB) + utxoFromBoxHolder(stateBench.boxesHolder, getRandomTempDir, None, stateBench.settings, VersionalStorage.IODB) val newState = stateBench.chain.foldLeft(innerState -> List.empty[VersionTag]) { case ((state, rootHashes), block) => val newState = state.applyModifier(block).right.get newState -> (rootHashes :+ newState.version) @@ -67,8 +67,9 @@ object StateRollbackBench extends BenchSettings { genHardcodedBox(privKey.publicImage.address.address, nonce) ) val boxesHolder: BoxHolder = BoxHolder(initialBoxes) - var state: UtxoState = ChainGenerator.utxoFromBoxHolder(boxesHolder, tmpDir, None, settings, VersionalStorage.LevelDB) - val genesisBlock: Block = generateGenesisBlockValidForState(state) + var state: UtxoState = utxoFromBoxHolder(boxesHolder, tmpDir, None, settings, VersionalStorage.LevelDB) + val genesisBlock: Block = genGenesisBlock(privKey.publicImage, settings.constants.InitialEmissionAmount, + settings.constants.InitialDifficulty, settings.constants.GenesisHeight) state = state.applyModifier(genesisBlock).right.get diff --git a/benchmarks/src/test/scala/benches/Utils.scala b/benchmarks/src/test/scala/benches/Utils.scala index 256bc04687..66e46e7a26 100644 --- a/benchmarks/src/test/scala/benches/Utils.scala +++ b/benchmarks/src/test/scala/benches/Utils.scala @@ -2,19 +2,15 @@ package benches import java.io.File -import akka.actor.ActorRef import com.typesafe.scalalogging.StrictLogging import encry.modifiers.mempool.TransactionFactory -import encry.settings.{Settings, EncryAppSettings} -import encry.storage.VersionalStorage -import encry.storage.VersionalStorage.{StorageKey, StorageType, StorageValue, StorageVersion} -import encry.storage.iodb.versionalIODB.IODBWrapper +import encry.settings.{EncryAppSettings, Settings} import encry.storage.levelDb.versionalLevelDB.VersionalLevelDBCompanion.{LevelDBVersion, VersionalLevelDbKey, VersionalLevelDbValue} import encry.storage.levelDb.versionalLevelDB._ -import encry.utils.{FileHelper, Mnemonic, NetworkTimeProvider} +import encry.utils.{FileHelper, Keys, NetworkTimeProvider} import encry.view.history.History import encry.view.history.storage.HistoryStorage -import encry.view.state.{BoxHolder, UtxoState} +import encry.view.state.UtxoState import io.iohk.iodb.LSMStore import org.encryfoundation.common.crypto.equihash.EquihashSolution import org.encryfoundation.common.crypto.{PrivateKey25519, PublicKey25519, Signature25519} @@ -27,19 +23,16 @@ import org.encryfoundation.common.modifiers.state.box.{AssetBox, EncryPropositio import org.encryfoundation.common.utils.TaggedTypes._ import org.encryfoundation.prismlang.core.wrapped.BoxedValue import org.iq80.leveldb.Options -import scorex.crypto.hash.{Blake2b256, Digest32} -import scorex.crypto.signatures.{Curve25519, PrivateKey, PublicKey} +import scorex.crypto.hash.Digest32 import scorex.utils.Random +import encry.utils.Utils.randomAddress import scala.collection.immutable import scala.util.{Random => R} -import encry.utils.ChainGenerator._ - -object Utils extends Settings with StrictLogging { - - val mnemonicKey: String = "index another island accuse valid aerobic little absurd bunker keep insect scissors" - val privKey: PrivateKey25519 = createPrivKey(Some(mnemonicKey)) +object Utils extends Keys + with Settings + with StrictLogging { val defaultKeySize: Int = 32 val defaultValueSize: Int = 256 @@ -63,15 +56,6 @@ object Utils extends Settings with StrictLogging { ) :: acc } - def generateGenesisBlockValidForState(state: UtxoState): Block = { - val txs = Seq(coinbaseTransaction(0)) - val header = genHeader.copy( - parentId = Header.GenesisParentId, - height = settings.constants.GenesisHeight - ) - Block(header, Payload(header.id, txs)) - } - def generateNextBlockValidForState(prevBlock: Block, state: UtxoState, box: Seq[AssetBox], @@ -185,8 +169,6 @@ object Utils extends Settings with StrictLogging { def genHardcodedBox(address: Address, nonce: Long): AssetBox = AssetBox(EncryProposition.addressLocked(address), nonce, 10000000L, None) - def randomAddress: Address = Pay2PubKeyAddress(PublicKey @@ Random.randomBytes()).address - def coinbaseTransaction(height: Int): Transaction = TransactionFactory.coinbaseTransactionScratch( privKey.publicImage, System.currentTimeMillis(), @@ -195,30 +177,6 @@ object Utils extends Settings with StrictLogging { height = Height @@ height ) - lazy val coinbaseTransaction: Transaction = { - TransactionFactory.coinbaseTransactionScratch( - privKey.publicImage, - System.currentTimeMillis(), - 10L, - 0, - Height @@ 100 - ) - } - - def createPrivKey(seed: Option[String]): PrivateKey25519 = { - val (privateKey: PrivateKey, publicKey: PublicKey) = Curve25519.createKeyPair( - Blake2b256.hash( - seed.map { - Mnemonic.seedFromMnemonic(_) - } - .getOrElse { - val phrase: String = Mnemonic.entropyToMnemonicCode(scorex.utils.Random.randomBytes(16)) - Mnemonic.seedFromMnemonic(phrase) - }) - ) - PrivateKey25519(privateKey, publicKey) - } - def generateInitialBoxes(qty: Int): immutable.IndexedSeq[AssetBox] = (0 until qty).map(_ => genAssetBox(privKey.publicImage.address.address)) diff --git a/it/src/main/scala/encry/it/docker/Node.scala b/it/src/main/scala/encry/it/docker/Node.scala index 02d8402a43..4a484d8768 100644 --- a/it/src/main/scala/encry/it/docker/Node.scala +++ b/it/src/main/scala/encry/it/docker/Node.scala @@ -5,7 +5,7 @@ import java.net.{InetAddress, InetSocketAddress, URL} import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.it.api.HttpApi -import encry.it.util.KeyHelper.createPrivKey +import encry.utils.Utils.createPrivKey import encry.settings.Settings import org.asynchttpclient._ import org.encryfoundation.common.crypto.{PrivateKey25519, PublicKey25519} diff --git a/it/src/main/scala/encry/it/util/KeyHelper.scala b/it/src/main/scala/encry/it/util/KeyHelper.scala deleted file mode 100644 index 82c8480520..0000000000 --- a/it/src/main/scala/encry/it/util/KeyHelper.scala +++ /dev/null @@ -1,21 +0,0 @@ -package encry.it.util - -import encry.utils.Mnemonic -import org.encryfoundation.common.crypto.PrivateKey25519 -import scorex.crypto.hash.Blake2b256 -import scorex.crypto.signatures.{Curve25519, PrivateKey, PublicKey} - -object KeyHelper { - - def createPrivKey(seed: Option[String]): PrivateKey25519 = { - val (privateKey: PrivateKey, publicKey: PublicKey) = Curve25519.createKeyPair( - Blake2b256.hash( - seed.map { Mnemonic.seedFromMnemonic(_) } - .getOrElse { - val phrase: String = Mnemonic.entropyToMnemonicCode(scorex.utils.Random.randomBytes(16)) - Mnemonic.seedFromMnemonic(phrase) - }) - ) - PrivateKey25519(privateKey, publicKey) - } -} \ No newline at end of file diff --git a/it/src/test/scala/encry/it/transactions/AssetTokenTransactionTest.scala b/it/src/test/scala/encry/it/transactions/AssetTokenTransactionTest.scala index 0544b6a181..20a5eff9f3 100644 --- a/it/src/test/scala/encry/it/transactions/AssetTokenTransactionTest.scala +++ b/it/src/test/scala/encry/it/transactions/AssetTokenTransactionTest.scala @@ -6,9 +6,9 @@ import com.typesafe.scalalogging.StrictLogging import encry.consensus.EncrySupplyController import encry.it.configs.Configs import encry.it.docker.NodesFromDocker -import encry.it.util.KeyHelper._ +import encry.utils.Keys import encry.settings.Settings -import org.encryfoundation.common.crypto.{PrivateKey25519, PublicKey25519} +import org.encryfoundation.common.crypto.PublicKey25519 import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.mempool.transaction.EncryAddress.Address import org.encryfoundation.common.modifiers.mempool.transaction.{PubKeyLockedContract, Transaction} @@ -29,6 +29,7 @@ class AssetTokenTransactionTest extends AsyncFunSuite with ScalaFutures with StrictLogging with NodesFromDocker + with Keys with Settings { override protected def nodeConfigs: Seq[Config] = Seq(Configs.mining(true) @@ -41,8 +42,6 @@ class AssetTokenTransactionTest extends AsyncFunSuite val firstHeightToWait: Int = 5 val secondHeightToWait: Int = 8 val thirdHeightToWait: Int = 11 - val mnemonicKey: String = "index another island accuse valid aerobic little absurd bunker keep insect scissors" - val privKey: PrivateKey25519 = createPrivKey(Some(mnemonicKey)) val waitTime: FiniteDuration = 30.minutes val amount: Int = scala.util.Random.nextInt(2000) val fee: Int = scala.util.Random.nextInt(500) diff --git a/it/src/test/scala/encry/it/transactions/DataTransactionTest.scala b/it/src/test/scala/encry/it/transactions/DataTransactionTest.scala index 7388629245..c025496a90 100644 --- a/it/src/test/scala/encry/it/transactions/DataTransactionTest.scala +++ b/it/src/test/scala/encry/it/transactions/DataTransactionTest.scala @@ -5,8 +5,7 @@ import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.it.configs.Configs import encry.it.docker.NodesFromDocker -import encry.it.util.KeyHelper._ -import org.encryfoundation.common.crypto.PrivateKey25519 +import encry.utils.Keys import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.mempool.transaction.{PubKeyLockedContract, Transaction} import org.encryfoundation.common.modifiers.state.box.{AssetBox, EncryBaseBox} @@ -19,6 +18,7 @@ import scala.concurrent.duration._ class DataTransactionTest extends AsyncFunSuite with Matchers with ScalaFutures + with Keys with StrictLogging with NodesFromDocker { @@ -30,8 +30,6 @@ class DataTransactionTest extends AsyncFunSuite val firstHeightToWait: Int = 5 val secondHeightToWait: Int = 8 - val mnemonicKey: String = "index another island accuse valid aerobic little absurd bunker keep insect scissors" - val privKey: PrivateKey25519 = createPrivKey(Some(mnemonicKey)) val waitTime: FiniteDuration = 30.minutes val fee: Long = scala.util.Random.nextInt(500) @@ -44,7 +42,7 @@ class DataTransactionTest extends AsyncFunSuite fee, System.currentTimeMillis(), IndexedSeq(oneBox).map(_ -> None), - PubKeyLockedContract(privKey.publicImage.pubKeyBytes).contract, + PubKeyLockedContract(publicKey.pubKeyBytes).contract, Random.randomBytes(32) ) diff --git a/it/src/test/scala/encry/it/transactions/ProcessingTransferTransactionWithEncryCoinsTest.scala b/it/src/test/scala/encry/it/transactions/ProcessingTransferTransactionWithEncryCoinsTest.scala index d8ee6981e4..9dfe0a5130 100644 --- a/it/src/test/scala/encry/it/transactions/ProcessingTransferTransactionWithEncryCoinsTest.scala +++ b/it/src/test/scala/encry/it/transactions/ProcessingTransferTransactionWithEncryCoinsTest.scala @@ -6,9 +6,9 @@ import com.typesafe.scalalogging.StrictLogging import encry.consensus.EncrySupplyController import encry.it.configs.Configs import encry.it.docker.NodesFromDocker -import encry.it.util.KeyHelper._ +import encry.utils.Keys import encry.settings.Settings -import org.encryfoundation.common.crypto.{PrivateKey25519, PublicKey25519} +import org.encryfoundation.common.crypto.PublicKey25519 import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.mempool.transaction.EncryAddress.Address import org.encryfoundation.common.modifiers.mempool.transaction.Transaction @@ -28,6 +28,7 @@ class ProcessingTransferTransactionWithEncryCoinsTest extends AsyncFunSuite with ScalaFutures with StrictLogging with NodesFromDocker + with Keys with Settings { override protected def nodeConfigs: Seq[Config] = Seq(Configs.mining(true) @@ -40,8 +41,6 @@ class ProcessingTransferTransactionWithEncryCoinsTest extends AsyncFunSuite val fee: Long = scala.util.Random.nextInt(500) val firstHeightToWait: Int = 5 val secondHeightToWait: Int = 8 - val mnemonicKey: String = "index another island accuse valid aerobic little absurd bunker keep insect scissors" - val privKey: PrivateKey25519 = createPrivKey(Some(mnemonicKey)) val recipientAddress: Address = PublicKey25519(Curve25519.createKeyPair(Random.randomBytes())._2).address.address val waitTime: FiniteDuration = 30.minutes diff --git a/src/main/scala/encry/modifiers/mempool/TransactionFactory.scala b/src/main/scala/encry/modifiers/mempool/TransactionFactory.scala index 999f209300..899fa717d8 100644 --- a/src/main/scala/encry/modifiers/mempool/TransactionFactory.scala +++ b/src/main/scala/encry/modifiers/mempool/TransactionFactory.scala @@ -2,7 +2,7 @@ package encry.modifiers.mempool import com.typesafe.scalalogging.StrictLogging import org.encryfoundation.common.crypto.{PrivateKey25519, PublicKey25519, Signature25519} -import org.encryfoundation.common.modifiers.mempool.directive.{DataDirective, Directive, TransferDirective} +import org.encryfoundation.common.modifiers.mempool.directive.{AssetIssuingDirective, DataDirective, Directive, ScriptedAssetDirective, TransferDirective} import org.encryfoundation.common.modifiers.mempool.transaction.EncryAddress.Address import org.encryfoundation.common.modifiers.mempool.transaction._ import org.encryfoundation.common.modifiers.state.box.Box.Amount @@ -82,8 +82,8 @@ object TransactionFactory extends StrictLogging { else directivesSeq val uTransaction: UnsignedTransaction = UnsignedTransaction(fee, timestamp, uInputs, directives) - val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) - val proofs: IndexedSeq[Seq[Proof]] = useOutputs.flatMap(_._2.map(_._2)).toIndexedSeq + val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) + val proofs: IndexedSeq[Seq[Proof]] = useOutputs.flatMap(_._2.map(_._2)).toIndexedSeq uTransaction.toSigned(proofs, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) } @@ -150,9 +150,36 @@ object TransactionFactory extends StrictLogging { else directives val uTransaction: UnsignedTransaction = UnsignedTransaction(fee, timestamp, uInputs, newDirectives) - val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) + uTransaction.toSigned(IndexedSeq.empty, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) + } + + def assetIssuingTransactionScratch(privKey: PrivateKey25519, + fee: Long, + timestamp: Long, + useOutputs: IndexedSeq[MonetaryBox], + amount: Long, + numOfOutputs: Int = 5): Transaction = { + val directives: IndexedSeq[AssetIssuingDirective] = + (0 until numOfOutputs).foldLeft(IndexedSeq.empty[AssetIssuingDirective]) { case (directivesAll, _) => + directivesAll :+ AssetIssuingDirective(PubKeyLockedContract(privKey.publicImage.pubKeyBytes).contract.hash, amount) + } + val pubKey: PublicKey25519 = privKey.publicImage + + val uInputs: IndexedSeq[Input] = useOutputs + .map(bx => Input.unsigned(bx.id, Right(PubKeyLockedContract(pubKey.pubKeyBytes)))) + .toIndexedSeq + + val change: Amount = useOutputs.map(_.amount).sum - (amount + fee) + + val newDirectives: IndexedSeq[Directive] = + if (change > 0) TransferDirective(pubKey.address.address, amount, None) +: (0 until numOfOutputs).map(_ => + TransferDirective(pubKey.address.address, change / numOfOutputs, None)) ++: directives + else directives + + val uTransaction: UnsignedTransaction = UnsignedTransaction(fee, timestamp, uInputs, newDirectives) + val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) uTransaction.toSigned(IndexedSeq.empty, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) } diff --git a/src/main/scala/encry/network/PeerConnectionHandler.scala b/src/main/scala/encry/network/PeerConnectionHandler.scala index d4fe617311..31beac438b 100755 --- a/src/main/scala/encry/network/PeerConnectionHandler.scala +++ b/src/main/scala/encry/network/PeerConnectionHandler.scala @@ -14,11 +14,12 @@ import encry.network.PeerConnectionHandler.{AwaitingHandshake, CommunicationStat import encry.network.PeerConnectionHandler.ReceivableMessages._ import encry.network.PeersKeeper.{ConnectionStopped, HandshakedDone} import encry.settings.NetworkSettings +import encry.utils.Utils.protocolToBytes import org.encryfoundation.common.network.BasicMessagesRepo.{GeneralizedNetworkMessage, Handshake, NetworkMessage} import org.encryfoundation.common.utils.Algos import scala.annotation.tailrec -import scala.collection.immutable.{HashMap, SortedMap} +import scala.collection.immutable. SortedMap import scala.concurrent.ExecutionContextExecutor import scala.concurrent.duration._ import scala.util.{Failure, Random, Success} @@ -281,7 +282,6 @@ class PeerConnectionHandler(connection: ActorRef, multiPacket(List[ByteString](), data) } - private def protocolToBytes(protocol: String): Array[Byte] = protocol.split("\\.").map(elem => elem.toByte) } object PeerConnectionHandler { diff --git a/src/main/scala/encry/utils/Utils.scala b/src/main/scala/encry/utils/Utils.scala index 3e9c03c4a1..0fab0190dc 100644 --- a/src/main/scala/encry/utils/Utils.scala +++ b/src/main/scala/encry/utils/Utils.scala @@ -1,9 +1,15 @@ package encry.utils import com.google.common.primitives.Longs +import org.encryfoundation.common.crypto.PrivateKey25519 +import org.encryfoundation.common.modifiers.mempool.transaction.EncryAddress.Address +import org.encryfoundation.common.modifiers.mempool.transaction.Pay2PubKeyAddress import org.encryfoundation.common.network.BasicMessagesRepo.BasicMsgDataTypes.InvData import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} +import scorex.crypto.hash.Blake2b256 +import scorex.crypto.signatures.{Curve25519, PrivateKey, PublicKey} +import scorex.utils.{Random => ScorexRandom} object Utils { @@ -19,4 +25,21 @@ object Utils { idsToString(ids.map(id => (modifierType, id))) def idsToString(invData: InvData): String = idsToString(invData._1, invData._2) + + def randomAddress: Address = Pay2PubKeyAddress(PublicKey @@ ScorexRandom.randomBytes()).address + + def protocolToBytes(protocol: String): Array[Byte] = protocol.split("\\.").map(elem => elem.toByte) + + def createPrivKey(seed: Option[String]): PrivateKey25519 = { + val (privateKey: PrivateKey, publicKey: PublicKey) = Curve25519.createKeyPair( + Blake2b256.hash( + seed.map { Mnemonic.seedFromMnemonic(_) } + .getOrElse { + val phrase: String = Mnemonic.entropyToMnemonicCode(ScorexRandom.randomBytes(16)) + Mnemonic.seedFromMnemonic(phrase) + }) + ) + PrivateKey25519(privateKey, publicKey) + } + } \ No newline at end of file diff --git a/src/test/scala/encry/api/http/routes/TransactionsApiRouteSpec.scala b/src/test/scala/encry/api/http/routes/TransactionsApiRouteSpec.scala index b26f5a8f47..8a591f7540 100644 --- a/src/test/scala/encry/api/http/routes/TransactionsApiRouteSpec.scala +++ b/src/test/scala/encry/api/http/routes/TransactionsApiRouteSpec.scala @@ -1,12 +1,12 @@ package encry.api.http.routes -import encry.modifiers.InstanceFactory import io.circe.Decoder.Result import io.circe.Json import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.scalatest.{Matchers, PropSpec} +import encry.utils.TestEntityGenerator.paymentTransactionValid -class TransactionsApiRouteSpec extends PropSpec with Matchers with InstanceFactory { +class TransactionsApiRouteSpec extends PropSpec with Matchers { private val tx = paymentTransactionValid diff --git a/src/test/scala/encry/consensus/emission/EncrySupplyControllerTest.scala b/src/test/scala/encry/consensus/emission/EncrySupplyControllerTest.scala index 655da749b5..fd71731e87 100644 --- a/src/test/scala/encry/consensus/emission/EncrySupplyControllerTest.scala +++ b/src/test/scala/encry/consensus/emission/EncrySupplyControllerTest.scala @@ -2,13 +2,12 @@ package encry.consensus.emission import encry.consensus.EncrySupplyController import encry.settings.Settings -import encry.utils.EncryGenerator import org.encryfoundation.common.utils.TaggedTypes.Height import org.scalatest.{Matchers, PropSpec} import scala.concurrent.duration._ -class EncrySupplyControllerTest extends PropSpec with Matchers with EncryGenerator with Settings { +class EncrySupplyControllerTest extends PropSpec with Matchers with Settings { val epochLen = 10 diff --git a/src/test/scala/encry/modifiers/InstanceFactory.scala b/src/test/scala/encry/modifiers/InstanceFactory.scala deleted file mode 100755 index 2610aea5ec..0000000000 --- a/src/test/scala/encry/modifiers/InstanceFactory.scala +++ /dev/null @@ -1,243 +0,0 @@ -package encry.modifiers - -import encry.consensus.EquihashPowScheme -import encry.crypto.equihash.EquihashValidationErrors -import encry.modifiers.mempool._ -import encry.modifiers.state.Keys -import encry.settings.{EncryAppSettings, NodeSettings} -import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} -import encry.utils.{EncryGenerator, FileHelper, NetworkTimeProvider, TestHelper} -import encry.view.history.History -import encry.view.history.storage.HistoryStorage -import io.iohk.iodb.LSMStore -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} -import org.encryfoundation.common.modifiers.mempool.transaction.{Input, Transaction} -import org.encryfoundation.common.modifiers.state.box.{AssetBox, EncryProposition} -import org.encryfoundation.common.modifiers.state.box.Box.Amount -import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{Height, _} - -import org.encryfoundation.prismlang.compiler.CompiledContract -import org.encryfoundation.prismlang.core.Ast.Expr -import org.encryfoundation.prismlang.core.{Ast, Types} -import org.iq80.leveldb.Options -import scorex.crypto.hash.Digest32 -import scorex.utils.Random - -import scala.util.{Random => Scarand} - -trait InstanceFactory extends Keys with EncryGenerator { - - private val genHelper = TestHelper - - lazy val fakeTransaction: Transaction = { - val fee = genHelper.Props.txFee - val useBoxes = IndexedSeq(genHelper.genAssetBox(publicKey.address.address), - genHelper.genAssetBox(publicKey.address.address)) - - TransactionFactory.defaultPaymentTransactionScratch(secret, fee, timestamp, useBoxes, - publicKey.address.address, 12345678L) - } - - lazy val paymentTransactionValid: Transaction = { - val fee: Amount = genHelper.Props.txFee - val useBoxes: IndexedSeq[AssetBox] = IndexedSeq(genHelper.genAssetBox(publicKey.address.address), - genHelper.genAssetBox(publicKey.address.address)) - - TransactionFactory.defaultPaymentTransactionScratch(secret, fee, timestamp, useBoxes, - publicKey.address.address, genHelper.Props.txAmount) - } - - def generateGenesisBlock(genesisHeight: Height): Block = { - val txs: Seq[Transaction] = Seq(coinbaseTransaction) - val txsRoot: Digest32 = Payload.rootHash(txs.map(_.id)) - val header = genHeader.copy( - parentId = Header.GenesisParentId, - height = genesisHeight, - transactionsRoot = txsRoot - ) - Block(header, Payload(header.id, txs)) - } - - def paymentTransactionDynamic: Transaction = { - val fee = genHelper.Props.txFee - val useBoxes = (0 to 5).map(_ => { - AssetBox( - EncryProposition.pubKeyLocked(secret.publicImage.pubKeyBytes), - Scarand.nextLong(), - 999L - ) - }) - - TransactionFactory.defaultPaymentTransactionScratch(secret, fee, Scarand.nextLong(), useBoxes, - publicKey.address.address, genHelper.Props.txAmount) - } - - lazy val paymentTransactionInvalid: Transaction = { - val useBoxes = IndexedSeq(genHelper.genAssetBox(publicKey.address.address)) - - TransactionFactory.defaultPaymentTransactionScratch(secret, -100, timestamp, useBoxes, - randomAddress, genHelper.Props.txAmount) - } - - lazy val coinbaseTransaction: Transaction = { - TransactionFactory.coinbaseTransactionScratch(secret.publicImage, timestamp, 10L, 0, Height @@ 100) - } - - def coinbaseTransactionWithDiffSupply(supply: Long = 10L): Transaction = { - TransactionFactory.coinbaseTransactionScratch(secret.publicImage, timestamp, supply, 0, Height @@ 100) - } - - lazy val AssetBoxI: AssetBox = - AssetBox( - EncryProposition.pubKeyLocked(secret.publicImage.pubKeyBytes), - 999L, - 100000L - ) - - lazy val OpenAssetBoxI: AssetBox = - AssetBox( - EncryProposition.open, - 999L, - 100000L - ) - - lazy val Contract: CompiledContract = CompiledContract( - List("state" -> Types.EncryState), - Expr.If( - Expr.Compare( - Expr.Attribute( - Expr.Name( - Ast.Ident("state"), - Types.EncryState - ), - Ast.Ident("height"), - Types.PInt - ), - List(Ast.CompOp.GtE), - List(Expr.IntConst(1000L)) - ), - Expr.True, - Expr.False, - Types.PBoolean - ) - ) - - lazy val UnsignedInput: Input = Input(ADKey @@ Random.randomBytes(), Left(Contract), List.empty) - - def generateFakeChain(blocksQty: Int, genesisHeight: Height): Seq[Block] = { - val srand = new Scarand() - (0 until blocksQty).foldLeft(Seq.empty[Block], Seq.empty[AssetBox]) { - case ((fakeBlockchain, utxo), blockHeight) => - val block = if (fakeBlockchain.isEmpty) generateGenesisBlock(genesisHeight) else { - val addr = randomAddress - val txs = - utxo.map(box => genValidPaymentTxToAddrWithSpentBoxes(IndexedSeq(box), addr)) ++ - Seq(coinbaseTransactionWithDiffSupply(Math.abs(srand.nextInt(1000)))) - val txsRoot = Algos.merkleTreeRoot(txs.map(tx => LeafData @@ tx.id.untag(ModifierId))) - val header = genHeaderAtHeight(blockHeight, txsRoot) - val payload = Payload(header.id, txs) - Block(header, payload) - } - val newUtxo = block.payload.txs.flatMap(_.newBoxes) - (fakeBlockchain :+ block) -> newUtxo.collect{case ab: AssetBox => ab} - } - }._1 - - def generateNextBlock(history: History, - difficultyDiff: BigInt = 0, - prevId: Option[ModifierId] = None, - txsQty: Int = 100, - additionalDifficulty: BigInt = 0): Block = { - val previousHeaderId: ModifierId = - prevId.getOrElse(history.getBestHeader.map(_.id).getOrElse(Header.GenesisParentId)) - val requiredDifficulty: Difficulty = history.getBestHeader.map(parent => - history.requiredDifficultyAfter(parent).getOrElse(Difficulty @@ BigInt(0))) - .getOrElse(history.settings.constants.InitialDifficulty) - val txs = (if (txsQty != 0) genValidPaymentTxs(Scarand.nextInt(txsQty)) else Seq.empty) ++ - Seq(coinbaseTransaction) - val header = genHeader.copy( - parentId = previousHeaderId, - height = history.getBestHeaderHeight + 1, - difficulty = Difficulty @@ (requiredDifficulty + difficultyDiff + additionalDifficulty), - transactionsRoot = Payload.rootHash(txs.map(_.id)) - ) - - Block(header, Payload(header.id, txs)) - } - - def genForkOn(qty: Int, - addDifficulty: BigInt = 0, - from: Int, - to: Int, - settings: EncryAppSettings): (List[Block], List[Block]) = { - val history: History = generateDummyHistory(settings) - val forkInterval = (from until to).toList - (0 until qty).foldLeft((List(history), List.empty[Block] -> List.empty[Block])) { - case ((histories, blocks), blockHeight) => - if (forkInterval.contains(blockHeight)) { - if (histories.length == 1) { - val secondHistory = generateDummyHistory(settings) - blocks._1.foldLeft(secondHistory) { - case (prevHistory, blockToApply) => - prevHistory.append(blockToApply.header) - prevHistory.append(blockToApply.payload) - prevHistory.reportModifierIsValid(blockToApply) - prevHistory - } - val nextBlockInFirstChain = generateNextBlock(histories.head) - val nextBlockInSecondChain = generateNextBlock(secondHistory, additionalDifficulty = addDifficulty) - histories.head.append(nextBlockInFirstChain.header) - histories.head.append(nextBlockInFirstChain.payload) - val a = histories.head.reportModifierIsValid(nextBlockInFirstChain) - secondHistory.append(nextBlockInSecondChain.header) - secondHistory.append(nextBlockInSecondChain.payload) - val b = secondHistory.reportModifierIsValid(nextBlockInSecondChain) - (List(a, b), (blocks._1 :+ nextBlockInFirstChain) -> List(nextBlockInSecondChain)) - } else { - val nextBlockInFirstChain = generateNextBlock(histories.head) - val nextBlockInSecondChain = generateNextBlock(histories.last, additionalDifficulty = addDifficulty) - histories.head.append(nextBlockInFirstChain.header) - histories.head.append(nextBlockInFirstChain.payload) - val a = histories.head.reportModifierIsValid(nextBlockInFirstChain) - histories.last.append(nextBlockInSecondChain.header) - histories.last.append(nextBlockInSecondChain.payload) - val b = histories.last.reportModifierIsValid(nextBlockInSecondChain) - (List(a, b), (blocks._1 :+ nextBlockInFirstChain) -> (blocks._2 :+ nextBlockInSecondChain)) - } - } else { - val block: Block = generateNextBlock(histories.head) - histories.head.append(block.header) - histories.head.append(block.payload) - val a = histories.head.reportModifierIsValid(block) - (List(a), (blocks._1 :+ block) -> blocks._2) - } - }._2 - } - - def generateDummyHistory(settings: EncryAppSettings): History = { - - val indexStore: LSMStore = new LSMStore(FileHelper.getRandomTempDir, keepVersions = 0) - val objectsStore: LSMStore = new LSMStore(FileHelper.getRandomTempDir, keepVersions = 0) - val levelDBInit = LevelDbFactory.factory.open(FileHelper.getRandomTempDir, new Options) - val vldbInit = VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settings.levelDB)) - val storage: HistoryStorage = new HistoryStorage(vldbInit) - - val ntp: NetworkTimeProvider = new NetworkTimeProvider(settings.ntp) - - class EquihashPowSchemeWithoutValidateSolution(n: Char, k: Char, version: Byte, preGenesisHeight: Height, maxTarget: BigInt) - extends EquihashPowScheme(n: Char, k: Char, version: Byte, preGenesisHeight: Height, maxTarget: BigInt) { - override def verify(header: Header): Either[EquihashValidationErrors, Boolean] = Right(true) - } - - val equihashPowSchemeWithoutValidateSolution: EquihashPowScheme = - new EquihashPowSchemeWithoutValidateSolution(settings.constants.n, settings.constants.k, - settings.constants.Version, settings.constants.PreGenesisHeight, settings.constants.MaxTarget) - - new History { - override val historyStorage: HistoryStorage = storage - override val timeProvider: NetworkTimeProvider = ntp - override val powScheme: EquihashPowScheme = equihashPowSchemeWithoutValidateSolution - } - } -} \ No newline at end of file diff --git a/src/test/scala/encry/modifiers/history/BlockSerializerTest.scala b/src/test/scala/encry/modifiers/history/BlockSerializerTest.scala index 5c799b1a3b..02a5d1e3e5 100755 --- a/src/test/scala/encry/modifiers/history/BlockSerializerTest.scala +++ b/src/test/scala/encry/modifiers/history/BlockSerializerTest.scala @@ -2,7 +2,7 @@ package encry.modifiers.history import encry.modifiers.mempool.TransactionFactory import encry.settings.Settings -import encry.utils.{EncryGenerator, TestHelper} +import encry.utils.TestHelper import org.encryfoundation.common.crypto.equihash.EquihashSolution import org.encryfoundation.common.modifiers.history._ import org.encryfoundation.common.utils.Algos @@ -10,8 +10,9 @@ import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.scalatest.FunSuite import scorex.crypto.hash.Digest32 import scorex.utils.Random +import encry.utils.Utils.randomAddress -class BlockSerializerTest extends FunSuite with EncryGenerator with Settings { +class BlockSerializerTest extends FunSuite with Settings { test("testToBytes $ testFromBytes") { diff --git a/src/test/scala/encry/modifiers/history/HeaderSerializerTest.scala b/src/test/scala/encry/modifiers/history/HeaderSerializerTest.scala index 3baea5e231..691687af81 100755 --- a/src/test/scala/encry/modifiers/history/HeaderSerializerTest.scala +++ b/src/test/scala/encry/modifiers/history/HeaderSerializerTest.scala @@ -1,10 +1,10 @@ package encry.modifiers.history -import encry.utils.EncryGenerator import org.encryfoundation.common.modifiers.history.HeaderSerializer import org.scalatest.{Matchers, PropSpec} +import encry.utils.TestEntityGenerator.genHeader -class HeaderSerializerTest extends PropSpec with Matchers with EncryGenerator{ +class HeaderSerializerTest extends PropSpec with Matchers { property("testToBytes & testParseBytes") { diff --git a/src/test/scala/encry/modifiers/history/HeaderSpec.scala b/src/test/scala/encry/modifiers/history/HeaderSpec.scala index 569c3a95da..60c4c22e43 100644 --- a/src/test/scala/encry/modifiers/history/HeaderSpec.scala +++ b/src/test/scala/encry/modifiers/history/HeaderSpec.scala @@ -1,12 +1,12 @@ package encry.modifiers.history -import encry.utils.EncryGenerator import org.scalatest.{Matchers, PropSpec} import scorex.crypto.hash.Blake2b256 +import encry.utils.TestEntityGenerator.genHeader -class HeaderSpec extends PropSpec with Matchers with EncryGenerator{ +class HeaderSpec extends PropSpec with Matchers { - property("Different headers should have different hash"){ + property("Different headers should have different hash") { val header = genHeader diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 2bad2fb691..8cee2ff896 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -4,12 +4,11 @@ import java.io.File import akka.actor.ActorSystem import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} -import encry.modifiers.InstanceFactory import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.settings.{EncryAppSettings, Settings} import encry.storage.VersionalStorage import encry.utils.ChainGenerator._ -import encry.utils.FileHelper +import encry.utils.{FileHelper, Keys} import encry.view.actors.HistoryApplicator import encry.view.actors.NodeViewHolder.ReceivableMessages.{LocallyGeneratedBlock, ModifierFromRemote} import encry.view.history.History @@ -18,22 +17,28 @@ import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.utils.Algos import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +import encry.utils.HistoryGenerator.dummyHistory import scala.collection.Seq import scala.concurrent.duration._ -class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike with ImplicitSender with BeforeAndAfterAll with Matchers - with InstanceFactory with OneInstancePerTest with Settings { +class HistoryApplicatorTest extends TestKit(ActorSystem()) + with WordSpecLike + with ImplicitSender + with BeforeAndAfterAll + with Matchers + with OneInstancePerTest + with Keys + with Settings { + + override def afterAll: Unit = shutdown(system) - override def afterAll: Unit = { - TestKit.shutdownActorSystem(system) - } val testSettings: EncryAppSettings = settings.copy(storage = settings.storage.copy(state = VersionalStorage.LevelDB)) def blockToModifiers(block: Block): Seq[PersistentModifier] = Seq(block.header, block.payload) def rollbackTest(transQty: Int) { - val history: History = generateDummyHistory(testSettings) + val history: History = dummyHistory(testSettings, withoutPow = true) //generate invalid blocks begining from 6 blocks val (initialState, state, chain) = genChain(privKey, dir, testSettings, 10, transQty, Some(6)) @@ -76,7 +81,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w "apply locall blocks and check chain sync" in { val blockQty = 10 - val history: History = generateDummyHistory(testSettings) + val history: History = dummyHistory(testSettings, withoutPow = true) val (initialState, state, chain) = genChain(privKey, dir, testSettings, blockQty) val historyApplicator: TestActorRef[HistoryApplicator] = @@ -94,10 +99,11 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w checkBestBlock(history, chain(blockQty - 1)) } + "apply remote blocks and check chain sync" in { val blockQty = 10 - val history: History = generateDummyHistory(testSettings) + val history: History = dummyHistory(testSettings, withoutPow = true) val (initialState, state, chain) = genChain(privKey, dir, testSettings, blockQty) val modifiers: Seq[PersistentModifier] = chain.flatMap(blockToModifiers) @@ -121,7 +127,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w val overQty = 30 - val history: History = generateDummyHistory(testSettings) + val history: History = dummyHistory(testSettings, withoutPow = true) val (initialState, state, chain) = genChain(privKey, dir, testSettings, testSettings.levelDB.maxVersions + overQty, 10) val historyApplicator: TestActorRef[HistoryApplicator] = @@ -152,7 +158,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w "sync when apply headers and payloads separately" in { val blockQty = 10 - val history: History = generateDummyHistory(testSettings) + val history: History = dummyHistory(testSettings, withoutPow = true) val (initialState, state, chain) = genChain(privKey, dir, testSettings, blockQty) val headers: Seq[PersistentModifier] = chain.map(_.header) @@ -180,7 +186,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w "sync and rollback headers for invalid payload" in { val blockQty = 10 - val history: History = generateDummyHistory(testSettings) + val history: History = dummyHistory(testSettings, withoutPow = true) //generate invalid blocks begining from 6 blocks val (initialState, state, chain) = genChain(privKey, dir, testSettings, blockQty, 100, Some(6)) @@ -196,7 +202,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w headers.foreach(historyApplicator ! ModifierFromRemote(_)) - awaitCond(history.getBestHeaderHeight == blockQty - 1, timeout, 100 millis, + awaitCond(history.getBestHeaderHeight == blockQty - 1, timeout, 500 millis, s"history.getBestBlockHeight ${history.getBestHeaderHeight} expected ${blockQty - 1}") history.getBestHeaderHeight shouldBe blockQty - 1 history.getBestHeader.map(h => Algos.encode(h.id)) shouldBe Some(Algos.encode(chain(blockQty - 1).header.id)) @@ -204,11 +210,10 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) with WordSpecLike w payloads.foreach(historyApplicator ! ModifierFromRemote(_)) expectMsg(timeout, FullBlockChainIsSynced()) - awaitCond(history.getBestBlockHeight == 4, timeout, 100 millis, + awaitCond(history.getBestBlockHeight == 4, timeout, 500 millis, s"history.getBestBlockHeight ${history.getBestBlockHeight} expected 4") checkBestBlock(history, chain(4)) } - } } diff --git a/src/test/scala/encry/modifiers/history/ModifiersProtoTest.scala b/src/test/scala/encry/modifiers/history/ModifiersProtoTest.scala index ef8b9812e8..34c1345d84 100644 --- a/src/test/scala/encry/modifiers/history/ModifiersProtoTest.scala +++ b/src/test/scala/encry/modifiers/history/ModifiersProtoTest.scala @@ -5,25 +5,65 @@ import BoxesProto.BoxProtoMessage import HeaderProto.HeaderProtoMessage import PayloadProto.PayloadProtoMessage import TransactionProto.TransactionProtoMessage -import encry.modifiers.InstanceFactory +import encry.utils.TestEntityGenerator._ +import org.encryfoundation.common.crypto.{PrivateKey25519, PublicKey25519, Signature25519} import org.encryfoundation.common.modifiers.history._ import org.encryfoundation.common.modifiers.mempool.directive._ -import org.encryfoundation.common.modifiers.mempool.transaction.{Pay2PubKeyAddress, PubKeyLockedContract, Transaction, TransactionProtoSerializer} +import org.encryfoundation.common.modifiers.mempool.transaction._ +import org.encryfoundation.common.modifiers.state.box.Box.Amount import org.encryfoundation.common.modifiers.state.box._ import org.encryfoundation.common.utils.TaggedTypes.ADKey +import org.encryfoundation.prismlang.core.wrapped.BoxedValue import org.scalatest.{Matchers, PropSpec} import scorex.crypto.signatures.PublicKey import scorex.utils.Random import scala.util.Try -class ModifiersProtoTest extends PropSpec with Matchers with InstanceFactory { +class ModifiersProtoTest extends PropSpec with Matchers { //todo add tests for merkel root, sing + def universalTransactionScratch(privKey: PrivateKey25519, + fee: Long, + timestamp: Long, + useOutputs: IndexedSeq[MonetaryBox], + amount: Long, + numOfOutputs: Int = 5): Transaction = { + val directives: IndexedSeq[Directive] = IndexedSeq( + AssetIssuingDirective(PubKeyLockedContract(publicKey.pubKeyBytes).contract.hash, amount), + DataDirective(PubKeyLockedContract(publicKey.pubKeyBytes).contract.hash, Random.randomBytes()), + ScriptedAssetDirective(PubKeyLockedContract(publicKey.pubKeyBytes).contract.hash, 10L, + Option(ADKey @@ Random.randomBytes())), + ScriptedAssetDirective(PubKeyLockedContract(publicKey.pubKeyBytes).contract.hash, 10L, + Option.empty[ADKey]), + TransferDirective(publicKey.address.address, 10L, Option(ADKey @@ Random.randomBytes())), + TransferDirective(publicKey.address.address, 10L, Option.empty[ADKey]) + ) + + val pubKey: PublicKey25519 = publicKey + + val uInputs: IndexedSeq[Input] = useOutputs + .map(bx => Input.unsigned(bx.id, Right(PubKeyLockedContract(pubKey.pubKeyBytes)))) + .toIndexedSeq + + val change: Amount = useOutputs.map(_.amount).sum - (amount + fee) + + val newDirectives: IndexedSeq[Directive] = + if (change > 0) TransferDirective(pubKey.address.address, amount, None) +: (0 until numOfOutputs).map(_ => + TransferDirective(pubKey.address.address, change / numOfOutputs, None)) ++: directives + else directives + + val uTransaction: UnsignedTransaction = UnsignedTransaction(fee, timestamp, uInputs, newDirectives) + + val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) + + uTransaction.toSigned(IndexedSeq.empty, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) + } + property("AssetIssuingDirective should be serialized correctly") { val assetIssuingDirective: AssetIssuingDirective = - AssetIssuingDirective(PubKeyLockedContract(privKey.publicImage.pubKeyBytes).contract.hash, 1000L) + AssetIssuingDirective(PubKeyLockedContract(publicKey.pubKeyBytes).contract.hash, 1000L) val assetIssuingDirectiveToProto: TransactionProtoMessage.DirectiveProtoMessage = assetIssuingDirective.toDirectiveProto val assetIssuingDirectiveFromProto: Option[AssetIssuingDirective] = AssetIssuingDirectiveProtoSerializer.fromProto(assetIssuingDirectiveToProto) @@ -34,7 +74,7 @@ class ModifiersProtoTest extends PropSpec with Matchers with InstanceFactory { property("DataDirective should be serialized correctly") { val dataDirective: DataDirective = - DataDirective(PubKeyLockedContract(privKey.publicImage.pubKeyBytes).contract.hash, Random.randomBytes()) + DataDirective(PubKeyLockedContract(publicKey.pubKeyBytes).contract.hash, Random.randomBytes()) val dataDirectiveToProto: TransactionProtoMessage.DirectiveProtoMessage = dataDirective.toDirectiveProto val dataDirectiveFromProto = DataDirectiveProtoSerializer.fromProto(dataDirectiveToProto) dataDirectiveFromProto.isDefined shouldBe true @@ -44,7 +84,7 @@ class ModifiersProtoTest extends PropSpec with Matchers with InstanceFactory { property("ScriptedAssetDirective should be serialized correctly") { val scriptedAssetDirectiveWithTokenId: ScriptedAssetDirective = - ScriptedAssetDirective(PubKeyLockedContract(privKey.publicImage.pubKeyBytes).contract.hash, 10L, + ScriptedAssetDirective(PubKeyLockedContract(publicKey.pubKeyBytes).contract.hash, 10L, Option(ADKey @@ Random.randomBytes())) val scriptedAssetDirectiveWithTokenIdToProto: TransactionProtoMessage.DirectiveProtoMessage = scriptedAssetDirectiveWithTokenId.toDirectiveProto @@ -56,7 +96,7 @@ class ModifiersProtoTest extends PropSpec with Matchers with InstanceFactory { scriptedAssetDirectiveWithTokenId.tokenIdOpt.get.sameElements(scriptedAssetDirectiveWithTokenIdFromProto.get.tokenIdOpt.get) val scriptedAssetDirectiveWithoutTokenId: ScriptedAssetDirective = - ScriptedAssetDirective(PubKeyLockedContract(privKey.publicImage.pubKeyBytes).contract.hash, 10L, + ScriptedAssetDirective(PubKeyLockedContract(publicKey.pubKeyBytes).contract.hash, 10L, Option.empty[ADKey]) val scriptedAssetDirectiveWithoutTokenIdToProto: TransactionProtoMessage.DirectiveProtoMessage = scriptedAssetDirectiveWithoutTokenId.toDirectiveProto @@ -70,7 +110,7 @@ class ModifiersProtoTest extends PropSpec with Matchers with InstanceFactory { property("TransferDirective should be serialized correctly") { val transferDirectiveWithTokenId: TransferDirective = - TransferDirective(privKey.publicImage.address.address, 10L, Option(ADKey @@ Random.randomBytes())) + TransferDirective(publicKey.address.address, 10L, Option(ADKey @@ Random.randomBytes())) val transferDirectiveDirectiveWithTokenIdToProto: TransactionProtoMessage.DirectiveProtoMessage = transferDirectiveWithTokenId.toDirectiveProto val transferDirectiveWithTokenIdFromProto: Option[TransferDirective] = @@ -81,7 +121,7 @@ class ModifiersProtoTest extends PropSpec with Matchers with InstanceFactory { transferDirectiveWithTokenId.tokenIdOpt.get.sameElements(transferDirectiveWithTokenIdFromProto.get.tokenIdOpt.get) val transferDirectiveWithoutTokenId: TransferDirective = - TransferDirective(privKey.publicImage.address.address, 10L, Option.empty[ADKey]) + TransferDirective(publicKey.address.address, 10L, Option.empty[ADKey]) val transferDirectiveWithoutTokenIdToProto: TransactionProtoMessage.DirectiveProtoMessage = transferDirectiveWithoutTokenId.toDirectiveProto val transferDirectiveWithoutTokenIdFromProto: Option[TransferDirective] = @@ -93,7 +133,7 @@ class ModifiersProtoTest extends PropSpec with Matchers with InstanceFactory { } property("Transaction should be serialized correctly") { - val boxes: Seq[AssetBox] = (0 to 10).map(_ => genAssetBox(privKey.publicImage.address.address)) + val boxes: Seq[AssetBox] = (0 to 10).map(_ => genAssetBox(publicKey.address.address)) val transaction: Transaction = universalTransactionScratch(privKey, 10, 10, boxes.toIndexedSeq, 10, 1) val transactionToProto: TransactionProtoMessage = transaction.toTransactionProto diff --git a/src/test/scala/encry/modifiers/mempool/TransactionSerializerTest.scala b/src/test/scala/encry/modifiers/mempool/TransactionSerializerTest.scala index 0c7b303ba1..5935c48561 100755 --- a/src/test/scala/encry/modifiers/mempool/TransactionSerializerTest.scala +++ b/src/test/scala/encry/modifiers/mempool/TransactionSerializerTest.scala @@ -1,10 +1,10 @@ package encry.modifiers.mempool -import encry.modifiers.InstanceFactory import org.encryfoundation.common.modifiers.mempool.transaction.TransactionSerializer import org.scalatest.FunSuite +import encry.utils.TestEntityGenerator.{paymentTransactionDynamic, coinbaseTransaction} -class TransactionSerializerTest extends FunSuite with InstanceFactory { +class TransactionSerializerTest extends FunSuite { test("toBytes & parseBytes (Transfer)") { diff --git a/src/test/scala/encry/modifiers/mempool/TransactionSpec.scala b/src/test/scala/encry/modifiers/mempool/TransactionSpec.scala index a962d24daa..9552379ba2 100755 --- a/src/test/scala/encry/modifiers/mempool/TransactionSpec.scala +++ b/src/test/scala/encry/modifiers/mempool/TransactionSpec.scala @@ -1,17 +1,16 @@ package encry.modifiers.mempool -import encry.modifiers.InstanceFactory +import encry.utils.Utils.randomAddress +import encry.utils.TestEntityGenerator._ import encry.utils.TestHelper import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.modifiers.state.box.AssetBox import org.scalatest.{Matchers, PropSpec} -class TransactionSpec extends PropSpec with Matchers with InstanceFactory { +class TransactionSpec extends PropSpec with Matchers { private val txValid = paymentTransactionValid - private val txInvalid = paymentTransactionInvalid - property("semanticValidity of valid tx") { txValid.semanticValidity.isSuccess shouldBe true @@ -22,7 +21,7 @@ class TransactionSpec extends PropSpec with Matchers with InstanceFactory { val tx: Transaction = { val useBoxes: IndexedSeq[AssetBox] = IndexedSeq(TestHelper.genAssetBox(publicKey.address.address)) - TransactionFactory.defaultPaymentTransactionScratch(secret, 100, timestamp, useBoxes ++ useBoxes ++ useBoxes, + TransactionFactory.defaultPaymentTransactionScratch(privKey, 100, timestamp, useBoxes ++ useBoxes ++ useBoxes, randomAddress, TestHelper.Props.txAmount) } @@ -34,7 +33,7 @@ class TransactionSpec extends PropSpec with Matchers with InstanceFactory { val tx: Transaction = { val useBoxes: IndexedSeq[AssetBox] = IndexedSeq(TestHelper.genAssetBox(publicKey.address.address)) - TransactionFactory.defaultPaymentTransactionScratch(secret, -100, timestamp, useBoxes, + TransactionFactory.defaultPaymentTransactionScratch(privKey, -100, timestamp, useBoxes, randomAddress, TestHelper.Props.txAmount) } @@ -46,7 +45,7 @@ class TransactionSpec extends PropSpec with Matchers with InstanceFactory { val tx: Transaction = { val useBoxes: IndexedSeq[AssetBox] = IndexedSeq.empty - TransactionFactory.defaultPaymentTransactionScratch(secret, -100, timestamp, useBoxes, + TransactionFactory.defaultPaymentTransactionScratch(privKey, -100, timestamp, useBoxes, randomAddress, TestHelper.Props.txAmount) } @@ -59,7 +58,7 @@ class TransactionSpec extends PropSpec with Matchers with InstanceFactory { val useBoxes: IndexedSeq[AssetBox] = (0 to Short.MaxValue + 10) .foldLeft(IndexedSeq.empty[AssetBox]) { case (acc, _) => acc :+ TestHelper.genAssetBox(publicKey.address.address) } - TransactionFactory.defaultPaymentTransactionScratch(secret, 100, timestamp, useBoxes, + TransactionFactory.defaultPaymentTransactionScratch(privKey, 100, timestamp, useBoxes, randomAddress, TestHelper.Props.txAmount) } diff --git a/src/test/scala/encry/modifiers/state/AssetBoxSerializerTest.scala b/src/test/scala/encry/modifiers/state/AssetBoxSerializerTest.scala index 094f52c111..3346afae96 100644 --- a/src/test/scala/encry/modifiers/state/AssetBoxSerializerTest.scala +++ b/src/test/scala/encry/modifiers/state/AssetBoxSerializerTest.scala @@ -1,14 +1,17 @@ package encry.modifiers.state -import encry.modifiers.InstanceFactory -import org.encryfoundation.common.modifiers.state.box.AssetBoxSerializer +import encry.utils.Keys +import org.encryfoundation.common.modifiers.state.box.{AssetBox, AssetBoxSerializer, EncryProposition} import org.scalatest.FunSuite -class AssetBoxSerializerTest extends FunSuite with InstanceFactory { +class AssetBoxSerializerTest extends FunSuite with Keys { + + lazy val assetBoxI: AssetBox = AssetBox(EncryProposition.pubKeyLocked(publicKey.pubKeyBytes), 999L, 100000L) + lazy val openAssetBoxI: AssetBox = AssetBox(EncryProposition.open, 999L, 100000L) test("toBytes & parseBytes") { - val bx = AssetBoxI + val bx = assetBoxI val bxSerialized = bx.bytes @@ -21,7 +24,7 @@ class AssetBoxSerializerTest extends FunSuite with InstanceFactory { test("toBytes & parseBytes (OpenProposition)") { - val bx = OpenAssetBoxI + val bx = openAssetBoxI val bxSerialized = bx.bytes diff --git a/src/test/scala/encry/modifiers/state/Keys.scala b/src/test/scala/encry/modifiers/state/Keys.scala deleted file mode 100644 index a61be652e9..0000000000 --- a/src/test/scala/encry/modifiers/state/Keys.scala +++ /dev/null @@ -1,12 +0,0 @@ -package encry.modifiers.state - -import encry.utils.TestHelper -import org.encryfoundation.common.crypto.{PrivateKey25519, PublicKey25519} - -trait Keys { - - val secrets: Seq[PrivateKey25519] = TestHelper.genKeys(TestHelper.Props.keysQty) - - val secret: PrivateKey25519 = secrets.head - val publicKey: PublicKey25519 = secret.publicImage -} diff --git a/src/test/scala/encry/network/BlackListTests.scala b/src/test/scala/encry/network/BlackListTests.scala index b90463a9ca..3b303e319c 100644 --- a/src/test/scala/encry/network/BlackListTests.scala +++ b/src/test/scala/encry/network/BlackListTests.scala @@ -4,7 +4,6 @@ import java.net.{InetAddress, InetSocketAddress} import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestProbe} -import encry.modifiers.InstanceFactory import encry.network.BlackList.BanReason._ import encry.network.PeerConnectionHandler.{ConnectedPeer, Outgoing} import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection @@ -12,12 +11,12 @@ import encry.network.PeersKeeper.BanPeer import encry.settings.TestNetSettings import org.encryfoundation.common.network.BasicMessagesRepo.Handshake import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +import encry.utils.Utils.protocolToBytes import scala.concurrent.duration._ class BlackListTests extends WordSpecLike with Matchers with BeforeAndAfterAll - with InstanceFactory with OneInstancePerTest with TestNetSettings { @@ -26,11 +25,11 @@ class BlackListTests extends WordSpecLike override def afterAll(): Unit = system.terminate() val knowPeersSettings = testNetSettings.copy( - network = settings.network.copy( + network = testNetSettings.network.copy( knownPeers = Seq(new InetSocketAddress("172.16.11.11", 9001)), connectOnlyWithKnownPeers = Some(true) ), - blackList = settings.blackList.copy( + blackList = testNetSettings.blackList.copy( banTime = 2 seconds, cleanupTime = 3 seconds )) diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index d64de7aa11..42546527b7 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -4,7 +4,6 @@ import java.net.InetSocketAddress import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestProbe} -import encry.modifiers.InstanceFactory import encry.network.BlackList.BanReason._ import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming, Outgoing} @@ -12,12 +11,12 @@ import encry.network.PeersKeeper._ import encry.settings.TestNetSettings import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, PeersNetworkMessage} import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +import encry.utils.Utils.protocolToBytes import scala.concurrent.duration._ class ConnectWithNewPeerTests extends WordSpecLike with Matchers with BeforeAndAfterAll - with InstanceFactory with OneInstancePerTest with TestNetSettings { diff --git a/src/test/scala/encry/network/ConnectedPeersCollectionsTests.scala b/src/test/scala/encry/network/ConnectedPeersCollectionsTests.scala index 76abcb9bf6..be0a029d69 100644 --- a/src/test/scala/encry/network/ConnectedPeersCollectionsTests.scala +++ b/src/test/scala/encry/network/ConnectedPeersCollectionsTests.scala @@ -5,17 +5,16 @@ import java.net.InetSocketAddress import akka.actor.ActorSystem import akka.testkit.TestProbe import encry.consensus.HistoryConsensus.{Fork, Older, Unknown, Younger} -import encry.modifiers.InstanceFactory import encry.network.ConnectedPeersCollection.PeerInfo import encry.network.PeerConnectionHandler.{ConnectedPeer, Outgoing} import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ import encry.settings.TestNetSettings import org.encryfoundation.common.network.BasicMessagesRepo.Handshake import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} +import encry.utils.Utils.protocolToBytes class ConnectedPeersCollectionsTests extends WordSpecLike with Matchers - with InstanceFactory with OneInstancePerTest with TestNetSettings { diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala index 410373e2b3..c2c6e3265f 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala @@ -2,12 +2,11 @@ package encry.network.DeliveryManagerTests import java.net.InetSocketAddress -import encry.network.DeliveryManagerTests.DMUtils.{createPeer, generateBlocks, initialiseDeliveryManager} +import encry.network.DeliveryManagerTests.DeliveryManagerUtils.{createPeer, generateBlocks, initialiseDeliveryManager} import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestKit} import encry.consensus.HistoryConsensus import encry.consensus.HistoryConsensus.{Equal, Older, Younger} -import encry.modifiers.InstanceFactory import encry.network.DeliveryManager import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.RequestFromLocal @@ -20,11 +19,11 @@ import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoS import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +import encry.utils.HistoryGenerator.dummyHistory class DeliveryManagerPriorityTests extends WordSpecLike with BeforeAndAfterAll with Matchers - with InstanceFactory with OneInstancePerTest with TestNetSettings { @@ -45,7 +44,7 @@ class DeliveryManagerPriorityTests extends WordSpecLike val (_: InetSocketAddress, cp7: ConnectedPeer) = createPeer(9007, "172.16.13.16", testNetSettings) val (_: InetSocketAddress, cp8: ConnectedPeer) = createPeer(9008, "172.16.13.17", testNetSettings) val (_: InetSocketAddress, cp9: ConnectedPeer) = createPeer(9009, "172.16.13.18", testNetSettings) - val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 + val blocks: List[Block] = generateBlocks(10, dummyHistory(testNetSettings))._2 val headersIds: List[ModifierId] = blocks.map(_.header.id) (deliveryManager, cp1, cp2, cp3, cp4, cp5, cp6,cp7, cp8, cp9, blocks, headersIds) } diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala index 97acbe4976..a3f4906de2 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala @@ -5,9 +5,8 @@ import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestProbe} import encry.consensus.HistoryConsensus import encry.consensus.HistoryConsensus.Older -import encry.modifiers.InstanceFactory import encry.network.DeliveryManager -import encry.network.DeliveryManagerTests.DMUtils._ +import encry.network.DeliveryManagerTests.DeliveryManagerUtils._ import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} @@ -23,11 +22,13 @@ import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} import scala.concurrent.duration._ import scala.collection.mutable.WrappedArray +import encry.utils.HistoryGenerator.dummyHistory +import encry.utils.TestEntityGenerator._ +import encry.utils.Utils.protocolToBytes class DeliveryManagerReRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll with Matchers - with InstanceFactory with OneInstancePerTest with TestNetSettings { @@ -42,7 +43,7 @@ class DeliveryManagerReRequestModifiesSpec extends WordSpecLike val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) - val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 + val blocks: List[Block] = generateBlocks(10, dummyHistory(testNetSettings))._2 val headersIds: List[ModifierId] = blocks.map(_.header.id) val headersAsKey = headersIds.map(toKey) (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey, history) diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala index 7bdcf9eafb..0a05a6af37 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala @@ -5,7 +5,6 @@ import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestKit, TestProbe} import encry.consensus.HistoryConsensus import encry.consensus.HistoryConsensus.{Fork, Older, Younger} -import encry.modifiers.InstanceFactory import encry.network.DeliveryManager import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.RequestFromLocal @@ -13,7 +12,7 @@ import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} import encry.settings.TestNetSettings import encry.view.actors.NodeViewHolder.DownloadRequest import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -import encry.network.DeliveryManagerTests.DMUtils._ +import encry.network.DeliveryManagerTests.DeliveryManagerUtils._ import encry.network.PeersKeeper.UpdatedPeersCollection import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ @@ -23,10 +22,13 @@ import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, Modifier import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.mutable.WrappedArray +import encry.utils.HistoryGenerator.dummyHistory +import encry.utils.Utils.protocolToBytes +import encry.utils.TestEntityGenerator._ -class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll +class DeliveryManagerRequestModifiesSpec extends WordSpecLike + with BeforeAndAfterAll with Matchers - with InstanceFactory with OneInstancePerTest with TestNetSettings { @@ -40,7 +42,7 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) - val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 + val blocks: List[Block] = generateBlocks(10, dummyHistory(testNetSettings))._2 val headersIds: List[ModifierId] = blocks.map(_.header.id) val headersAsKey = headersIds.map(toKey) (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerUtils.scala similarity index 92% rename from src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala rename to src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerUtils.scala index 288b3b276b..990955a376 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerUtils.scala @@ -4,7 +4,6 @@ import java.net.InetSocketAddress import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestProbe} import encry.local.miner.Miner.{DisableMining, StartMining} -import encry.modifiers.InstanceFactory import encry.network.DeliveryManager import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages.UpdatedHistory @@ -14,16 +13,19 @@ import encry.view.history.History import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.network.BasicMessagesRepo.Handshake import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import encry.utils.HistoryGenerator.dummyHistory +import encry.utils.TestEntityGenerator._ import scala.collection.mutable import scala.collection.mutable.WrappedArray +import encry.utils.Utils.protocolToBytes -object DMUtils extends InstanceFactory { +object DeliveryManagerUtils { def initialiseDeliveryManager(isBlockChainSynced: Boolean, isMining: Boolean, settings: EncryAppSettings) (implicit actorSystem: ActorSystem): (TestActorRef[DeliveryManager], History) = { - val history: History = generateDummyHistory(settings) + val history: History = dummyHistory(settings) val deliveryManager: TestActorRef[DeliveryManager] = TestActorRef[DeliveryManager](DeliveryManager .props(None, TestProbe().ref, TestProbe().ref, TestProbe().ref, TestProbe().ref, TestProbe().ref, settings)) diff --git a/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala b/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala index a3e9f5677b..ef87aa8ffa 100644 --- a/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala +++ b/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala @@ -4,7 +4,6 @@ import java.net.InetSocketAddress import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestProbe} -import encry.modifiers.InstanceFactory import encry.network.BlackList.BanReason._ import encry.network.DownloadedModifiersValidator.{InvalidModifier, ModifiersForValidating} import encry.network.NodeViewSynchronizer.ReceivableMessages.UpdatedHistory @@ -20,11 +19,13 @@ import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} import scorex.crypto.hash.Digest32 import scorex.utils.Random +import encry.utils.HistoryGenerator.dummyHistory +import encry.utils.Utils.protocolToBytes +import encry.utils.TestEntityGenerator._ class DownloadedModifiersValidatorTests extends WordSpecLike with Matchers with BeforeAndAfterAll - with InstanceFactory with OneInstancePerTest with TestNetSettings { @@ -43,7 +44,7 @@ class DownloadedModifiersValidatorTests extends WordSpecLike val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator](DownloadedModifiersValidator.props( testNetSettings.constants.ModifierIdSize, nodeViewHolder.ref, peersKeeper.ref, nodeViewSync.ref, mempool.ref, None) ) - val history: History = generateDummyHistory(testNetSettings) + val history: History = dummyHistory(testNetSettings) val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) val peerHandler: TestProbe = TestProbe() @@ -111,7 +112,7 @@ class DownloadedModifiersValidatorTests extends WordSpecLike val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator](DownloadedModifiersValidator.props( testNetSettings.constants.ModifierIdSize, nodeViewHolder.ref, peersKeeper.ref, nodeViewSync.ref, mempool.ref, None) ) - val history: History = generateDummyHistory(testNetSettings) + val history: History = dummyHistory(testNetSettings) val historyWith10Blocks = (0 until 10).foldLeft(history, Seq.empty[Block]) { case ((prevHistory, blocks), _) => diff --git a/src/test/scala/encry/network/NetworkMessagesProtoTest/BasicNetworkMessagesProtoTest.scala b/src/test/scala/encry/network/NetworkMessagesProtoTest/BasicNetworkMessagesProtoTest.scala index 7746922350..49a6d02ee6 100644 --- a/src/test/scala/encry/network/NetworkMessagesProtoTest/BasicNetworkMessagesProtoTest.scala +++ b/src/test/scala/encry/network/NetworkMessagesProtoTest/BasicNetworkMessagesProtoTest.scala @@ -4,21 +4,24 @@ import java.net.InetSocketAddress import NetworkMessagesProto.GeneralizedNetworkProtoMessage import NetworkMessagesProto.GeneralizedNetworkProtoMessage.InnerMessage -import encry.EncryApp -import encry.modifiers.InstanceFactory -import encry.settings.{EncryAppSettings, Settings} +import encry.settings. Settings import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo._ import org.encryfoundation.common.network.{BasicMessagesRepo, SyncInfo} import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import org.scalatest.{Matchers, PropSpec} +import encry.utils.HistoryGenerator.dummyHistory +import encry.utils.Utils.protocolToBytes +import encry.utils.TestEntityGenerator._ import scala.util.Try -class BasicNetworkMessagesProtoTest extends PropSpec with Matchers with InstanceFactory with Settings { +class BasicNetworkMessagesProtoTest extends PropSpec + with Matchers + with Settings { - val testedBlocks: Vector[Block] = (0 until 10).foldLeft(generateDummyHistory(settings), Vector.empty[Block]) { + val testedBlocks: Vector[Block] = (0 until 10).foldLeft(dummyHistory(settings), Vector.empty[Block]) { case ((prevHistory, blocks), _) => val block: Block = generateNextBlock(prevHistory) prevHistory.append(block.header) diff --git a/src/test/scala/encry/utils/ChainGenerator.scala b/src/test/scala/encry/utils/ChainGenerator.scala index 18dde6d537..61a2e4f3a7 100644 --- a/src/test/scala/encry/utils/ChainGenerator.scala +++ b/src/test/scala/encry/utils/ChainGenerator.scala @@ -11,22 +11,18 @@ import encry.storage.iodb.versionalIODB.IODBWrapper import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} import encry.view.state.{BoxHolder, UtxoState} import io.iohk.iodb.LSMStore -import org.encryfoundation.common.crypto.{PrivateKey25519, PublicKey25519, Signature25519} import org.encryfoundation.common.crypto.equihash.EquihashSolution +import org.encryfoundation.common.crypto.{PrivateKey25519, PublicKey25519} import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} -import org.encryfoundation.common.modifiers.mempool.directive.TransferDirective import org.encryfoundation.common.modifiers.mempool.transaction.EncryAddress.Address -import org.encryfoundation.common.modifiers.mempool.transaction.{Input, Pay2PubKeyAddress, Proof, PubKeyLockedContract, Transaction, UnsignedTransaction} -import org.encryfoundation.common.modifiers.state.box.Box.Amount -import org.encryfoundation.common.modifiers.state.box.{AssetBox, MonetaryBox} -import org.encryfoundation.common.utils.TaggedTypes.{ADKey, Difficulty, Height} -import org.encryfoundation.prismlang.core.wrapped.BoxedValue +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.modifiers.state.box.AssetBox +import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height} import org.iq80.leveldb.Options import scorex.crypto.hash.Digest32 -import scorex.crypto.signatures.PublicKey +import encry.utils.Utils.randomAddress import scala.collection.Seq -import scorex.utils.{Random => ScorexRandom} import scala.util.Random object ChainGenerator { @@ -34,8 +30,6 @@ object ChainGenerator { def genChain(privKey: PrivateKey25519, dir: File, settings: EncryAppSettings, blockQty: Int, transPerBlock: Int = 100, genInvalidBlockFrom: Option[Int] = None): (UtxoState, UtxoState, List[Block]) = { - def randomAddress: Address = Pay2PubKeyAddress(PublicKey @@ ScorexRandom.randomBytes()).address - assert(blockQty >= 2, "chain at least 2 blocks") val genesisBlock: Block = genGenesisBlock(privKey.publicImage, settings.constants.InitialEmissionAmount, @@ -80,9 +74,10 @@ object ChainGenerator { } def genGenesisBlock(pubKey: PublicKey25519, initialEmissionAmount: Int, initialDifficulty: Difficulty, genesisHeight: Height): Block = { + val timestamp = System.currentTimeMillis() val coinbaseTrans = TransactionFactory.coinbaseTransactionScratch( pubKey, - System.currentTimeMillis(), + timestamp, initialEmissionAmount, amount = 0L, height = genesisHeight @@ -90,12 +85,11 @@ object ChainGenerator { val txs: Seq[Transaction] = Seq(coinbaseTrans) val txsRoot: Digest32 = Payload.rootHash(txs.map(_.id)) - val header = - Header( + val header = Header( 1.toByte, Header.GenesisParentId, txsRoot, - System.currentTimeMillis(), + timestamp, genesisHeight, Random.nextLong(), initialDifficulty, diff --git a/src/test/scala/encry/utils/EncryGenerator.scala b/src/test/scala/encry/utils/EncryGenerator.scala deleted file mode 100644 index 19c6a6aa31..0000000000 --- a/src/test/scala/encry/utils/EncryGenerator.scala +++ /dev/null @@ -1,279 +0,0 @@ -package encry.utils - -import encry.modifiers.mempool.TransactionFactory -import encry.modifiers.mempool.TransactionFactory.{dataTransactionScratch, paymentTransactionWithMultipleOutputs} -import encry.settings.Settings -import encry.utils.TestHelper.Props -import org.encryfoundation.common.crypto.equihash.EquihashSolution -import org.encryfoundation.common.crypto.{PrivateKey25519, PublicKey25519, Signature25519} -import org.encryfoundation.common.modifiers.history.Header -import org.encryfoundation.common.modifiers.mempool.directive._ -import org.encryfoundation.common.modifiers.mempool.transaction.EncryAddress.Address -import org.encryfoundation.common.modifiers.mempool.transaction._ -import org.encryfoundation.common.modifiers.state.box.Box.Amount -import org.encryfoundation.common.modifiers.state.box._ -import org.encryfoundation.common.utils.TaggedTypes.{ADKey, ModifierId} -import org.encryfoundation.prismlang.core.wrapped.BoxedValue -import scorex.crypto.hash.{Blake2b256, Digest32} -import scorex.crypto.signatures.{Curve25519, PrivateKey, PublicKey} -import scorex.utils.Random - -import scala.util.{Random => ScRand} - -trait EncryGenerator extends Settings { - - val mnemonicKey: String = "index another island accuse valid aerobic little absurd bunker keep insect scissors" - val privKey: PrivateKey25519 = createPrivKey(Some(mnemonicKey)) - - - def createPrivKey(seed: Option[String]): PrivateKey25519 = { - val (privateKey: PrivateKey, publicKey: PublicKey) = Curve25519.createKeyPair( - Blake2b256.hash( - seed.map { - Mnemonic.seedFromMnemonic(_) - } - .getOrElse { - val phrase: String = Mnemonic.entropyToMnemonicCode(scorex.utils.Random.randomBytes(16)) - Mnemonic.seedFromMnemonic(phrase) - }) - ) - PrivateKey25519(privateKey, publicKey) - } - - def protocolToBytes(protocol: String): Array[Byte] = protocol.split("\\.").map(elem => elem.toByte) - - def timestamp: Long = System.currentTimeMillis() - - def randomAddress: Address = Pay2PubKeyAddress(PublicKey @@ Random.randomBytes()).address - - def genAssetBox(address: Address, amount: Amount = 100000L, tokenIdOpt: Option[ADKey] = None): AssetBox = - AssetBox(EncryProposition.addressLocked(address), ScRand.nextLong(), amount, tokenIdOpt) - - def generateDataBox(address: Address, nonce: Long, data: Array[Byte]): DataBox = - DataBox(EncryProposition.addressLocked(address), nonce, data) - - def generateTokenIssuingBox(address: Address, amount: Amount = 100000L, tokenIdOpt: ADKey): TokenIssuingBox = - TokenIssuingBox(EncryProposition.addressLocked(address), ScRand.nextLong(), amount, tokenIdOpt) - - def genTxOutputs(boxes: Traversable[EncryBaseBox]): IndexedSeq[ADKey] = - boxes.foldLeft(IndexedSeq[ADKey]()) { case (s, box) => - s :+ box.id - } - - def genValidAssetBoxes(secret: PrivateKey25519, amount: Amount, qty: Int): Seq[AssetBox] = - (0 to qty).foldLeft(IndexedSeq[AssetBox]()) { case (bxs, _) => - bxs :+ AssetBox(EncryProposition.pubKeyLocked(secret.publicKeyBytes), ScRand.nextLong(), amount) - } - - def genPrivKeys(qty: Int): Seq[PrivateKey25519] = (0 until qty).map { _ => - val keys: (PrivateKey, PublicKey) = Curve25519.createKeyPair(Random.randomBytes()) - PrivateKey25519(keys._1, keys._2) - } - - def genValidPaymentTxs(qty: Int): Seq[Transaction] = { - val keys: Seq[PrivateKey25519] = genPrivKeys(qty) - val now = System.currentTimeMillis() - - keys.map { k => - val useBoxes: IndexedSeq[AssetBox] = IndexedSeq(genAssetBox(k.publicImage.address.address)) - TransactionFactory.defaultPaymentTransactionScratch(k, Props.txFee, - now + scala.util.Random.nextInt(5000), useBoxes, randomAddress, Props.boxValue) - } - } - - def genValidPaymentTxsToAddr(qty: Int, address: Address): Seq[Transaction] = { - val keys: Seq[PrivateKey25519] = genPrivKeys(qty) - keys.map { k => - val useBoxes = IndexedSeq(genAssetBox(address)) - TransactionFactory.defaultPaymentTransactionScratch(k, Props.txFee, - scala.util.Random.nextLong(), useBoxes, address, Props.boxValue) - } - } - - def genValidPaymentTxToAddrWithSpentBoxes(boxes: IndexedSeq[AssetBox], address: Address): Transaction = { - val key: PrivateKey25519 = genPrivKeys(1).head - TransactionFactory.defaultPaymentTransactionScratch(key, Props.txFee, - scala.util.Random.nextLong(), boxes, address, Props.boxValue) - } - - def genValidPaymentTxsToAddrWithDiffTokens(qty: Int, address: Address): Seq[Transaction] = { - val keys: Seq[PrivateKey25519] = genPrivKeys(qty) - val tokens: Seq[ADKey] = (0 until qty).foldLeft(Seq[ADKey]()) { - case (seq, _) => seq :+ (ADKey @@ Random.randomBytes()) - } - val pksZipTokens: Seq[(PrivateKey25519, ADKey)] = keys.zip(tokens) - val timestamp: Amount = System.currentTimeMillis() - pksZipTokens.map { k => - val useBoxes = IndexedSeq(genAssetBox(address, tokenIdOpt = Some(k._2))) - TransactionFactory.defaultPaymentTransactionScratch(k._1, Props.txFee, - timestamp, useBoxes, address, Props.boxValue, tokenIdOpt = Some(k._2)) - } - } - - def genSelfSpendingTxs(qty: Int): Seq[Transaction] = { - val keys: Seq[PrivateKey25519] = genPrivKeys(qty) - val timestamp: Amount = System.currentTimeMillis() - keys.foldLeft(Seq[Transaction]()) { (seq, key) => - val useBoxes: IndexedSeq[MonetaryBox] = if (seq.isEmpty) IndexedSeq(genAssetBox(key.publicImage.address.address)) - else seq.last.newBoxes.map(_.asInstanceOf[MonetaryBox]).toIndexedSeq - seq :+ TransactionFactory.defaultPaymentTransactionScratch(key, Props.txFee, - timestamp, useBoxes, randomAddress, Props.boxValue) - } - } - - def genInvalidPaymentTxs(qty: Int): Seq[Transaction] = { - val timestamp: Amount = System.currentTimeMillis() - genPrivKeys(qty).map { key => - val useBoxes: IndexedSeq[AssetBox] = - IndexedSeq(genAssetBox(PublicKey25519(PublicKey @@ Random.randomBytes(32)).address.address)) - TransactionFactory.defaultPaymentTransactionScratch(key, -100, timestamp, useBoxes, randomAddress, Props.boxValue) - } - } - - def genHeader: Header = { - val random = new scala.util.Random - Header( - 1.toByte, - ModifierId @@ Random.randomBytes(), - Digest32 @@ Random.randomBytes(), - Math.abs(random.nextLong()), - Math.abs(random.nextInt(10000)), - random.nextLong(), - settings.constants.InitialDifficulty, - EquihashSolution(Seq(1, 3)) - ) - } - - def genHeaderAtHeight(height: Int = 0, transactionsRoot: Digest32 = Digest32 @@ Random.randomBytes()): Header = { - val random = new scala.util.Random - Header( - 1.toByte, - ModifierId @@ Random.randomBytes(), - transactionsRoot, - Math.abs(random.nextLong()), - height, - random.nextLong(), - settings.constants.InitialDifficulty, - EquihashSolution(Seq(1, 3)) - ) - } - - def generatePaymentTransactions(boxes: IndexedSeq[AssetBox], - numberOfInputs: Int, - numberOfOutputs: Int): Vector[Transaction] = - (0 until boxes.size / numberOfInputs).foldLeft(boxes, Vector.empty[Transaction]) { - case ((boxesLocal, transactions), _) => - val tx: Transaction = paymentTransactionWithMultipleOutputs( - privKey, - fee = 111, - timestamp = 11L, - useBoxes = boxesLocal.take(numberOfInputs), - recipient = randomAddress, - amount = 10000, - numOfOutputs = numberOfOutputs - ) - (boxesLocal.drop(numberOfInputs), transactions :+ tx) - }._2 - - def generateDataTransactions(boxes: IndexedSeq[AssetBox], - numberOfInputs: Int, - numberOfOutputs: Int, - bytesQty: Int): Vector[Transaction] = - (0 until boxes.size / numberOfInputs).foldLeft(boxes, Vector.empty[Transaction]) { - case ((boxesLocal, transactions), _) => - val tx: Transaction = dataTransactionScratch( - privKey, - fee = 111, - timestamp = 11L, - useOutputs = boxesLocal.take(numberOfInputs), - data = Random.randomBytes(bytesQty), - amount = 200L, - numOfOutputs = numberOfOutputs - ) - (boxesLocal.drop(numberOfInputs), tx +: transactions) - }._2 - - def generateAssetTransactions(boxes: IndexedSeq[AssetBox], - numberOfInputs: Int, - numberOfOutputs: Int): Vector[Transaction] = - (0 until boxes.size / numberOfInputs).foldLeft(boxes, Vector.empty[Transaction]) { - case ((boxesLocal, transactions), _) => - val tx: Transaction = assetIssuingTransactionScratch( - privKey, - fee = 111, - timestamp = 11L, - useOutputs = boxesLocal.take(numberOfInputs), - amount = 200L, - numOfOutputs = numberOfOutputs - ) - (boxesLocal.drop(numberOfInputs), tx +: transactions) - }._2 - - def assetIssuingTransactionScratch(privKey: PrivateKey25519, - fee: Long, - timestamp: Long, - useOutputs: IndexedSeq[MonetaryBox], - amount: Long, - numOfOutputs: Int = 5): Transaction = { - val directives: IndexedSeq[AssetIssuingDirective] = - (0 until numOfOutputs).foldLeft(IndexedSeq.empty[AssetIssuingDirective]) { case (directivesAll, _) => - directivesAll :+ AssetIssuingDirective(PubKeyLockedContract(privKey.publicImage.pubKeyBytes).contract.hash, amount) - } - - val pubKey: PublicKey25519 = privKey.publicImage - - val uInputs: IndexedSeq[Input] = useOutputs - .map(bx => Input.unsigned(bx.id, Right(PubKeyLockedContract(pubKey.pubKeyBytes)))) - .toIndexedSeq - - val change: Amount = useOutputs.map(_.amount).sum - (amount + fee) - - val newDirectives: IndexedSeq[Directive] = - if (change > 0) TransferDirective(pubKey.address.address, amount, None) +: (0 until numOfOutputs).map(_ => - TransferDirective(pubKey.address.address, change / numOfOutputs, None)) ++: directives - else directives - - val uTransaction: UnsignedTransaction = UnsignedTransaction(fee, timestamp, uInputs, newDirectives) - - val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) - - uTransaction.toSigned(IndexedSeq.empty, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) - } - - def universalTransactionScratch(privKey: PrivateKey25519, - fee: Long, - timestamp: Long, - useOutputs: IndexedSeq[MonetaryBox], - amount: Long, - numOfOutputs: Int = 5): Transaction = { - val directives: IndexedSeq[Directive] = IndexedSeq( - AssetIssuingDirective(PubKeyLockedContract(privKey.publicImage.pubKeyBytes).contract.hash, amount), - DataDirective(PubKeyLockedContract(privKey.publicImage.pubKeyBytes).contract.hash, Random.randomBytes()), - ScriptedAssetDirective(PubKeyLockedContract(privKey.publicImage.pubKeyBytes).contract.hash, 10L, - Option(ADKey @@ Random.randomBytes())), - ScriptedAssetDirective(PubKeyLockedContract(privKey.publicImage.pubKeyBytes).contract.hash, 10L, - Option.empty[ADKey]), - TransferDirective(privKey.publicImage.address.address, 10L, Option(ADKey @@ Random.randomBytes())), - TransferDirective(privKey.publicImage.address.address, 10L, Option.empty[ADKey]) - ) - - val pubKey: PublicKey25519 = privKey.publicImage - - val uInputs: IndexedSeq[Input] = useOutputs - .map(bx => Input.unsigned(bx.id, Right(PubKeyLockedContract(pubKey.pubKeyBytes)))) - .toIndexedSeq - - val change: Amount = useOutputs.map(_.amount).sum - (amount + fee) - - val newDirectives: IndexedSeq[Directive] = - if (change > 0) TransferDirective(pubKey.address.address, amount, None) +: (0 until numOfOutputs).map(_ => - TransferDirective(pubKey.address.address, change / numOfOutputs, None)) ++: directives - else directives - - val uTransaction: UnsignedTransaction = UnsignedTransaction(fee, timestamp, uInputs, newDirectives) - - val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) - - uTransaction.toSigned(IndexedSeq.empty, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) - } -} diff --git a/src/test/scala/encry/utils/HistoryGenerator.scala b/src/test/scala/encry/utils/HistoryGenerator.scala new file mode 100644 index 0000000000..49e01a4b5f --- /dev/null +++ b/src/test/scala/encry/utils/HistoryGenerator.scala @@ -0,0 +1,47 @@ +package encry.utils + +import encry.consensus.EquihashPowScheme +import encry.crypto.equihash.EquihashValidationErrors +import encry.settings.EncryAppSettings +import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} +import encry.view.history.History +import encry.view.history.storage.HistoryStorage +import io.iohk.iodb.LSMStore +import org.encryfoundation.common.modifiers.history.Header +import org.encryfoundation.common.utils.TaggedTypes.Height +import org.iq80.leveldb.Options + +object HistoryGenerator { + + def dummyHistory(settings: EncryAppSettings, withoutPow: Boolean = false): History = { + + val indexStore: LSMStore = new LSMStore(FileHelper.getRandomTempDir, keepVersions = 0) + val objectsStore: LSMStore = new LSMStore(FileHelper.getRandomTempDir, keepVersions = 0) + val levelDBInit = LevelDbFactory.factory.open(FileHelper.getRandomTempDir, new Options) + val vldbInit = VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settings.levelDB)) + val storage: HistoryStorage = new HistoryStorage(vldbInit) + + val ntp: NetworkTimeProvider = new NetworkTimeProvider(settings.ntp) + + class EquihashPowSchemeWithoutValidateSolution(n: Char, k: Char, version: Byte, preGenesisHeight: Height, maxTarget: BigInt) + extends EquihashPowScheme(n: Char, k: Char, version: Byte, preGenesisHeight: Height, maxTarget: BigInt) { + override def verify(header: Header): Either[EquihashValidationErrors, Boolean] = Right(true) + } + + if (withoutPow) { + new History { + override val historyStorage: HistoryStorage = storage + override val timeProvider: NetworkTimeProvider = ntp + override val powScheme: EquihashPowScheme = + new EquihashPowSchemeWithoutValidateSolution(settings.constants.n, settings.constants.k, + settings.constants.Version, settings.constants.PreGenesisHeight, settings.constants.MaxTarget) + } + } else + new History { + override val historyStorage: HistoryStorage = storage + override val timeProvider: NetworkTimeProvider = ntp + } + + } + +} diff --git a/src/test/scala/encry/utils/Keys.scala b/src/test/scala/encry/utils/Keys.scala new file mode 100644 index 0000000000..2d62223af9 --- /dev/null +++ b/src/test/scala/encry/utils/Keys.scala @@ -0,0 +1,9 @@ +package encry.utils + +import org.encryfoundation.common.crypto.{PrivateKey25519, PublicKey25519} + +trait Keys { + val mnemonicKey: String = "index another island accuse valid aerobic little absurd bunker keep insect scissors" + val privKey: PrivateKey25519 = Utils.createPrivKey(Some(mnemonicKey)) + val publicKey: PublicKey25519 = privKey.publicImage +} diff --git a/src/test/scala/encry/utils/TestEntityGenerator.scala b/src/test/scala/encry/utils/TestEntityGenerator.scala new file mode 100644 index 0000000000..24b7690af6 --- /dev/null +++ b/src/test/scala/encry/utils/TestEntityGenerator.scala @@ -0,0 +1,292 @@ +package encry.utils + + +import encry.modifiers.mempool.TransactionFactory +import encry.modifiers.mempool.TransactionFactory._ +import encry.settings.{EncryAppSettings, Settings} +import encry.utils.HistoryGenerator.dummyHistory +import encry.utils.TestHelper.Props +import encry.view.history.History +import org.encryfoundation.common.crypto.equihash.EquihashSolution +import org.encryfoundation.common.crypto.{PrivateKey25519, PublicKey25519} +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.mempool.transaction.EncryAddress.Address +import org.encryfoundation.common.modifiers.mempool.transaction._ +import org.encryfoundation.common.modifiers.state.box.Box.Amount +import org.encryfoundation.common.modifiers.state.box._ +import org.encryfoundation.common.utils.TaggedTypes.{ADKey, Difficulty, Height, ModifierId} +import scorex.crypto.hash.Digest32 +import scorex.crypto.signatures.{Curve25519, PrivateKey, PublicKey} +import encry.utils.Utils.randomAddress + +import scala.util.Random +import scorex.utils.{Random => ScorexRandom} + +object TestEntityGenerator extends Keys with Settings { + + def timestamp: Long = System.currentTimeMillis() + + def genAssetBox(address: Address, amount: Amount = 100000L, tokenIdOpt: Option[ADKey] = None): AssetBox = + AssetBox(EncryProposition.addressLocked(address), Random.nextLong(), amount, tokenIdOpt) + + def generateDataBox(address: Address, nonce: Long, data: Array[Byte]): DataBox = + DataBox(EncryProposition.addressLocked(address), nonce, data) + + def generateTokenIssuingBox(address: Address, amount: Amount = 100000L, tokenIdOpt: ADKey): TokenIssuingBox = + TokenIssuingBox(EncryProposition.addressLocked(address), Random.nextLong(), amount, tokenIdOpt) + + def genTxOutputs(boxes: Traversable[EncryBaseBox]): IndexedSeq[ADKey] = + boxes.foldLeft(IndexedSeq[ADKey]()) { case (s, box) => + s :+ box.id + } + + def genValidAssetBoxes(secret: PrivateKey25519, amount: Amount, qty: Int): Seq[AssetBox] = + (0 to qty).foldLeft(IndexedSeq[AssetBox]()) { case (bxs, _) => + bxs :+ AssetBox(EncryProposition.pubKeyLocked(secret.publicKeyBytes), Random.nextLong(), amount) + } + + def genPrivKeys(qty: Int): Seq[PrivateKey25519] = (0 until qty).map { _ => + val keys: (PrivateKey, PublicKey) = Curve25519.createKeyPair(ScorexRandom.randomBytes()) + PrivateKey25519(keys._1, keys._2) + } + + def genValidPaymentTxs(qty: Int): Seq[Transaction] = { + val keys: Seq[PrivateKey25519] = genPrivKeys(qty) + val now = System.currentTimeMillis() + + keys.map { k => + val useBoxes: IndexedSeq[AssetBox] = IndexedSeq(genAssetBox(k.publicImage.address.address)) + defaultPaymentTransactionScratch(k, Props.txFee, + now + Random.nextInt(5000), useBoxes, randomAddress, Props.boxValue) + } + } + + def genValidPaymentTxsToAddr(qty: Int, address: Address): Seq[Transaction] = { + val keys: Seq[PrivateKey25519] = genPrivKeys(qty) + keys.map { k => + val useBoxes = IndexedSeq(genAssetBox(address)) + defaultPaymentTransactionScratch(k, Props.txFee, + Random.nextLong(), useBoxes, address, Props.boxValue) + } + } + + def genValidPaymentTxToAddrWithSpentBoxes(boxes: IndexedSeq[AssetBox], address: Address): Transaction = { + val key: PrivateKey25519 = genPrivKeys(1).head + defaultPaymentTransactionScratch(key, Props.txFee, + Random.nextLong(), boxes, address, Props.boxValue) + } + + def genValidPaymentTxsToAddrWithDiffTokens(qty: Int, address: Address): Seq[Transaction] = { + val keys: Seq[PrivateKey25519] = genPrivKeys(qty) + val tokens: Seq[ADKey] = (0 until qty).foldLeft(Seq[ADKey]()) { + case (seq, _) => seq :+ (ADKey @@ ScorexRandom.randomBytes()) + } + val pksZipTokens: Seq[(PrivateKey25519, ADKey)] = keys.zip(tokens) + val timestamp: Amount = System.currentTimeMillis() + pksZipTokens.map { k => + val useBoxes = IndexedSeq(genAssetBox(address, tokenIdOpt = Some(k._2))) + defaultPaymentTransactionScratch(k._1, Props.txFee, + timestamp, useBoxes, address, Props.boxValue, tokenIdOpt = Some(k._2)) + } + } + + def genSelfSpendingTxs(qty: Int): Seq[Transaction] = { + val keys: Seq[PrivateKey25519] = genPrivKeys(qty) + val timestamp: Amount = System.currentTimeMillis() + keys.foldLeft(Seq[Transaction]()) { (seq, key) => + val useBoxes: IndexedSeq[MonetaryBox] = if (seq.isEmpty) IndexedSeq(genAssetBox(key.publicImage.address.address)) + else seq.last.newBoxes.map(_.asInstanceOf[MonetaryBox]).toIndexedSeq + seq :+ defaultPaymentTransactionScratch(key, Props.txFee, + timestamp, useBoxes, randomAddress, Props.boxValue) + } + } + + def genInvalidPaymentTxs(qty: Int): Seq[Transaction] = { + val timestamp: Amount = System.currentTimeMillis() + genPrivKeys(qty).map { key => + val useBoxes: IndexedSeq[AssetBox] = + IndexedSeq(genAssetBox(PublicKey25519(PublicKey @@ ScorexRandom.randomBytes(32)).address.address)) + defaultPaymentTransactionScratch(key, -100, timestamp, useBoxes, randomAddress, Props.boxValue) + } + } + + def genHeader: Header = { + val random = new scala.util.Random + Header( + 1.toByte, + ModifierId @@ ScorexRandom.randomBytes(), + Digest32 @@ ScorexRandom.randomBytes(), + Math.abs(Random.nextLong()), + Math.abs(Random.nextInt(10000)), + Random.nextLong(), + settings.constants.InitialDifficulty, + EquihashSolution(Seq(1, 3)) + ) + } + + def genHeaderAtHeight(height: Int = 0, transactionsRoot: Digest32 = Digest32 @@ ScorexRandom.randomBytes()): Header = { + val random = new scala.util.Random + Header( + 1.toByte, + ModifierId @@ ScorexRandom.randomBytes(), + transactionsRoot, + Math.abs(Random.nextLong()), + height, + Random.nextLong(), + settings.constants.InitialDifficulty, + EquihashSolution(Seq(1, 3)) + ) + } + + def generatePaymentTransactions(boxes: IndexedSeq[AssetBox], + numberOfInputs: Int, + numberOfOutputs: Int): Vector[Transaction] = + (0 until boxes.size / numberOfInputs).foldLeft(boxes, Vector.empty[Transaction]) { + case ((boxesLocal, transactions), _) => + val tx: Transaction = paymentTransactionWithMultipleOutputs( + privKey, + fee = 111, + timestamp = 11L, + useBoxes = boxesLocal.take(numberOfInputs), + recipient = randomAddress, + amount = 10000, + numOfOutputs = numberOfOutputs + ) + (boxesLocal.drop(numberOfInputs), transactions :+ tx) + }._2 + + def generateDataTransactions(boxes: IndexedSeq[AssetBox], + numberOfInputs: Int, + numberOfOutputs: Int, + bytesQty: Int): Vector[Transaction] = + (0 until boxes.size / numberOfInputs).foldLeft(boxes, Vector.empty[Transaction]) { + case ((boxesLocal, transactions), _) => + val tx: Transaction = dataTransactionScratch( + privKey, + fee = 111, + timestamp = 11L, + useOutputs = boxesLocal.take(numberOfInputs), + data = ScorexRandom.randomBytes(bytesQty), + amount = 200L, + numOfOutputs = numberOfOutputs + ) + (boxesLocal.drop(numberOfInputs), tx +: transactions) + }._2 + + def generateAssetTransactions(boxes: IndexedSeq[AssetBox], + numberOfInputs: Int, + numberOfOutputs: Int): Vector[Transaction] = + (0 until boxes.size / numberOfInputs).foldLeft(boxes, Vector.empty[Transaction]) { + case ((boxesLocal, transactions), _) => + val tx: Transaction = assetIssuingTransactionScratch( + privKey, + fee = 111, + timestamp = 11L, + useOutputs = boxesLocal.take(numberOfInputs), + amount = 200L, + numOfOutputs = numberOfOutputs + ) + (boxesLocal.drop(numberOfInputs), tx +: transactions) + }._2 + + private val genHelper = TestHelper + + lazy val paymentTransactionValid: Transaction = { + val fee: Amount = genHelper.Props.txFee + val useBoxes: IndexedSeq[AssetBox] = IndexedSeq(genHelper.genAssetBox(publicKey.address.address), + genHelper.genAssetBox(publicKey.address.address)) + + TransactionFactory.defaultPaymentTransactionScratch(privKey, fee, timestamp, useBoxes, + publicKey.address.address, genHelper.Props.txAmount) + } + + def paymentTransactionDynamic: Transaction = { + val fee = genHelper.Props.txFee + val useBoxes = (0 to 5).map(_ => { + AssetBox( + EncryProposition.pubKeyLocked(publicKey.pubKeyBytes), + Random.nextLong(), + 999L + ) + }) + + TransactionFactory.defaultPaymentTransactionScratch(privKey, fee, Random.nextLong(), useBoxes, + publicKey.address.address, genHelper.Props.txAmount) + } + + def coinbaseTransaction: Transaction = { + TransactionFactory.coinbaseTransactionScratch(publicKey, timestamp, 10L, 0, Height @@ 100) + } + + def generateNextBlock(history: History, + difficultyDiff: BigInt = 0, + prevId: Option[ModifierId] = None, + txsQty: Int = 100, + additionalDifficulty: BigInt = 0): Block = { + val previousHeaderId: ModifierId = + prevId.getOrElse(history.getBestHeader.map(_.id).getOrElse(Header.GenesisParentId)) + val requiredDifficulty: Difficulty = history.getBestHeader.map(parent => + history.requiredDifficultyAfter(parent).getOrElse(Difficulty @@ BigInt(0))) + .getOrElse(history.settings.constants.InitialDifficulty) + val txs = (if (txsQty != 0) genValidPaymentTxs(Random.nextInt(txsQty)) else Seq.empty) ++ + Seq(coinbaseTransaction) + val header = genHeader.copy( + parentId = previousHeaderId, + height = history.getBestHeaderHeight + 1, + difficulty = Difficulty @@ (requiredDifficulty + difficultyDiff + additionalDifficulty), + transactionsRoot = Payload.rootHash(txs.map(_.id)) + ) + + Block(header, Payload(header.id, txs)) + } + + def generateForkOn(qty: Int, + addDifficulty: BigInt = 0, + from: Int, + to: Int, + settings: EncryAppSettings): (List[Block], List[Block]) = { + val history: History = dummyHistory(settings) + val forkInterval = (from until to).toList + (0 until qty).foldLeft((List(history), List.empty[Block] -> List.empty[Block])) { + case ((histories, blocks), blockHeight) => + if (forkInterval.contains(blockHeight)) { + if (histories.length == 1) { + val secondHistory = dummyHistory(settings) + blocks._1.foldLeft(secondHistory) { + case (prevHistory, blockToApply) => + prevHistory.append(blockToApply.header) + prevHistory.append(blockToApply.payload) + prevHistory.reportModifierIsValid(blockToApply) + prevHistory + } + val nextBlockInFirstChain = generateNextBlock(histories.head) + val nextBlockInSecondChain = generateNextBlock(secondHistory, additionalDifficulty = addDifficulty) + histories.head.append(nextBlockInFirstChain.header) + histories.head.append(nextBlockInFirstChain.payload) + val a = histories.head.reportModifierIsValid(nextBlockInFirstChain) + secondHistory.append(nextBlockInSecondChain.header) + secondHistory.append(nextBlockInSecondChain.payload) + val b = secondHistory.reportModifierIsValid(nextBlockInSecondChain) + (List(a, b), (blocks._1 :+ nextBlockInFirstChain) -> List(nextBlockInSecondChain)) + } else { + val nextBlockInFirstChain = generateNextBlock(histories.head) + val nextBlockInSecondChain = generateNextBlock(histories.last, additionalDifficulty = addDifficulty) + histories.head.append(nextBlockInFirstChain.header) + histories.head.append(nextBlockInFirstChain.payload) + val a = histories.head.reportModifierIsValid(nextBlockInFirstChain) + histories.last.append(nextBlockInSecondChain.header) + histories.last.append(nextBlockInSecondChain.payload) + val b = histories.last.reportModifierIsValid(nextBlockInSecondChain) + (List(a, b), (blocks._1 :+ nextBlockInFirstChain) -> (blocks._2 :+ nextBlockInSecondChain)) + } + } else { + val block: Block = generateNextBlock(histories.head) + histories.head.append(block.header) + histories.head.append(block.payload) + val a = histories.head.reportModifierIsValid(block) + (List(a), (blocks._1 :+ block) -> blocks._2) + } + }._2 + } + +} diff --git a/src/test/scala/encry/view/history/EncryHistoryTest.scala b/src/test/scala/encry/view/history/EncryHistoryTest.scala index 45b528b971..77a629ff20 100644 --- a/src/test/scala/encry/view/history/EncryHistoryTest.scala +++ b/src/test/scala/encry/view/history/EncryHistoryTest.scala @@ -1,10 +1,8 @@ package encry.view.history -import encry.modifiers.InstanceFactory -import encry.utils.EncryGenerator import org.scalatest.{Matchers, PropSpec} -class EncryHistoryTest extends PropSpec with Matchers with InstanceFactory with EncryGenerator { +class EncryHistoryTest extends PropSpec with Matchers { property("PreGenesis height test check.") { diff --git a/src/test/scala/encry/view/history/HistoryComparisionResultTest.scala b/src/test/scala/encry/view/history/HistoryComparisionResultTest.scala index 9c448ee320..146f9ebe34 100644 --- a/src/test/scala/encry/view/history/HistoryComparisionResultTest.scala +++ b/src/test/scala/encry/view/history/HistoryComparisionResultTest.scala @@ -1,23 +1,24 @@ package encry.view.history import encry.consensus.HistoryConsensus._ -import encry.modifiers.InstanceFactory -import encry.network.DeliveryManagerTests.DMUtils.generateBlocks -import encry.settings.{EncryAppSettings, TestNetSettings} +import encry.network.DeliveryManagerTests.DeliveryManagerUtils.generateBlocks +import encry.settings. TestNetSettings +import encry.utils.TestEntityGenerator import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.network.SyncInfo import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} +import encry.utils.HistoryGenerator.dummyHistory +import encry.utils.TestEntityGenerator._ class HistoryComparisionResultTest extends WordSpecLike with Matchers - with InstanceFactory with OneInstancePerTest with TestNetSettings { "History Reader" should { "mark history as Equal where our best header is the same as other history best header" in { - val history: History = generateDummyHistory(testNetSettings) - val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 + val history: History = dummyHistory(testNetSettings) + val blocks: List[Block] = generateBlocks(100, dummyHistory(testNetSettings))._2 val syncInfo: SyncInfo = SyncInfo(blocks.map(_.header.id)) val updatedHistory: History = blocks.foldLeft(history) { case (hst, block) => @@ -31,8 +32,8 @@ class HistoryComparisionResultTest extends WordSpecLike } "mark history as Older where our best header is in other history best chain, but not at the last position" in { - val history: History = generateDummyHistory(testNetSettings) - val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 + val history: History = dummyHistory(testNetSettings) + val blocks: List[Block] = generateBlocks(100, dummyHistory(testNetSettings))._2 val syncInfo: SyncInfo = SyncInfo(blocks.map(_.header.id)) val updatedHistory: History = blocks.take(50).foldLeft(history) { case (hst, block) => @@ -46,8 +47,8 @@ class HistoryComparisionResultTest extends WordSpecLike } "mark history as Younger when comparing history is empty" in { - val history: History = generateDummyHistory(testNetSettings) - val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 + val history: History = dummyHistory(testNetSettings) + val blocks: List[Block] = generateBlocks(100, dummyHistory(testNetSettings))._2 val syncInfo: SyncInfo = SyncInfo(Seq.empty) val updatedHistory: History = blocks.foldLeft(history) { case (hst, block) => @@ -62,8 +63,8 @@ class HistoryComparisionResultTest extends WordSpecLike "mark history as Younger when our history contains all other history but other history " + "doesn't contain our last 70 headers" in { - val history: History = generateDummyHistory(testNetSettings) - val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 + val history: History = dummyHistory(testNetSettings) + val blocks: List[Block] = generateBlocks(100, dummyHistory(testNetSettings))._2 val syncInfo: SyncInfo = SyncInfo(blocks.take(30).map(_.header.id)) val updatedHistory: History = blocks.foldLeft(history) { case (hst, block) => @@ -77,9 +78,9 @@ class HistoryComparisionResultTest extends WordSpecLike } "mark history as Fork when we have same point in histories" in { - val history: History = generateDummyHistory(testNetSettings) + val history: History = dummyHistory(testNetSettings) - val fork = genForkOn(100, 1000, 25, 30, testNetSettings) + val fork = generateForkOn(100, 1000, 25, 30, testNetSettings) val syncInfo: SyncInfo = SyncInfo( fork._1.take(25).map(_.header.id) ++: fork._2.map(_.header.id) @@ -96,7 +97,7 @@ class HistoryComparisionResultTest extends WordSpecLike } "mark history as Equal where both nodes do not keep any blocks" in { - val history: History = generateDummyHistory(testNetSettings) + val history: History = dummyHistory(testNetSettings) val syncInfo: SyncInfo = SyncInfo(Seq.empty) val comparisonResult = history.compare(syncInfo) @@ -104,9 +105,9 @@ class HistoryComparisionResultTest extends WordSpecLike } "mark history as Older " in { - val history: History = generateDummyHistory(testNetSettings) + val history: History = dummyHistory(testNetSettings) val syncInfo: SyncInfo = SyncInfo( - generateBlocks(30, generateDummyHistory(testNetSettings))._2.map(_.header.id)) + generateBlocks(30, dummyHistory(testNetSettings))._2.map(_.header.id)) val comparisonResult = history.compare(syncInfo) assert(comparisonResult == Older) diff --git a/src/test/scala/encry/view/history/ModifiersValidationTest.scala b/src/test/scala/encry/view/history/ModifiersValidationTest.scala index e879864ea8..6225f1a383 100644 --- a/src/test/scala/encry/view/history/ModifiersValidationTest.scala +++ b/src/test/scala/encry/view/history/ModifiersValidationTest.scala @@ -1,29 +1,32 @@ package encry.view.history -import encry.modifiers.InstanceFactory -import encry.network.DeliveryManagerTests.DMUtils.generateBlocks -import encry.settings.{EncryAppSettings, TestNetSettings} +import encry.network.DeliveryManagerTests.DeliveryManagerUtils.generateBlocks +import encry.settings.TestNetSettings +import encry.utils.ChainGenerator.genGenesisBlock +import encry.utils.HistoryGenerator.dummyHistory +import encry.utils.Keys import org.encryfoundation.common.modifiers.history.Block import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} class ModifiersValidationTest extends WordSpecLike with Matchers - with InstanceFactory with OneInstancePerTest + with Keys with TestNetSettings { "Modifiers validator" should { "validate genesis block" in { - val newHistory: History = generateDummyHistory(testNetSettings) - val genesisBlock: Block = generateGenesisBlock(testNetSettings.constants.GenesisHeight) + val newHistory: History = dummyHistory(testNetSettings) + val genesisBlock: Block = genGenesisBlock(privKey.publicImage, testNetSettings.constants.InitialEmissionAmount, + testNetSettings.constants.InitialDifficulty, testNetSettings.constants.GenesisHeight) newHistory.testApplicable(genesisBlock.header).isRight shouldBe true newHistory.append(genesisBlock.header) val updatedHistory: History = newHistory.reportModifierIsValid(genesisBlock.header) updatedHistory.testApplicable(genesisBlock.payload).isRight shouldBe true } "reject incorrect modifiers" in { - val blocks: List[Block] = generateBlocks(2, generateDummyHistory(testNetSettings))._2 - val newHistory: History = generateDummyHistory(testNetSettings) + val blocks: List[Block] = generateBlocks(2, dummyHistory(testNetSettings))._2 + val newHistory: History = dummyHistory(testNetSettings) blocks.take(1).foldLeft(newHistory) { case (history, block) => history.testApplicable(block.header).isRight shouldBe true history.append(block.header) diff --git a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala index fae9457a7b..a8fd032ea0 100644 --- a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala +++ b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala @@ -3,9 +3,9 @@ package encry.view.mempool import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestProbe} import com.typesafe.scalalogging.StrictLogging -import encry.modifiers.InstanceFactory -import encry.settings.{EncryAppSettings, TestNetSettings} +import encry.settings.TestNetSettings import encry.utils.NetworkTimeProvider +import encry.utils.TestEntityGenerator.genValidPaymentTxs import encry.view.mempool.MemoryPool.{NewTransaction, TransactionsForMiner} import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} @@ -13,7 +13,6 @@ import scala.concurrent.duration._ class MemoryPoolTests extends WordSpecLike with Matchers - with InstanceFactory with BeforeAndAfterAll with OneInstancePerTest with TestNetSettings diff --git a/src/test/scala/encry/view/state/EncryStateSpec.scala b/src/test/scala/encry/view/state/EncryStateSpec.scala index 3cae5063cb..6102b51692 100644 --- a/src/test/scala/encry/view/state/EncryStateSpec.scala +++ b/src/test/scala/encry/view/state/EncryStateSpec.scala @@ -1,10 +1,9 @@ package encry.view.state -import encry.utils.EncryGenerator import org.encryfoundation.common.modifiers.state.box.AssetBox import org.scalatest.{Matchers, PropSpec} -class EncryStateSpec extends PropSpec with Matchers with EncryGenerator { +class EncryStateSpec extends PropSpec with Matchers { property("EncryState.genesisBoxes() output equality") { diff --git a/src/test/scala/encry/view/state/UtxoStateSpec.scala b/src/test/scala/encry/view/state/UtxoStateSpec.scala index aabb14de53..3211d871c4 100755 --- a/src/test/scala/encry/view/state/UtxoStateSpec.scala +++ b/src/test/scala/encry/view/state/UtxoStateSpec.scala @@ -9,7 +9,7 @@ import encry.storage.VersionalStorage import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} import encry.storage.iodb.versionalIODB.IODBWrapper import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} -import encry.utils.{EncryGenerator, FileHelper, TestHelper} +import encry.utils.{TestEntityGenerator, FileHelper, TestHelper} import io.iohk.iodb.LSMStore import org.encryfoundation.common.modifiers.history.{Block, Payload} import org.encryfoundation.common.utils.constants.TestNetConstants @@ -18,7 +18,7 @@ import org.scalatest.{Matchers, PropSpec} import scala.concurrent.ExecutionContextExecutor -class UtxoStateSpec extends PropSpec with Matchers with EncryGenerator { +class UtxoStateSpec extends PropSpec with Matchers { // def utxoFromBoxHolder(bh: BoxHolder, // dir: File, diff --git a/src/test/scala/encry/view/wallet/WalletSpec.scala b/src/test/scala/encry/view/wallet/WalletSpec.scala index c039060746..69d63906c3 100644 --- a/src/test/scala/encry/view/wallet/WalletSpec.scala +++ b/src/test/scala/encry/view/wallet/WalletSpec.scala @@ -1,18 +1,21 @@ package encry.view.wallet import com.typesafe.scalalogging.StrictLogging -import encry.EncryApp -import encry.modifiers.InstanceFactory -import encry.settings.{EncryAppSettings, LevelDBSettings, Settings} +import encry.settings.{LevelDBSettings, Settings} import encry.utils.TestHelper.Props -import encry.utils.{EncryGenerator, FileHelper} +import encry.utils.FileHelper import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.modifiers.state.box.{AssetBox, MonetaryBox} import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.scalatest.{Matchers, PropSpec} +import encry.utils.TestEntityGenerator._ +import encry.utils.Utils.randomAddress -class WalletSpec extends PropSpec with Matchers with InstanceFactory with EncryGenerator with StrictLogging with Settings { +class WalletSpec extends PropSpec + with Matchers + with Settings + with StrictLogging { val dummyLevelDBSettings = LevelDBSettings(5) From aa0253bf5a4618ca3e8e18e56b29dfda709a9b94 Mon Sep 17 00:00:00 2001 From: capdev Date: Mon, 30 Sep 2019 16:19:22 +0500 Subject: [PATCH 33/39] fix HistoryGenerator settings --- src/test/scala/encry/utils/HistoryGenerator.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/scala/encry/utils/HistoryGenerator.scala b/src/test/scala/encry/utils/HistoryGenerator.scala index 49e01a4b5f..3e79ceb7d0 100644 --- a/src/test/scala/encry/utils/HistoryGenerator.scala +++ b/src/test/scala/encry/utils/HistoryGenerator.scala @@ -13,15 +13,15 @@ import org.iq80.leveldb.Options object HistoryGenerator { - def dummyHistory(settings: EncryAppSettings, withoutPow: Boolean = false): History = { + def dummyHistory(historySettings: EncryAppSettings, withoutPow: Boolean = false): History = { val indexStore: LSMStore = new LSMStore(FileHelper.getRandomTempDir, keepVersions = 0) val objectsStore: LSMStore = new LSMStore(FileHelper.getRandomTempDir, keepVersions = 0) val levelDBInit = LevelDbFactory.factory.open(FileHelper.getRandomTempDir, new Options) - val vldbInit = VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settings.levelDB)) + val vldbInit = VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, historySettings.levelDB)) val storage: HistoryStorage = new HistoryStorage(vldbInit) - val ntp: NetworkTimeProvider = new NetworkTimeProvider(settings.ntp) + val ntp: NetworkTimeProvider = new NetworkTimeProvider(historySettings.ntp) class EquihashPowSchemeWithoutValidateSolution(n: Char, k: Char, version: Byte, preGenesisHeight: Height, maxTarget: BigInt) extends EquihashPowScheme(n: Char, k: Char, version: Byte, preGenesisHeight: Height, maxTarget: BigInt) { @@ -33,8 +33,8 @@ object HistoryGenerator { override val historyStorage: HistoryStorage = storage override val timeProvider: NetworkTimeProvider = ntp override val powScheme: EquihashPowScheme = - new EquihashPowSchemeWithoutValidateSolution(settings.constants.n, settings.constants.k, - settings.constants.Version, settings.constants.PreGenesisHeight, settings.constants.MaxTarget) + new EquihashPowSchemeWithoutValidateSolution(historySettings.constants.n, historySettings.constants.k, + historySettings.constants.Version, historySettings.constants.PreGenesisHeight, historySettings.constants.MaxTarget) } } else new History { From 699131eeba8512d3ac3e6bf3d04595a9c85e7bbb Mon Sep 17 00:00:00 2001 From: capdev Date: Mon, 30 Sep 2019 16:36:23 +0500 Subject: [PATCH 34/39] merge master --- .../src/test/scala/benches/HistoryBenches.scala | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/benchmarks/src/test/scala/benches/HistoryBenches.scala b/benchmarks/src/test/scala/benches/HistoryBenches.scala index fb9ea64587..15933e9a97 100644 --- a/benchmarks/src/test/scala/benches/HistoryBenches.scala +++ b/benchmarks/src/test/scala/benches/HistoryBenches.scala @@ -4,14 +4,11 @@ import java.io.File import java.util.concurrent.TimeUnit import benches.HistoryBenches.HistoryBenchState -import benches.Utils.{generateHistory, generateNextBlockValidForHistory} -import encry.EncryApp -import encry.settings.EncryAppSettings +import benches.Utils.{coinbaseTransaction, generateHistory, generateNextBlockValidForHistory} import encry.utils.FileHelper import encry.view.history.History -import encryBenchmark.{BenchSettings, Settings} +import encryBenchmark.BenchSettings import org.encryfoundation.common.modifiers.history.Block -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.openjdk.jmh.annotations._ import org.openjdk.jmh.infra.Blackhole import org.openjdk.jmh.profile.GCProfiler @@ -73,9 +70,7 @@ object HistoryBenches extends BenchSettings { .foldLeft(initialHistory, Option.empty[Block], Vector.empty[Block]) { case ((prevHistory, prevBlock, vector), _) => val block: Block = - generateNextBlockValidForHistory( - prevHistory, 0, prevBlock, Seq.empty[Transaction] - ) + generateNextBlockValidForHistory(prevHistory, 0, prevBlock, Seq(coinbaseTransaction(0))) prevHistory.append(block.header) prevHistory.append(block.payload) (prevHistory.reportModifierIsValid(block), Some(block), vector :+ block) From 31111fc2d35ac0356882f724376849511ec9744d Mon Sep 17 00:00:00 2001 From: capdev Date: Mon, 30 Sep 2019 18:39:57 +0500 Subject: [PATCH 35/39] refactoring it.TransactionsUtil -> TransactionFactory, TestEntityGenerator --- .../TransactionsUtil.scala | 180 ------------------ .../AssetTokenTransactionTest.scala | 34 ++-- .../it/transactions/DataTransactionTest.scala | 9 +- ...ransferTransactionWithEncryCoinsTest.scala | 6 +- .../scala/encry/cli/commands/Transfer.scala | 4 +- .../mempool/TransactionFactory.scala | 77 ++------ 6 files changed, 40 insertions(+), 270 deletions(-) delete mode 100644 it/src/test/scala/TransactionGenerator/TransactionsUtil.scala diff --git a/it/src/test/scala/TransactionGenerator/TransactionsUtil.scala b/it/src/test/scala/TransactionGenerator/TransactionsUtil.scala deleted file mode 100644 index 7bb546a1fd..0000000000 --- a/it/src/test/scala/TransactionGenerator/TransactionsUtil.scala +++ /dev/null @@ -1,180 +0,0 @@ -package TransactionGenerator - -import com.google.common.primitives.{Bytes, Longs} -import com.typesafe.scalalogging.StrictLogging -import org.encryfoundation.common.crypto.{PrivateKey25519, PublicKey25519, Signature25519} -import org.encryfoundation.common.modifiers.mempool.directive._ -import org.encryfoundation.common.modifiers.mempool.transaction.{Input, Proof, PubKeyLockedContract, Transaction} -import org.encryfoundation.common.modifiers.state.box.{AssetBox, MonetaryBox, TokenIssuingBox} -import org.encryfoundation.common.modifiers.state.box.TokenIssuingBox.TokenId -import org.encryfoundation.common.utils.TaggedTypes.ADKey -import org.encryfoundation.prismlang.compiler.CompiledContract -import org.encryfoundation.prismlang.core.wrapped.BoxedValue -import scorex.crypto.hash.{Blake2b256, Digest32} - -object CreateTransaction extends StrictLogging { - - def defaultPaymentTransaction(privKey: PrivateKey25519, - fee: Long, - timestamp: Long, - useOutputs: Seq[(MonetaryBox, Option[(CompiledContract, Seq[Proof])])], - recipient: String, - encryCoinAmount: Long, - tokensAmount: Map[TokenId, Long] = Map()): Transaction = { - - val filteredBoxes: Map[Option[TokenId], (List[MonetaryBox], Long)] = - useOutputs.map(_._1).foldLeft(Map[Option[TokenId], (List[MonetaryBox], Long)]()) { - case (resultCollection, box) => - box match { - case tib: TokenIssuingBox => - resultCollection.updated( - Some(tib.tokenId), - (List(tib), tokensAmount.filter(elem => tib.tokenId.sameElements(elem._1)).head._2) - ) - case ab: AssetBox => - val currentBoxesByTokenId: Option[(List[MonetaryBox], Long)] = resultCollection.get(ab.tokenIdOpt) - currentBoxesByTokenId match { - case Some(collection) => resultCollection.updated(ab.tokenIdOpt, (collection._1 :+ ab, collection._2)) - case None => - val neededAmount: Long = ab.tokenIdOpt match { - case Some(tokenId) => tokensAmount.filter(elem => tokenId.sameElements(elem._1)).head._2 - case None => encryCoinAmount - } - resultCollection.updated(ab.tokenIdOpt, (List(ab), neededAmount)) - } - } - } - - val directivesForTransferAndChangeForAllCoins: IndexedSeq[TransferDirective] = - filteredBoxes.foldLeft(IndexedSeq[TransferDirective]()) { - case (collectionForTransfer, (tId, element)) => - val directiveForChange: IndexedSeq[TransferDirective] = - createDirectiveForChange( - privKey, - tId.map(_ => 0L).getOrElse(fee), - element._2, - element._1.map(_.amount).sum, - tId - ) - val collectionWithDirectiveForTransfer: IndexedSeq[TransferDirective] = - collectionForTransfer :+ TransferDirective(recipient, element._2, tId.map(id => ADKey @@ id)) - collectionWithDirectiveForTransfer ++ directiveForChange - } - - prepareTransaction( - privKey, - fee, - timestamp, - useOutputs, - directivesForTransferAndChangeForAllCoins - ) - } - - def scriptedAssetTransactionScratch(privKey: PrivateKey25519, - fee: Long, - timestamp: Long, - useOutputs: Seq[(MonetaryBox, Option[(CompiledContract, Seq[Proof])])], - contract: CompiledContract, - numberOfTokensForIssue: Long, - tokenIdOpt: Option[ADKey] = None): Transaction = { - val directiveForChange: IndexedSeq[TransferDirective] = - createDirectiveForChange(privKey, fee, amount = 0, useOutputs.map(_._1.amount).sum) - val directiveForScriptedAssetIssue: IndexedSeq[Directive] = - directiveForChange :+ ScriptedAssetDirective(contract.hash, numberOfTokensForIssue, tokenIdOpt) - prepareTransaction(privKey, fee, timestamp, useOutputs, directiveForScriptedAssetIssue) - } - - def assetIssuingTransactionScratch(privKey: PrivateKey25519, - fee: Long, - timestamp: Long, - useOutputs: Seq[(MonetaryBox, Option[(CompiledContract, Seq[Proof])])], - contract: CompiledContract, - numberOfTokensForIssue: Long): Transaction = { - val directiveForChange: IndexedSeq[TransferDirective] = - createDirectiveForChange(privKey, fee, amount = 0, useOutputs.map(_._1.amount).sum) - val directiveForTokenIssue: IndexedSeq[Directive] = - directiveForChange :+ AssetIssuingDirective(contract.hash, numberOfTokensForIssue) - prepareTransaction(privKey, fee, timestamp, useOutputs, directiveForTokenIssue) - } - - def dataTransactionScratch(privKey: PrivateKey25519, - fee: Long, - timestamp: Long, - useOutputs: Seq[(MonetaryBox, Option[(CompiledContract, Seq[Proof])])], - contract: CompiledContract, - data: Array[Byte]): Transaction = { - val directiveForChange: IndexedSeq[TransferDirective] = - createDirectiveForChange(privKey, fee, amount = 0, useOutputs.map(_._1.amount).sum) - val directiveForDataTransaction: IndexedSeq[Directive] = - directiveForChange :+ DataDirective(contract.hash, data) - prepareTransaction(privKey, fee, timestamp, useOutputs, directiveForDataTransaction) - } - - private def createDirectiveForChange(privKey: PrivateKey25519, - fee: Long, - amount: Long, - totalAmount: Long, - tokenId: Option[TokenId] = None): IndexedSeq[TransferDirective] = { - val change: Long = totalAmount - (amount + fee) - if (change < 0) { - logger.warn(s"Transaction impossible: required amount is bigger than available. Change is: $change.") - throw new RuntimeException(s"Transaction impossible: required amount is bigger than available $change") - } - if (change > 0) - IndexedSeq(TransferDirective(privKey.publicImage.address.address, change, tokenId.map(element => ADKey @@ element))) - else IndexedSeq() - } - - private def prepareTransaction(privKey: PrivateKey25519, - fee: Long, - timestamp: Long, - useOutputs: Seq[(MonetaryBox, Option[(CompiledContract, Seq[Proof])])], - directivesForTransfer: IndexedSeq[Directive]): Transaction = { - - val pubKey: PublicKey25519 = privKey.publicImage - val uInputs: IndexedSeq[Input] = useOutputs.toIndexedSeq.map { case (box, contractOpt) => - Input.unsigned( - box.id, - contractOpt match { - case Some((ct, _)) => Left(ct) - case None => Right(PubKeyLockedContract(pubKey.pubKeyBytes)) - } - ) - } - - val uTransaction: UnsignedEncryTransaction = UnsignedEncryTransaction(fee, timestamp, uInputs, directivesForTransfer) - val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) - val proofs: IndexedSeq[Seq[Proof]] = useOutputs.flatMap(_._2.map(_._2)).toIndexedSeq - - uTransaction.toSigned(proofs, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) - } -} - -case class UnsignedEncryTransaction(fee: Long, - timestamp: Long, - inputs: IndexedSeq[Input], - directives: IndexedSeq[Directive]) { - - val messageToSign: Array[Byte] = UnsignedEncryTransaction.bytesToSign(fee, timestamp, inputs, directives) - - def toSigned(proofs: IndexedSeq[Seq[Proof]], defaultProofOpt: Option[Proof]): Transaction = { - val signedInputs: IndexedSeq[Input] = inputs.zipWithIndex.map { case (input, idx) => - if (proofs.nonEmpty && proofs.lengthCompare(idx + 1) <= 0) input.copy(proofs = proofs(idx).toList) else input - } - Transaction(fee, timestamp, signedInputs, directives, defaultProofOpt) - } -} - -object UnsignedEncryTransaction { - - def bytesToSign(fee: Long, - timestamp: Long, - inputs: IndexedSeq[Input], - directives: IndexedSeq[Directive]): Digest32 = - Blake2b256.hash(Bytes.concat( - inputs.flatMap(_.bytesWithoutProof).toArray, - directives.flatMap(_.bytes).toArray, - Longs.toByteArray(timestamp), - Longs.toByteArray(fee) - )) -} \ No newline at end of file diff --git a/it/src/test/scala/encry/it/transactions/AssetTokenTransactionTest.scala b/it/src/test/scala/encry/it/transactions/AssetTokenTransactionTest.scala index 20a5eff9f3..dca97af232 100644 --- a/it/src/test/scala/encry/it/transactions/AssetTokenTransactionTest.scala +++ b/it/src/test/scala/encry/it/transactions/AssetTokenTransactionTest.scala @@ -1,17 +1,17 @@ package encry.it.transactions -import TransactionGenerator.CreateTransaction import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.consensus.EncrySupplyController import encry.it.configs.Configs import encry.it.docker.NodesFromDocker -import encry.utils.Keys +import encry.modifiers.mempool.TransactionFactory import encry.settings.Settings +import encry.utils.Keys import org.encryfoundation.common.crypto.PublicKey25519 import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.mempool.transaction.EncryAddress.Address -import org.encryfoundation.common.modifiers.mempool.transaction.{PubKeyLockedContract, Transaction} +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.modifiers.state.box.TokenIssuingBox.TokenId import org.encryfoundation.common.modifiers.state.box.{AssetBox, EncryBaseBox, TokenIssuingBox} import org.encryfoundation.common.utils.Algos @@ -19,10 +19,11 @@ import org.encryfoundation.common.utils.TaggedTypes.Height import org.scalatest.concurrent.ScalaFutures import org.scalatest.{AsyncFunSuite, Matchers} import scorex.crypto.signatures.Curve25519 -import scorex.utils.Random +import scorex.utils.{Random => ScorexRandom} -import scala.concurrent.{Await, Future} import scala.concurrent.duration._ +import scala.concurrent.{Await, Future} +import scala.util.Random class AssetTokenTransactionTest extends AsyncFunSuite with Matchers @@ -43,22 +44,21 @@ class AssetTokenTransactionTest extends AsyncFunSuite val secondHeightToWait: Int = 8 val thirdHeightToWait: Int = 11 val waitTime: FiniteDuration = 30.minutes - val amount: Int = scala.util.Random.nextInt(2000) - val fee: Int = scala.util.Random.nextInt(500) - val createdTokensAmount: Int = scala.util.Random.nextInt(2000) - val tokenAmount: Int = scala.util.Random.nextInt(500) - val recipientAddress: Address = PublicKey25519(Curve25519.createKeyPair(Random.randomBytes())._2).address.address + val amount: Int = Random.nextInt(2000) + val fee: Int = Random.nextInt(500) + val createdTokensAmount: Int = Random.nextInt(2000) + val tokenAmount: Int = Random.nextInt(500) + val recipientAddress: Address = PublicKey25519(Curve25519.createKeyPair(ScorexRandom.randomBytes())._2).address.address Await.result(dockerNodes().head.waitForHeadersHeight(firstHeightToWait), waitTime) val getBoxes: Seq[EncryBaseBox] = Await.result(dockerNodes().head.outputs, waitTime) val oneBox: AssetBox = getBoxes.collect { case ab: AssetBox => ab }.head - val transaction: Transaction = CreateTransaction.assetIssuingTransactionScratch( + val transaction: Transaction = TransactionFactory.assetIssuingTransactionScratch( privKey, fee, - timestamp = System.currentTimeMillis(), - useOutputs = IndexedSeq(oneBox).map(_ -> None), - contract = PubKeyLockedContract(privKey.publicImage.pubKeyBytes).contract, + System.currentTimeMillis(), + IndexedSeq(oneBox), createdTokensAmount ) @@ -103,14 +103,14 @@ class AssetTokenTransactionTest extends AsyncFunSuite case tb: TokenIssuingBox if tb.amount >= tokenAmount => tb }.head - val transactionWithAssetToken: Transaction = CreateTransaction.defaultPaymentTransaction( + val transactionWithAssetToken: Transaction = TransactionFactory.defaultPaymentTransactionScratch( privKey, fee, System.currentTimeMillis(), - IndexedSeq(assetBoxForFee, tokenIssuingBox).map(_ -> None), + IndexedSeq(assetBoxForFee, tokenIssuingBox), recipientAddress, amount, - Map(tokenId -> tokenAmount) + Some(tokenId) ) Await.result(dockerNodes().head.sendTransaction(transactionWithAssetToken), waitTime) diff --git a/it/src/test/scala/encry/it/transactions/DataTransactionTest.scala b/it/src/test/scala/encry/it/transactions/DataTransactionTest.scala index c025496a90..41632dd693 100644 --- a/it/src/test/scala/encry/it/transactions/DataTransactionTest.scala +++ b/it/src/test/scala/encry/it/transactions/DataTransactionTest.scala @@ -1,10 +1,10 @@ package encry.it.transactions -import TransactionGenerator.CreateTransaction import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.it.configs.Configs import encry.it.docker.NodesFromDocker +import encry.modifiers.mempool.TransactionFactory import encry.utils.Keys import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.mempool.transaction.{PubKeyLockedContract, Transaction} @@ -12,6 +12,7 @@ import org.encryfoundation.common.modifiers.state.box.{AssetBox, EncryBaseBox} import org.scalatest.concurrent.ScalaFutures import org.scalatest.{AsyncFunSuite, Matchers} import scorex.utils.Random + import scala.concurrent.{Await, Future} import scala.concurrent.duration._ @@ -37,12 +38,12 @@ class DataTransactionTest extends AsyncFunSuite val boxes: Seq[EncryBaseBox] = Await.result(dockerNodes().head.outputs, waitTime) val oneBox: AssetBox = boxes.collect { case ab: AssetBox => ab }.head - val transaction: Transaction = CreateTransaction.dataTransactionScratch( + val transaction: Transaction = TransactionFactory.dataTransactionScratch( privKey, fee, System.currentTimeMillis(), - IndexedSeq(oneBox).map(_ -> None), - PubKeyLockedContract(publicKey.pubKeyBytes).contract, + IndexedSeq(oneBox), + 0, Random.randomBytes(32) ) diff --git a/it/src/test/scala/encry/it/transactions/ProcessingTransferTransactionWithEncryCoinsTest.scala b/it/src/test/scala/encry/it/transactions/ProcessingTransferTransactionWithEncryCoinsTest.scala index 9dfe0a5130..ae5ee25053 100644 --- a/it/src/test/scala/encry/it/transactions/ProcessingTransferTransactionWithEncryCoinsTest.scala +++ b/it/src/test/scala/encry/it/transactions/ProcessingTransferTransactionWithEncryCoinsTest.scala @@ -1,11 +1,11 @@ package encry.it.transactions -import TransactionGenerator.CreateTransaction import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.consensus.EncrySupplyController import encry.it.configs.Configs import encry.it.docker.NodesFromDocker +import encry.modifiers.mempool.TransactionFactory import encry.utils.Keys import encry.settings.Settings import org.encryfoundation.common.crypto.PublicKey25519 @@ -53,11 +53,11 @@ class ProcessingTransferTransactionWithEncryCoinsTest extends AsyncFunSuite val boxes: Seq[EncryBaseBox] = Await.result(dockerNodes().head.outputs, waitTime) val oneBox: AssetBox = boxes.collect { case ab: AssetBox => ab }.head - val transaction: Transaction = CreateTransaction.defaultPaymentTransaction( + val transaction: Transaction = TransactionFactory.defaultPaymentTransactionScratch( privKey, fee, System.currentTimeMillis(), - IndexedSeq(oneBox).map(_ -> None), + IndexedSeq(oneBox), recipientAddress, amount ) diff --git a/src/main/scala/encry/cli/commands/Transfer.scala b/src/main/scala/encry/cli/commands/Transfer.scala index 7b8a6663fb..9cd2a28b53 100644 --- a/src/main/scala/encry/cli/commands/Transfer.scala +++ b/src/main/scala/encry/cli/commands/Transfer.scala @@ -37,11 +37,11 @@ object Transfer extends Command { .map(_.asInstanceOf[AssetBox]).foldLeft(Seq[AssetBox]()) { case (seq, box) => if (seq.map(_.amount).sum < (amount + fee)) seq :+ box else seq }.toIndexedSeq - TransactionFactory.defaultPaymentTransaction( + TransactionFactory.defaultPaymentTransactionScratch( secret, fee, System.currentTimeMillis(), - boxes.map(_ -> None), + boxes, recipient, amount) }.toOption diff --git a/src/main/scala/encry/modifiers/mempool/TransactionFactory.scala b/src/main/scala/encry/modifiers/mempool/TransactionFactory.scala index f02b8ad407..970c3f2d6b 100644 --- a/src/main/scala/encry/modifiers/mempool/TransactionFactory.scala +++ b/src/main/scala/encry/modifiers/mempool/TransactionFactory.scala @@ -1,5 +1,6 @@ package encry.modifiers.mempool +import com.google.common.primitives.{Bytes, Longs} import com.typesafe.scalalogging.StrictLogging import org.encryfoundation.common.crypto.{PrivateKey25519, PublicKey25519, Signature25519} import org.encryfoundation.common.modifiers.mempool.directive.{AssetIssuingDirective, DataDirective, Directive, ScriptedAssetDirective, TransferDirective} @@ -10,11 +11,23 @@ import org.encryfoundation.common.modifiers.state.box.MonetaryBox import org.encryfoundation.common.utils.TaggedTypes.{ADKey, Height} import org.encryfoundation.prismlang.compiler.CompiledContract import org.encryfoundation.prismlang.core.wrapped.BoxedValue +import scorex.crypto.hash.{Blake2b256, Digest32} import scala.util.Random object TransactionFactory extends StrictLogging { + def coinbaseTransactionScratch(pubKey: PublicKey25519, + timestamp: Long, + supply: Amount, + amount: Amount, + height: Height): Transaction = { + val directives: IndexedSeq[Directive with Product] = + IndexedSeq(TransferDirective(pubKey.address.address, amount + supply)) + + Transaction(0, timestamp, IndexedSeq.empty, directives, None) + } + def defaultPaymentTransactionScratch(privKey: PrivateKey25519, fee: Amount, timestamp: Long, @@ -39,70 +52,6 @@ object TransactionFactory extends StrictLogging { uTransaction.toSigned(IndexedSeq.empty, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) } - def coinbaseTransactionScratch(pubKey: PublicKey25519, - timestamp: Long, - supply: Amount, - amount: Amount, - height: Height): Transaction = { - val directives: IndexedSeq[Directive with Product] = - IndexedSeq(TransferDirective(pubKey.address.address, amount + supply)) - - Transaction(0, timestamp, IndexedSeq.empty, directives, None) - } - - def defaultPaymentTransaction(privKey: PrivateKey25519, - fee: Long, - timestamp: Long, - useOutputs: Seq[(MonetaryBox, Option[(CompiledContract, Seq[Proof])])], - recipient: String, - amount: Long, - tokenIdOpt: Option[ADKey] = None): Transaction = { - val howMuchCanTransfer: Long = useOutputs.map(_._1.amount).sum - fee - val howMuchWillTransfer: Long = howMuchCanTransfer - Math.abs(Random.nextLong % howMuchCanTransfer) - val change: Long = howMuchCanTransfer - howMuchWillTransfer - val directives: IndexedSeq[TransferDirective] = - IndexedSeq(TransferDirective(recipient, howMuchWillTransfer, tokenIdOpt)) - prepareTransaction(privKey, fee, timestamp, useOutputs, directives, change, tokenIdOpt) - } - - def prepareTransaction(privKey: PrivateKey25519, - fee: Long, - timestamp: Long, - useOutputs: Seq[(MonetaryBox, Option[(CompiledContract, Seq[Proof])])], - directivesSeq: IndexedSeq[Directive], - amount: Long, - tokenIdOpt: Option[ADKey] = None): Transaction = { - - val pubKey: PublicKey25519 = privKey.publicImage - - val uInputs: IndexedSeq[Input] = useOutputs.toIndexedSeq.map { case (box, contractOpt) => - Input.unsigned( - box.id, - contractOpt match { - case Some((ct, _)) => Left(ct) - case None => Right(PubKeyLockedContract(pubKey.pubKeyBytes)) - } - ) - } - - val change: Long = amount - - if (change < 0) { - logger.warn(s"Transaction impossible: required amount is bigger than available. Change is: $change.") - throw new RuntimeException("Transaction impossible: required amount is bigger than available.") - } - - val directives: IndexedSeq[Directive] = - if (change > 0) directivesSeq ++: IndexedSeq(TransferDirective(pubKey.address.address, change, tokenIdOpt)) - else directivesSeq - - val uTransaction: UnsignedTransaction = UnsignedTransaction(fee, timestamp, uInputs, directives) - val signature: Signature25519 = privKey.sign(uTransaction.messageToSign) - val proofs: IndexedSeq[Seq[Proof]] = useOutputs.flatMap(_._2.map(_._2)).toIndexedSeq - - uTransaction.toSigned(proofs, Some(Proof(BoxedValue.Signature25519Value(signature.bytes.toList)))) - } - def paymentTransactionWithMultipleOutputs(privKey: PrivateKey25519, fee: Amount, timestamp: Long, useBoxes: IndexedSeq[MonetaryBox], recipient: Address, amount: Amount, tokenIdOpt: Option[ADKey] = None, numOfOutputs: Int): Transaction = { From 7abc70fa101ae027e35ed58f6446890834e54563 Mon Sep 17 00:00:00 2001 From: capdev Date: Tue, 1 Oct 2019 14:06:09 +0500 Subject: [PATCH 36/39] some fixes --- .../src/test/scala/benches/StateBenches.scala | 81 ++++++++++--------- .../scala/benches/StateRollbackBench.scala | 70 ++++++++-------- .../AssetTokenTransactionTest.scala | 4 +- .../scala/encry/utils/ChainGenerator.scala | 5 +- 4 files changed, 83 insertions(+), 77 deletions(-) diff --git a/benchmarks/src/test/scala/benches/StateBenches.scala b/benchmarks/src/test/scala/benches/StateBenches.scala index 77060df1af..b94947dc88 100644 --- a/benchmarks/src/test/scala/benches/StateBenches.scala +++ b/benchmarks/src/test/scala/benches/StateBenches.scala @@ -1,16 +1,18 @@ package benches import java.io.File +import java.security.PublicKey import java.util.concurrent.TimeUnit import benches.StateBenches.StateBenchState import org.openjdk.jmh.annotations._ import encry.modifiers.mempool.TransactionFactory.paymentTransactionWithMultipleOutputs +import encry.settings.Settings import encry.storage.VersionalStorage import encry.storage.VersionalStorage.IODB import encry.utils.ChainGenerator._ import encry.utils.FileHelper.getRandomTempDir -import encry.utils.TestEntityGenerator +import encry.utils.{Keys, TestEntityGenerator} import encry.utils.Utils.randomAddress import encry.view.state.{BoxHolder, UtxoState} import encryBenchmark.BenchSettings @@ -32,7 +34,7 @@ class StateBenches { def applyBlocksToTheState(stateBench: StateBenchState, bh: Blackhole): Unit = { bh.consume { val innerState: UtxoState = - utxoFromBoxHolder(stateBench.boxesHolder, getRandomTempDir, None, stateBench.settings, VersionalStorage.LevelDB) + utxoFromBoxHolder(stateBench.boxesHolder, getRandomTempDir, stateBench.settings, VersionalStorage.LevelDB) stateBench.chain.foldLeft(innerState) { case (state, block) => state.applyModifier(block).right.get } @@ -44,7 +46,7 @@ class StateBenches { def readStateFileBench(stateBench: StateBenchState, bh: Blackhole): Unit = { bh.consume { val localState: UtxoState = - utxoFromBoxHolder(stateBench.boxesHolder, stateBench.tmpDir, None, stateBench.settings, IODB) + utxoFromBoxHolder(stateBench.boxesHolder, stateBench.tmpDir, stateBench.settings, IODB) localState.close() } } @@ -71,16 +73,16 @@ object StateBenches extends BenchSettings { } @State(Scope.Benchmark) - class StateBenchState extends encry.settings.Settings { + class StateBenchState extends Settings with Keys { val tmpDir: File = getRandomTempDir val initialBoxes: IndexedSeq[AssetBox] = (0 until benchSettings.stateBenchSettings.totalBoxesNumber).map(nonce => - TestEntityGenerator.genHardcodedBox(privKey.publicImage.address.address, nonce) + TestEntityGenerator.genHardcodedBox(publicKey.address.address, nonce) ) val boxesHolder: BoxHolder = BoxHolder(initialBoxes) - var state: UtxoState = utxoFromBoxHolder(boxesHolder, tmpDir, None, settings, VersionalStorage.LevelDB) - val genesisBlock: Block = genGenesisBlock(privKey.publicImage, settings.constants.InitialEmissionAmount, + var state: UtxoState = utxoFromBoxHolder(boxesHolder, tmpDir, settings, VersionalStorage.LevelDB) + val genesisBlock: Block = genGenesisBlock(publicKey, settings.constants.InitialEmissionAmount, settings.constants.InitialDifficulty, settings.constants.GenesisHeight) state = state.applyModifier(genesisBlock).right.get @@ -112,38 +114,39 @@ object StateBenches extends BenchSettings { val chain: Vector[Block] = genesisBlock +: stateGenerationResults._1 state = stateGenerationResults._3 state.close() - } - def generateNextBlockValidForState(prevBlock: Block, - state: UtxoState, - box: Seq[AssetBox], - transactionsNumberInEachBlock: Int, - numberOfInputsInOneTransaction: Int, - numberOfOutputsInOneTransaction: Int): Block = { - - val transactions: Seq[Transaction] = (0 until transactionsNumberInEachBlock).foldLeft(box, Seq.empty[Transaction]) { - case ((boxes, transactionsL), _) => - val tx: Transaction = paymentTransactionWithMultipleOutputs( - privKey, - fee = 111, - timestamp = 11L, - useBoxes = boxes.take(numberOfInputsInOneTransaction).toIndexedSeq, - recipient = randomAddress, - amount = 10000, - numOfOutputs = numberOfOutputsInOneTransaction - ) - (boxes.drop(numberOfInputsInOneTransaction), transactionsL :+ tx) - }._2 ++ Seq(TestEntityGenerator.coinbaseTransaction(prevBlock.header.height + 1)) - val header = Header( - 1.toByte, - prevBlock.id, - Payload.rootHash(transactions.map(_.id)), - System.currentTimeMillis(), - prevBlock.header.height + 1, - Random.nextLong(), - Difficulty @@ BigInt(1), - EquihashSolution(Seq(1, 3)) - ) - Block(header, Payload(header.id, transactions)) + def generateNextBlockValidForState(prevBlock: Block, + state: UtxoState, + box: Seq[AssetBox], + transactionsNumberInEachBlock: Int, + numberOfInputsInOneTransaction: Int, + numberOfOutputsInOneTransaction: Int): Block = { + + val transactions: Seq[Transaction] = (0 until transactionsNumberInEachBlock).foldLeft(box, Seq.empty[Transaction]) { + case ((boxes, transactionsL), _) => + val tx: Transaction = paymentTransactionWithMultipleOutputs( + privKey, + fee = 111, + timestamp = 11L, + useBoxes = boxes.take(numberOfInputsInOneTransaction).toIndexedSeq, + recipient = randomAddress, + amount = 10000, + numOfOutputs = numberOfOutputsInOneTransaction + ) + (boxes.drop(numberOfInputsInOneTransaction), transactionsL :+ tx) + }._2 ++ Seq(TestEntityGenerator.coinbaseTransaction(prevBlock.header.height + 1)) + val header = Header( + 1.toByte, + prevBlock.id, + Payload.rootHash(transactions.map(_.id)), + System.currentTimeMillis(), + prevBlock.header.height + 1, + Random.nextLong(), + Difficulty @@ BigInt(1), + EquihashSolution(Seq(1, 3)) + ) + Block(header, Payload(header.id, transactions)) + } } + } \ No newline at end of file diff --git a/benchmarks/src/test/scala/benches/StateRollbackBench.scala b/benchmarks/src/test/scala/benches/StateRollbackBench.scala index c9f954d0a0..d1f11f7d4e 100644 --- a/benchmarks/src/test/scala/benches/StateRollbackBench.scala +++ b/benchmarks/src/test/scala/benches/StateRollbackBench.scala @@ -5,6 +5,7 @@ import java.util.concurrent.TimeUnit import benches.StateRollbackBench.StateRollbackState import encry.modifiers.mempool.TransactionFactory.paymentTransactionWithMultipleOutputs +import encry.settings.Settings import encry.storage.VersionalStorage import encry.utils.CoreTaggedTypes.VersionTag import encry.view.state.{BoxHolder, UtxoState} @@ -19,7 +20,7 @@ import org.openjdk.jmh.runner.{Runner, RunnerException} import org.openjdk.jmh.runner.options.{OptionsBuilder, TimeValue, VerboseMode} import encry.utils.FileHelper.getRandomTempDir import encry.utils.ChainGenerator.{genGenesisBlock, utxoFromBoxHolder} -import encry.utils.TestEntityGenerator +import encry.utils.{Keys, TestEntityGenerator} import org.encryfoundation.common.crypto.equihash.EquihashSolution import org.encryfoundation.common.modifiers.mempool.transaction.Transaction @@ -31,7 +32,7 @@ class StateRollbackBench { def applyBlocksToTheState(stateBench: StateRollbackState, bh: Blackhole): Unit = { bh.consume { val innerState: UtxoState = - utxoFromBoxHolder(stateBench.boxesHolder, getRandomTempDir, None, stateBench.settings, VersionalStorage.IODB) + utxoFromBoxHolder(stateBench.boxesHolder, getRandomTempDir, stateBench.settings, VersionalStorage.IODB) val newState = stateBench.chain.foldLeft(innerState -> List.empty[VersionTag]) { case ((state, rootHashes), block) => val newState = state.applyModifier(block).right.get newState -> (rootHashes :+ newState.version) @@ -64,7 +65,7 @@ object StateRollbackBench extends BenchSettings { } @State(Scope.Benchmark) - class StateRollbackState extends encry.settings.Settings { + class StateRollbackState extends Settings with Keys { val tmpDir: File = getRandomTempDir @@ -72,7 +73,7 @@ object StateRollbackBench extends BenchSettings { TestEntityGenerator.genHardcodedBox(privKey.publicImage.address.address, nonce) ) val boxesHolder: BoxHolder = BoxHolder(initialBoxes) - var state: UtxoState = utxoFromBoxHolder(boxesHolder, tmpDir, None, settings, VersionalStorage.LevelDB) + var state: UtxoState = utxoFromBoxHolder(boxesHolder, tmpDir, settings, VersionalStorage.LevelDB) val genesisBlock: Block = genGenesisBlock(privKey.publicImage, settings.constants.InitialEmissionAmount, settings.constants.InitialDifficulty, settings.constants.GenesisHeight) @@ -104,37 +105,38 @@ object StateRollbackBench extends BenchSettings { val forkBlocks: List[Block] = genesisBlock +: stateGenerationResults._1.map(_._2) state = stateGenerationResults._3 state.close() - } - def generateNextBlockForStateWithSpendingAllPreviousBoxes(prevBlock: Block, - state: UtxoState, - box: Seq[AssetBox], - splitCoef: Int = 2, - addDiff: Difficulty = Difficulty @@ BigInt(0)): Block = { + def generateNextBlockForStateWithSpendingAllPreviousBoxes(prevBlock: Block, + state: UtxoState, + box: Seq[AssetBox], + splitCoef: Int = 2, + addDiff: Difficulty = Difficulty @@ BigInt(0)): Block = { - val transactions: Seq[Transaction] = box.indices.foldLeft(box, Seq.empty[Transaction]) { - case ((boxes, transactionsL), _) => - val tx: Transaction = paymentTransactionWithMultipleOutputs( - privKey, - fee = 1, - timestamp = 11L, - useBoxes = IndexedSeq(boxes.head), - recipient = privKey.publicImage.address.address, - amount = boxes.head.amount - 1, - numOfOutputs = splitCoef - ) - (boxes.tail, transactionsL :+ tx) - }._2.filter(tx => state.validate(tx).isRight) ++ Seq(TestEntityGenerator.coinbaseTransaction(prevBlock.header.height + 1)) - val header = Header( - 1.toByte, - prevBlock.id, - Payload.rootHash(transactions.map(_.id)), - System.currentTimeMillis(), - prevBlock.header.height + 1, - Random.nextLong(), - Difficulty @@ (BigInt(1) + addDiff), - EquihashSolution(Seq(1, 3)) - ) - Block(header, Payload(header.id, transactions)) + val transactions: Seq[Transaction] = box.indices.foldLeft(box, Seq.empty[Transaction]) { + case ((boxes, transactionsL), _) => + val tx: Transaction = paymentTransactionWithMultipleOutputs( + privKey, + fee = 1, + timestamp = 11L, + useBoxes = IndexedSeq(boxes.head), + recipient = privKey.publicImage.address.address, + amount = boxes.head.amount - 1, + numOfOutputs = splitCoef + ) + (boxes.tail, transactionsL :+ tx) + }._2.filter(tx => state.validate(tx).isRight) ++ Seq(TestEntityGenerator.coinbaseTransaction(prevBlock.header.height + 1)) + val header = Header( + 1.toByte, + prevBlock.id, + Payload.rootHash(transactions.map(_.id)), + System.currentTimeMillis(), + prevBlock.header.height + 1, + Random.nextLong(), + Difficulty @@ (BigInt(1) + addDiff), + EquihashSolution(Seq(1, 3)) + ) + Block(header, Payload(header.id, transactions)) + } } + } \ No newline at end of file diff --git a/it/src/test/scala/encry/it/transactions/AssetTokenTransactionTest.scala b/it/src/test/scala/encry/it/transactions/AssetTokenTransactionTest.scala index dca97af232..ca9286b395 100644 --- a/it/src/test/scala/encry/it/transactions/AssetTokenTransactionTest.scala +++ b/it/src/test/scala/encry/it/transactions/AssetTokenTransactionTest.scala @@ -15,7 +15,7 @@ import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.modifiers.state.box.TokenIssuingBox.TokenId import org.encryfoundation.common.modifiers.state.box.{AssetBox, EncryBaseBox, TokenIssuingBox} import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.Height +import org.encryfoundation.common.utils.TaggedTypes.{ADKey, Height} import org.scalatest.concurrent.ScalaFutures import org.scalatest.{AsyncFunSuite, Matchers} import scorex.crypto.signatures.Curve25519 @@ -110,7 +110,7 @@ class AssetTokenTransactionTest extends AsyncFunSuite IndexedSeq(assetBoxForFee, tokenIssuingBox), recipientAddress, amount, - Some(tokenId) + Some(ADKey @@ tokenId) ) Await.result(dockerNodes().head.sendTransaction(transactionWithAssetToken), waitTime) diff --git a/src/test/scala/encry/utils/ChainGenerator.scala b/src/test/scala/encry/utils/ChainGenerator.scala index 61a2e4f3a7..ab0b995dbc 100644 --- a/src/test/scala/encry/utils/ChainGenerator.scala +++ b/src/test/scala/encry/utils/ChainGenerator.scala @@ -35,7 +35,7 @@ object ChainGenerator { val genesisBlock: Block = genGenesisBlock(privKey.publicImage, settings.constants.InitialEmissionAmount, settings.constants.InitialDifficulty, Height @@ 0) - val state: UtxoState = utxoFromBoxHolder(BoxHolder(Seq.empty), dir, None, settings, VersionalStorage.LevelDB) + val state: UtxoState = utxoFromBoxHolder(BoxHolder(Seq.empty), dir, settings, VersionalStorage.LevelDB) val genesisState = state.applyModifier(genesisBlock).right.get val (forkBlock, forkBoxes) = @@ -55,7 +55,7 @@ object ChainGenerator { (state, newState, chain) } - def utxoFromBoxHolder(bh: BoxHolder, dir: File, nodeViewHolderRef: Option[ActorRef], settings: EncryAppSettings, + def utxoFromBoxHolder(bh: BoxHolder, dir: File, settings: EncryAppSettings, storageType: StorageType): UtxoState = { val storage = settings.storage.state match { case VersionalStorage.IODB => @@ -85,6 +85,7 @@ object ChainGenerator { val txs: Seq[Transaction] = Seq(coinbaseTrans) val txsRoot: Digest32 = Payload.rootHash(txs.map(_.id)) + val header = Header( 1.toByte, Header.GenesisParentId, From 2a23b8715e53f273935fbfe980070450c50a9506 Mon Sep 17 00:00:00 2001 From: capdev Date: Tue, 1 Oct 2019 14:08:02 +0500 Subject: [PATCH 37/39] test comment --- .../modifiers/history/HistoryApplicatorTest.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 9940a58532..9dea1351a7 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -99,7 +99,6 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) checkBestBlock(history, chain(blockQty - 1)) } - "apply remote blocks and check chain sync" in { val blockQty = 10 @@ -185,6 +184,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) } "sync and rollback headers for invalid payload" in { + val blockQty = 10 val history: History = dummyHistory(testSettings, withoutPow = true) //generate invalid blocks begining from 6 blocks @@ -202,16 +202,18 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) headers.foreach(historyApplicator ! ModifierFromRemote(_)) - awaitCond(history.getBestHeaderHeight == blockQty - 1, timeout, 500 millis, - s"history.getBestHeaderHeight ${history.getBestHeaderHeight} expected ${blockQty - 1}") - history.getBestHeaderHeight shouldBe blockQty - 1 - history.getBestHeader.map(h => Algos.encode(h.id)) shouldBe Some(Algos.encode(chain(blockQty - 1).header.id)) + /* Если раскоментировать то тест падает, getBestHeaderHeight иногда 6 иногда 7, не доходит до 9 */ +// awaitCond(history.getBestHeaderHeight == blockQty - 1, timeout, 500 millis, +// s"history.getBestHeaderHeight ${history.getBestHeaderHeight} expected ${blockQty - 1}") +// history.getBestHeaderHeight shouldBe blockQty - 1 +// history.getBestHeader.map(h => Algos.encode(h.id)) shouldBe Some(Algos.encode(chain(blockQty - 1).header.id)) payloads.foreach(historyApplicator ! ModifierFromRemote(_)) expectMsg(timeout, FullBlockChainIsSynced()) awaitCond(history.getBestBlockHeight == 4, timeout, 500 millis, s"history.getBestBlockHeight ${history.getBestBlockHeight} expected 4") + checkBestBlock(history, chain(4)) } From 7fb77db6e780eedcae4a8bd73561ecfa068ef8ec Mon Sep 17 00:00:00 2001 From: capdev Date: Tue, 1 Oct 2019 14:12:02 +0500 Subject: [PATCH 38/39] rename --- .../history/HistoryApplicatorTest.scala | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 9dea1351a7..0b2d46e0ff 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -58,14 +58,14 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) awaitCond(history.getBestBlockHeight == 4, timeout) - checkBestBlock(history, chain(4)) + checkBest(history, chain(4)) } - def checkBestBlock(history: History, expectBlock: Block) { - history.getBestHeaderHeight shouldBe expectBlock.header.height - history.getBestHeader.map(h => Algos.encode(h.id)) shouldBe Some(Algos.encode(expectBlock.header.id)) - history.getBestBlock.map(b => Algos.encode(b.id)) shouldBe Some(Algos.encode(expectBlock.id)) - history.getBestBlockHeightDB shouldBe expectBlock.header.height + def checkBest(history: History, expectedBlock: Block) { + history.getBestHeaderHeight shouldBe expectedBlock.header.height + history.getBestHeader.map(h => Algos.encode(h.id)) shouldBe Some(Algos.encode(expectedBlock.header.id)) + history.getBestBlock.map(b => Algos.encode(b.id)) shouldBe Some(Algos.encode(expectedBlock.id)) + history.getBestBlockHeightDB shouldBe expectedBlock.header.height } val dir: File = FileHelper.getRandomTempDir @@ -96,7 +96,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) expectMsg(timeout, FullBlockChainIsSynced()) awaitCond(history.getBestBlockHeight == blockQty - 1, timeout) - checkBestBlock(history, chain(blockQty - 1)) + checkBest(history, chain(blockQty - 1)) } "apply remote blocks and check chain sync" in { @@ -119,7 +119,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) expectMsg(timeout, FullBlockChainIsSynced()) awaitCond(history.getBestBlockHeight == blockQty - 1, timeout) - checkBestBlock(history, chain(blockQty - 1)) + checkBest(history, chain(blockQty - 1)) } "apply remote blocks and check queue for rollback height" in { @@ -144,7 +144,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) expectMsg(timeout, FullBlockChainIsSynced()) awaitCond(history.getBestBlockHeight == testSettings.levelDB.maxVersions + overQty - 1) - checkBestBlock(history, chain(testSettings.levelDB.maxVersions + overQty - 1)) + checkBest(history, chain(testSettings.levelDB.maxVersions + overQty - 1)) } "should sync and check history rollback on invalid block " in { @@ -180,7 +180,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) expectMsg(timeout, FullBlockChainIsSynced()) awaitCond(history.getBestBlockHeight == blockQty - 1, timeout) - checkBestBlock(history, chain(blockQty - 1)) + checkBest(history, chain(blockQty - 1)) } "sync and rollback headers for invalid payload" in { @@ -214,7 +214,7 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) awaitCond(history.getBestBlockHeight == 4, timeout, 500 millis, s"history.getBestBlockHeight ${history.getBestBlockHeight} expected 4") - checkBestBlock(history, chain(4)) + checkBest(history, chain(4)) } } From 70ae6d6f84124a35aa469f16394fe9ca41516122 Mon Sep 17 00:00:00 2001 From: capdev Date: Tue, 1 Oct 2019 16:05:19 +0500 Subject: [PATCH 39/39] fix "timestamp 1569926861646 less than parent's 1569926861646" --- .../modifiers/history/HistoryApplicatorTest.scala | 11 +++++------ src/test/scala/encry/utils/ChainGenerator.scala | 1 + 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala index 0b2d46e0ff..ca4cb35e06 100644 --- a/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala +++ b/src/test/scala/encry/modifiers/history/HistoryApplicatorTest.scala @@ -202,16 +202,15 @@ class HistoryApplicatorTest extends TestKit(ActorSystem()) headers.foreach(historyApplicator ! ModifierFromRemote(_)) - /* Если раскоментировать то тест падает, getBestHeaderHeight иногда 6 иногда 7, не доходит до 9 */ -// awaitCond(history.getBestHeaderHeight == blockQty - 1, timeout, 500 millis, -// s"history.getBestHeaderHeight ${history.getBestHeaderHeight} expected ${blockQty - 1}") -// history.getBestHeaderHeight shouldBe blockQty - 1 -// history.getBestHeader.map(h => Algos.encode(h.id)) shouldBe Some(Algos.encode(chain(blockQty - 1).header.id)) + awaitCond(history.getBestHeaderHeight == blockQty - 1, timeout, 100 millis, + s"history.getBestHeaderHeight ${history.getBestHeaderHeight} expected ${blockQty - 1}") + history.getBestHeaderHeight shouldBe blockQty - 1 + history.getBestHeader.map(h => Algos.encode(h.id)) shouldBe Some(Algos.encode(chain(blockQty - 1).header.id)) payloads.foreach(historyApplicator ! ModifierFromRemote(_)) expectMsg(timeout, FullBlockChainIsSynced()) - awaitCond(history.getBestBlockHeight == 4, timeout, 500 millis, + awaitCond(history.getBestBlockHeight == 4, timeout, 100 millis, s"history.getBestBlockHeight ${history.getBestBlockHeight} expected 4") checkBest(history, chain(4)) diff --git a/src/test/scala/encry/utils/ChainGenerator.scala b/src/test/scala/encry/utils/ChainGenerator.scala index ab0b995dbc..fd0b901d74 100644 --- a/src/test/scala/encry/utils/ChainGenerator.scala +++ b/src/test/scala/encry/utils/ChainGenerator.scala @@ -46,6 +46,7 @@ object ChainGenerator { val (chain, _, newState, _) = (2 until blockQty).foldLeft(List(genesisBlock, forkBlock), forkBlock, forkState, forkBoxes) { case ((blocks, blockL, stateL, boxes), height) => + Thread.sleep(1)//timestamp 1569926861646 less than parent's 1569926861646 val invalid = genInvalidBlockFrom.exists(height + 1 >= _) val (newBlock, newBoxes) = genNextBlockForState(privKey, blockL, settings.constants.InitialEmissionAmount, stateL, boxes, randomAddress, invalid)