From cbe81b60b4e56994208a0a0eb6f7d66b0334a098 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 28 Feb 2020 17:05:28 +0300 Subject: [PATCH 001/177] add new actors --- src/main/scala/encry/network/DM.scala | 7 +++++++ src/main/scala/encry/network/MessageBuilder.scala | 7 +++++++ src/main/scala/encry/network/NetworkRouter.scala | 7 +++++++ src/main/scala/encry/network/PK.scala | 7 +++++++ 4 files changed, 28 insertions(+) create mode 100644 src/main/scala/encry/network/DM.scala create mode 100644 src/main/scala/encry/network/MessageBuilder.scala create mode 100644 src/main/scala/encry/network/NetworkRouter.scala create mode 100644 src/main/scala/encry/network/PK.scala diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala new file mode 100644 index 0000000000..ea5b777bf2 --- /dev/null +++ b/src/main/scala/encry/network/DM.scala @@ -0,0 +1,7 @@ +package encry.network + +import akka.actor.Actor + +class DM extends Actor{ + override def receive: Receive = ??? +} diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala new file mode 100644 index 0000000000..1dd6b4ea0c --- /dev/null +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -0,0 +1,7 @@ +package encry.network + +import akka.actor.Actor + +class MessageBuilder extends Actor { + override def receive: Receive = ??? +} diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala new file mode 100644 index 0000000000..5885fd127e --- /dev/null +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -0,0 +1,7 @@ +package encry.network + +import akka.actor.Actor + +class NetworkRouter extends Actor { + override def receive: Receive = ??? +} diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala new file mode 100644 index 0000000000..835cdb926e --- /dev/null +++ b/src/main/scala/encry/network/PK.scala @@ -0,0 +1,7 @@ +package encry.network + +import akka.actor.Actor + +class PK extends Actor{ + override def receive: Receive = ??? +} From 36f4f7d97c780722f379f001ae8d62f7ef4e9b95 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 2 Mar 2020 17:23:47 +0300 Subject: [PATCH 002/177] add some messages --- src/main/scala/encry/network/BlackList.scala | 9 +- src/main/scala/encry/network/DM.scala | 8 +- .../scala/encry/network/MessageBuilder.scala | 31 ++++- src/main/scala/encry/network/Messages.scala | 16 +++ .../encry/network/NetworkController.scala | 1 - .../scala/encry/network/NetworkRouter.scala | 112 +++++++++++++++++- .../encry/network/NodeViewSynchronizer.scala | 3 - src/main/scala/encry/network/PK.scala | 28 ++++- .../scala/encry/view/NodeViewHolder.scala | 1 + 9 files changed, 191 insertions(+), 18 deletions(-) create mode 100644 src/main/scala/encry/network/Messages.scala diff --git a/src/main/scala/encry/network/BlackList.scala b/src/main/scala/encry/network/BlackList.scala index 9cbb5ea8ff..a360757969 100644 --- a/src/main/scala/encry/network/BlackList.scala +++ b/src/main/scala/encry/network/BlackList.scala @@ -1,11 +1,12 @@ package encry.network import java.net.InetAddress + import encry.network.BlackList.BanType.{PermanentBan, TemporaryBan} import encry.network.BlackList._ -import encry.settings.EncryAppSettings +import encry.settings.BlackListSettings -final case class BlackList(settings: EncryAppSettings, +final case class BlackList(settings: BlackListSettings, private val blackList: Map[InetAddress, (BanReason, BanTime, BanType)]) { def contains(peer: InetAddress): Boolean = blackList.contains(peer) @@ -16,7 +17,7 @@ final case class BlackList(settings: EncryAppSettings, }))) def cleanupBlackList: BlackList = BlackList(settings, blackList.filterNot { case (_, (_, banTime, banType)) => - banType != PermanentBan && (System.currentTimeMillis() - banTime.time >= settings.blackList.banTime.toMillis) + banType != PermanentBan && (System.currentTimeMillis() - banTime.time >= settings.banTime.toMillis) }) def remove(peer: InetAddress): BlackList = BlackList(settings, blackList - peer) @@ -59,6 +60,6 @@ object BlackList { final case class BanTime(time: Long) extends AnyVal - def apply(settings: EncryAppSettings): BlackList = + def apply(settings: BlackListSettings): BlackList = BlackList(settings, Map.empty[InetAddress, (BanReason, BanTime, BanType)]) } \ No newline at end of file diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index ea5b777bf2..2133fe0742 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -1,7 +1,11 @@ package encry.network -import akka.actor.Actor +import akka.actor.{Actor, Props} -class DM extends Actor{ +class DM extends Actor { override def receive: Receive = ??? } + +object DM { + def props(): Props = ??? +} diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 1dd6b4ea0c..5efffac15a 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -1,7 +1,32 @@ package encry.network -import akka.actor.Actor +import akka.actor.{Actor, ActorRef, Props} +import akka.pattern._ +import akka.util.Timeout +import encry.network.MessageBuilder.GetPeers +import encry.network.Messages.MessageToNetwork +import encry.network.Messages.MessageToNetwork.{RequestFromLocal, SendSyncInfo} -class MessageBuilder extends Actor { - override def receive: Receive = ??? +import scala.concurrent.duration._ + +case class MessageBuilder(msg: MessageToNetwork, peersKeeper: ActorRef) extends Actor { + + import context.dispatcher + + implicit val timeout: Timeout = Timeout(10 seconds) + + override def receive: Receive = { + case RequestFromLocal(peer, modTypeId, modsIds) => + case SendSyncInfo(syncInfo) => + (peersKeeper ? GetPeers).mapTo[List[ActorRef]].map { peers => + peers.foreach(_ ! syncInfo) + } + } } + +object MessageBuilder { + + case object GetPeers + + def props(msg: MessageToNetwork, peersKeeper: ActorRef): Props = Props(new MessageBuilder(msg, peersKeeper)) +} \ No newline at end of file diff --git a/src/main/scala/encry/network/Messages.scala b/src/main/scala/encry/network/Messages.scala new file mode 100644 index 0000000000..193e887081 --- /dev/null +++ b/src/main/scala/encry/network/Messages.scala @@ -0,0 +1,16 @@ +package encry.network + +import encry.network.PeerConnectionHandler.ConnectedPeer +import org.encryfoundation.common.network.SyncInfo +import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} + +object Messages { + + sealed trait MessageToNetwork + object MessageToNetwork { + final case class RequestFromLocal(source: ConnectedPeer, + modifierTypeId: ModifierTypeId, + modifierIds: Seq[ModifierId]) extends MessageToNetwork + final case class SendSyncInfo(syncInfo: SyncInfo) extends MessageToNetwork + } +} diff --git a/src/main/scala/encry/network/NetworkController.scala b/src/main/scala/encry/network/NetworkController.scala index 983f5daf53..af2d242cac 100755 --- a/src/main/scala/encry/network/NetworkController.scala +++ b/src/main/scala/encry/network/NetworkController.scala @@ -15,7 +15,6 @@ import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeersKeeper._ import encry.settings.{EncryAppSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage - import scala.collection.JavaConverters._ import scala.concurrent.duration._ import scala.language.{existentials, postfixOps} diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 5885fd127e..a1ba21cf58 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -1,7 +1,113 @@ package encry.network -import akka.actor.Actor +import java.net.InetSocketAddress -class NetworkRouter extends Actor { - override def receive: Receive = ??? +import akka.actor.{Actor, ActorRef, Props} +import akka.io.Tcp.{Bind, Bound, CommandFailed, Connect, Connected} +import akka.io.{IO, Tcp} +import akka.io.Tcp.SO.KeepAlive +import com.typesafe.scalalogging.StrictLogging +import encry.network.BlackList.BanReason.InvalidNetworkMessage +import encry.network.Messages.MessageToNetwork +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} +import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction +import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} +import encry.network.PeersKeeper.{BanPeer, ConnectionStopped, ConnectionVerified, HandshakedDone, OutgoingConnectionFailed, PeerForConnection, VerifyConnection} +import encry.settings.NetworkSettings +import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage + +class NetworkRouter(settings: NetworkSettings) extends Actor with StrictLogging { + + import context.system + + var messagesHandlers: Map[Seq[Byte], ActorRef] = Map.empty + + IO(Tcp) ! Bind(self, settings.bindAddress, options = KeepAlive(true) :: Nil, pullMode = false) + + val peersKeeper = context.system.actorOf(PK.props(), "peersKeeper") + val deliveryManager = context.system.actorOf(DM.props(), "deliveryManager") + val externalSocketAddress: Option[InetSocketAddress] = settings.declaredAddress + + override def receive: Receive = bindingLogic orElse businessLogic orElse { + case RegisterMessagesHandler(types, handler) => + logger.info(s"Registering handlers for ${types.mkString(",")}.") + val ids = types.map(_._1) + messagesHandlers += (ids -> handler) + case CommandFailed(cmd: Tcp.Command) => logger.info(s"Failed to execute: $cmd.") + case msg => logger.warn(s"NetworkController: got something strange $msg.") + } + + def bindingLogic: Receive = { + case Bound(address) => + logger.info(s"Successfully bound to the port ${address.getPort}.") + case CommandFailed(add: Bind) => + logger.info(s"Node can't be bind to the address: ${add.localAddress}.") + context.stop(self) + } + + def businessLogic: Receive = { + case MessageFromNetwork(message, Some(remote)) if message.isValid(settings.syncPacketLength) => + logger.debug(s"Got ${message.messageName} on the NetworkController.") + findHandler(message, message.NetworkMessageTypeID, remote, messagesHandlers) + case MessageFromNetwork(message, Some(remote)) => + peersKeeper ! BanPeer(remote, InvalidNetworkMessage(message.messageName)) + logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") + case msg: MessageToNetwork => context.system.actorOf(MessageBuilder.props(msg), "peersKeeper") + } + + def peersLogic: Receive = { + case PeerForConnection(peer) => + logger.info(s"Network router got new peer for connection: $peer. Trying to set connection with remote...") + IO(Tcp) ! Connect( + peer, + None, + KeepAlive(true) :: Nil, + Some(settings.connectionTimeout), + pullMode = true + ) + + case Connected(remote, localAddress) => + logger.info(s"Network router got 'Connected' message from: $remote. " + + s"Trying to set stable connection with remote... " + + s"Local TCP endpoint is: $localAddress.") + peersKeeper ! VerifyConnection(remote, sender()) + + case ConnectionVerified(remote, remoteConnection, connectionType) => + logger.info(s"Network controller got approvement for stable connection with: $remote. Starting interaction process...") + val peerConnectionHandler: ActorRef = context.actorOf( + PeerConnectionHandler.props(remoteConnection, connectionType, externalSocketAddress, remote, settings) + .withDispatcher("network-dispatcher") + ) + peerConnectionHandler ! StartInteraction + + case HandshakedDone(remote) => + logger.info(s"Network controller got approvement from peer handler about successful handshake. " + + s"Sending to peerKeeper connected peer.") + peersKeeper ! HandshakedDone(remote) + + case ConnectionStopped(peer) => + logger.info(s"Network controller got signal about breaking connection with: $peer. " + + s"Sending to peerKeeper actual information.") + peersKeeper ! ConnectionStopped(peer) + + case CommandFailed(connect: Connect) => + logger.info(s"Failed to connect to: ${connect.remoteAddress}.") + peersKeeper ! OutgoingConnectionFailed(connect.remoteAddress) + } + + private def findHandler(message: NetworkMessage, + messageId: Byte, + remote: ConnectedPeer, + mH: Map[Seq[Byte], ActorRef]): Unit = + mH.find(_._1.contains(messageId)).map(_._2) match { + case Some(handler) => + handler ! DataFromPeer(message, remote) + logger.debug(s"Send message DataFromPeer with ${message.messageName} to $handler.") + case None => logger.info("No handlers found for message: " + message.messageName) + } +} + +object NetworkRouter { + + def props(settings: NetworkSettings): Props = Props(new NetworkRouter(settings)) } diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index db92cc82ad..931768507c 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -268,9 +268,6 @@ object NodeViewSynchronizer { status: encry.consensus.HistoryConsensus.HistoryComparisonResult, extension: Option[Seq[(ModifierTypeId, ModifierId)]]) - final case class RequestFromLocal(source: ConnectedPeer, - modifierTypeId: ModifierTypeId, - modifierIds: Seq[ModifierId]) sealed trait CLIPeer final case class PeerFromCli(address: InetSocketAddress) extends CLIPeer diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 835cdb926e..7adda9e5db 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -1,7 +1,31 @@ package encry.network -import akka.actor.Actor +import java.net.{InetAddress, InetSocketAddress} + +import akka.actor.{Actor, Props} +import encry.settings.{BlackListSettings, NetworkSettings} + +import scala.util.Try + +class PK(networkSettings: NetworkSettings, + blacklistSettings: BlackListSettings) extends Actor { + + var connectedPeers: ConnectedPeersCollection = ConnectedPeersCollection() + + var blackList: BlackList = BlackList(blacklistSettings) + + var knownPeers: Set[InetAddress] = networkSettings.knownPeers + .collect { case peer: InetSocketAddress if !isSelf(peer) => peer.getAddress }.toSet -class PK extends Actor{ override def receive: Receive = ??? + + def isSelf(address: InetSocketAddress): Boolean = Try(address == networkSettings.bindAddress || + networkSettings.declaredAddress.contains(address) || + InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) || + InetAddress.getLoopbackAddress.getAddress.sameElements(address.getAddress.getAddress)).getOrElse(true) +} + +object PK { + def props(networkSettings: NetworkSettings, + blacklistSettings: BlackListSettings): Props = Props(new PK(networkSettings, blacklistSettings)) } diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index eed63d2e9a..9099a4dc59 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -13,6 +13,7 @@ import encry.api.http.DataHolderForApi import encry.consensus.HistoryConsensus.ProgressInfo import encry.local.miner.Miner.{DisableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced +import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.ConnectedPeer import encry.settings.EncryAppSettings From 2395da36128d0daba095e1a6209f990c9c3b8465 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 2 Mar 2020 17:25:02 +0300 Subject: [PATCH 003/177] nvh group frame added --- src/main/resources/application.conf | 9 +++++- .../scala/encry/nvg/IntermediaryNVH.scala | 30 +++++++++++++++++++ .../scala/encry/nvg/ModifiersValidator.scala | 20 +++++++++++++ .../encry/nvg/NetworkMessagesProcessor.scala | 11 +++++++ src/main/scala/encry/nvg/NodeViewHolder.scala | 11 +++++++ 5 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 src/main/scala/encry/nvg/IntermediaryNVH.scala create mode 100644 src/main/scala/encry/nvg/ModifiersValidator.scala create mode 100644 src/main/scala/encry/nvg/NetworkMessagesProcessor.scala create mode 100644 src/main/scala/encry/nvg/NodeViewHolder.scala diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 6718d3b58d..2c97730e8b 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -151,7 +151,14 @@ akka.http { akka.http.routing { verbose-error-messages = on } - +modifiers-validator-router-dispatcher { + executor = "thread-pool-executor" + # allocate exactly 5 threads for this pool + thread-pool-executor { + core-pool-size-min = 5 + core-pool-size-max = 5 + } +} mining-dispatcher { type = Dispatcher executor = "thread-pool-executor" diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala new file mode 100644 index 0000000000..b8292741de --- /dev/null +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -0,0 +1,30 @@ +package encry.nvg + +import akka.actor.{ Actor, ActorRef } +import akka.routing.BalancingPool +import com.typesafe.scalalogging.StrictLogging +import encry.network.DownloadedModifiersValidator.ModifiersForValidating +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.settings.EncryAppSettings + +class IntermediaryNVH(settings: EncryAppSettings) extends Actor with StrictLogging { + + val networkMessagesProcessor: ActorRef = + context.actorOf(NetworkMessagesProcessor.props, name = "Network-messages-processor") + val nodeViewHolder: ActorRef = + context.actorOf(NodeViewHolder.props, name = "Node-view-holder") + val modifiersValidatorRouter: ActorRef = + context.actorOf( + BalancingPool(5) + .props(ModifiersValidator.props(nodeViewHolder, settings)), + name = "Modifiers-validator-router" + ) + + override def receive: Receive = { + case msg @ ModifiersForValidating(_, _, _) => modifiersValidatorRouter ! msg + case msg @ DataFromPeer(_, _) => networkMessagesProcessor ! msg + case _ => + } +} + +object IntermediaryNVH {} diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala new file mode 100644 index 0000000000..a4c57c36e3 --- /dev/null +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -0,0 +1,20 @@ +package encry.nvg + +import akka.actor.{Actor, ActorRef, Props} +import com.typesafe.scalalogging.StrictLogging +import encry.network.DownloadedModifiersValidator.ModifiersForValidating +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.settings.EncryAppSettings +import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage + +class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings) extends Actor with StrictLogging { + override def receive: Receive = { + case ModifiersForValidating(remote, typeId, filteredModifiers) => + case _ => + } +} + +object ModifiersValidator { + def props(nodeViewHolderRef: ActorRef, settings: EncryAppSettings): Props = + Props(new ModifiersValidator(nodeViewHolderRef, settings)) +} diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala new file mode 100644 index 0000000000..6cb0c78b59 --- /dev/null +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -0,0 +1,11 @@ +package encry.nvg + +import akka.actor.{ Actor, Props } + +class NetworkMessagesProcessor extends Actor { + override def receive: Receive = ??? +} + +object NetworkMessagesProcessor { + def props: Props = Props(new NetworkMessagesProcessor) +} diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala new file mode 100644 index 0000000000..2aaed78db1 --- /dev/null +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -0,0 +1,11 @@ +package encry.nvg + +import akka.actor.{ Actor, Props } + +class NodeViewHolder extends Actor { + override def receive: Receive = ??? +} + +object NodeViewHolder { + def props: Props = Props(new NodeViewHolder) +} From 45f7662fc275f6bebce45d5f03c89bc3c6c47cff Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 2 Mar 2020 18:19:13 +0300 Subject: [PATCH 004/177] validator frame added --- .../scala/encry/nvg/IntermediaryNVH.scala | 20 ++++- .../scala/encry/nvg/ModifiersValidator.scala | 90 +++++++++++++++++-- src/main/scala/encry/nvg/NodeViewHolder.scala | 6 +- .../encry/view/history/HistoryReader.scala | 17 ++++ 4 files changed, 123 insertions(+), 10 deletions(-) create mode 100644 src/main/scala/encry/view/history/HistoryReader.scala diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index b8292741de..994fc943dd 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -5,7 +5,11 @@ import akka.routing.BalancingPool import com.typesafe.scalalogging.StrictLogging import encry.network.DownloadedModifiersValidator.ModifiersForValidating import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.nvg.ModifiersValidator.ModifierForValidation +import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.settings.EncryAppSettings +import encry.view.history.HistoryReader +import org.encryfoundation.common.utils.TaggedTypes.ModifierId class IntermediaryNVH(settings: EncryAppSettings) extends Actor with StrictLogging { @@ -20,10 +24,20 @@ class IntermediaryNVH(settings: EncryAppSettings) extends Actor with StrictLoggi name = "Modifiers-validator-router" ) + var historyReader: HistoryReader = HistoryReader.empty + override def receive: Receive = { - case msg @ ModifiersForValidating(_, _, _) => modifiersValidatorRouter ! msg - case msg @ DataFromPeer(_, _) => networkMessagesProcessor ! msg - case _ => + case ModifiersForValidating(remote, typeId, modifiers) => + logger.info(s"Got ${modifiers.size} modifiers of type $typeId for validation.") + modifiers.foreach { + case (id: ModifierId, bytes: Array[Byte]) => + modifiersValidatorRouter ! ModifierForValidation(historyReader, id, typeId, bytes, remote) + } + case msg @ DataFromPeer(_, _) => networkMessagesProcessor ! msg + case UpdateHistoryReader(newReader: HistoryReader) => + historyReader = newReader + networkMessagesProcessor ! newReader + case _ => } } diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index a4c57c36e3..68861400c7 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -1,20 +1,98 @@ package encry.nvg -import akka.actor.{Actor, ActorRef, Props} +import HeaderProto.HeaderProtoMessage +import PayloadProto.PayloadProtoMessage +import akka.actor.{ Actor, ActorRef, Props } +import cats.syntax.either._ import com.typesafe.scalalogging.StrictLogging -import encry.network.DownloadedModifiersValidator.ModifiersForValidating -import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.modifiers.history.{ HeaderUtils, PayloadUtils } +import encry.modifiers.history.HeaderUtils.{ IllegalHeight, PreSemanticValidationException } +import encry.network.BlackList.BanReason.{ PreSemanticInvalidModifier, SyntacticallyInvalidPersistentModifier } +import encry.network.PeerConnectionHandler.ConnectedPeer +import encry.network.PeersKeeper.BanPeer +import encry.nvg.ModifiersValidator.ModifierForValidation import encry.settings.EncryAppSettings -import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage +import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote +import encry.view.history.HistoryReader +import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.history.{ Header, HeaderProtoSerializer, Payload, PayloadProtoSerializer } +import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } + +import scala.util.{ Failure, Try } class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings) extends Actor with StrictLogging { override def receive: Receive = { - case ModifiersForValidating(remote, typeId, filteredModifiers) => - case _ => + case ModifierForValidation(reader, modifierId, modifierTypeId, modifierBytes, remote) => + fromProto(modifierTypeId, modifierBytes) match { + case Left(error) => + case Right(modifier) => + val preSemanticValidation: Either[PreSemanticValidationException, Unit] = + isPreSemanticValid(modifier, reader, settings) + val syntacticValidation: Boolean = + isSyntacticallyValid(modifier, settings.constants.ModifierIdSize) + if (preSemanticValidation.isRight && syntacticValidation) { + logger.debug(s"Modifier ${modifier.encodedId} is valid.") + nodeViewHolderRef ! ModifierFromRemote(modifier) + } else if (!syntacticValidation) { + logger.info(s"Modifier ${modifier.encodedId} is syntactically invalid.") + context.parent ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) + } else { + preSemanticValidation.leftMap { + case IllegalHeight(error) => + logger.info(s"Modifier ${modifier.encodedId} is invalid cause: $error.") + context.parent ! BanPeer(remote, PreSemanticInvalidModifier(error)) + } + } + } + } + + private def isPreSemanticValid( + modifier: PersistentModifier, + historyReader: HistoryReader, + settings: EncryAppSettings + ): Either[PreSemanticValidationException, Unit] = + modifier match { + case header: Header => + val bestHeaderHeight: Int = historyReader.getBestHeaderHeight + Either.cond( + bestHeaderHeight - settings.constants.MaxRollbackDepth <= header.height, + (), + IllegalHeight( + s"Height of received header is ${header.height}. Current best header height is $bestHeaderHeight. " + + s"Max possible received header's height is ${bestHeaderHeight - settings.constants.MaxRollbackDepth}." + ) + ) + case _: Payload => ().asRight[PreSemanticValidationException] + } + + private def fromProto( + modType: ModifierTypeId, + bytes: Array[Byte] + ): Either[Throwable, PersistentModifier] = + Either.fromTry(modType match { + case Header.modifierTypeId => HeaderProtoSerializer.fromProto(HeaderProtoMessage.parseFrom(bytes)) + case Payload.modifierTypeId => PayloadProtoSerializer.fromProto(PayloadProtoMessage.parseFrom(bytes)) + }) + + def isSyntacticallyValid( + modifier: PersistentModifier, + modifierIdSize: Int + ): Boolean = modifier match { + case h: Header => HeaderUtils.syntacticallyValidity(h, modifierIdSize).isSuccess + case p: Payload => PayloadUtils.syntacticallyValidity(p, modifierIdSize).isSuccess } } object ModifiersValidator { + + final case class ModifierForValidation( + historyReader: HistoryReader, + modifierId: ModifierId, + modifierTypeId: ModifierTypeId, + modifierBytes: Array[Byte], + remote: ConnectedPeer + ) + def props(nodeViewHolderRef: ActorRef, settings: EncryAppSettings): Props = Props(new ModifiersValidator(nodeViewHolderRef, settings)) } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 2aaed78db1..4ad1757d18 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -1,11 +1,15 @@ package encry.nvg -import akka.actor.{ Actor, Props } +import akka.actor.{Actor, Props} +import encry.view.history.{History, HistoryReader} class NodeViewHolder extends Actor { override def receive: Receive = ??? } object NodeViewHolder { + + final case class UpdateHistoryReader(history: HistoryReader) extends AnyVal + def props: Props = Props(new NodeViewHolder) } diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala new file mode 100644 index 0000000000..7388d4af38 --- /dev/null +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -0,0 +1,17 @@ +package encry.view.history + +trait HistoryReader { + + def getBestHeaderHeight: Int + +} + +object HistoryReader { + def empty: HistoryReader = new HistoryReader { + def getBestHeaderHeight = 0 + } + + def apply(): HistoryReader = new HistoryReader { + def getBestHeaderHeight = 1 + } +} \ No newline at end of file From 17a01225fda736e57e9306ad49b6ad91f8adcd9b Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 2 Mar 2020 18:27:39 +0300 Subject: [PATCH 005/177] validator frame extended --- .../DownloadedModifiersValidator.scala | 35 ------------------- .../scala/encry/nvg/ModifiersValidator.scala | 18 +++++++--- 2 files changed, 14 insertions(+), 39 deletions(-) diff --git a/src/main/scala/encry/network/DownloadedModifiersValidator.scala b/src/main/scala/encry/network/DownloadedModifiersValidator.scala index 74f79146fb..bfeb7c691b 100644 --- a/src/main/scala/encry/network/DownloadedModifiersValidator.scala +++ b/src/main/scala/encry/network/DownloadedModifiersValidator.scala @@ -37,41 +37,6 @@ class DownloadedModifiersValidator(modifierIdSize: Int, } def workingCycle(history: History): Receive = { - case ModifiersForValidating(remote, typeId, filteredModifiers) if typeId != Transaction.modifierTypeId => - filteredModifiers.foreach { - case (id, bytes) => - ModifiersToNetworkUtils.fromProto(typeId, bytes) match { - case Success(modifier) => - val syntacticValidation: Boolean = ModifiersToNetworkUtils.isSyntacticallyValid(modifier, modifierIdSize) - val preSemanticValidation: Either[HeaderUtils.PreSemanticValidationException, Unit] = - ModifiersToNetworkUtils.isPreSemanticValidation(modifier, history, settings) - if (syntacticValidation && preSemanticValidation.isRight) { - logger.debug( - s"Modifier: ${modifier.encodedId} after testApplicable is correct. " + - s"Sending validated modifier to NodeViewHolder" - ) - influxRef.foreach(_ ! ValidatedModifierFromNetwork(typeId)) - nodeViewHolder ! ModifierFromRemote(modifier) - } else { - logger.info( - s"Modifier with id: ${modifier.encodedId} of type: $typeId invalid cause of:" + - s"isSyntacticallyValid = false or $preSemanticValidation" - ) - if (!syntacticValidation) peersKeeper ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) - else - preSemanticValidation match { - case Left(value) => peersKeeper ! BanPeer(remote, PreSemanticInvalidModifier(value.error)) - case Right(_) => - } - nodeViewSync ! InvalidModifier(id) - } - case Failure(ex) => - peersKeeper ! BanPeer(remote, CorruptedSerializedBytes) - logger.info(s"Received modifier from $remote can't be parsed cause of: ${ex.getMessage}.") - nodeViewSync ! InvalidModifier(id) - } - } - case ModifiersForValidating(remote, typeId, filteredModifiers) => typeId match { case Transaction.modifierTypeId => diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index 68861400c7..f16346b79d 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -5,9 +5,14 @@ import PayloadProto.PayloadProtoMessage import akka.actor.{ Actor, ActorRef, Props } import cats.syntax.either._ import com.typesafe.scalalogging.StrictLogging -import encry.modifiers.history.{ HeaderUtils, PayloadUtils } import encry.modifiers.history.HeaderUtils.{ IllegalHeight, PreSemanticValidationException } -import encry.network.BlackList.BanReason.{ PreSemanticInvalidModifier, SyntacticallyInvalidPersistentModifier } +import encry.modifiers.history.{ HeaderUtils, PayloadUtils } +import encry.network.BlackList.BanReason.{ + CorruptedSerializedBytes, + PreSemanticInvalidModifier, + SyntacticallyInvalidPersistentModifier +} +import encry.network.DownloadedModifiersValidator.InvalidModifier import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.ModifierForValidation @@ -16,15 +21,18 @@ import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Header, HeaderProtoSerializer, Payload, PayloadProtoSerializer } +import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } -import scala.util.{ Failure, Try } - class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings) extends Actor with StrictLogging { + override def receive: Receive = { case ModifierForValidation(reader, modifierId, modifierTypeId, modifierBytes, remote) => fromProto(modifierTypeId, modifierBytes) match { case Left(error) => + logger.info(s"Modifier ${Algos.encode(modifierId)} is incorrect cause: ${error.getMessage}.") + context.parent ! BanPeer(remote, CorruptedSerializedBytes) + context.parent ! InvalidModifier(modifierId) case Right(modifier) => val preSemanticValidation: Either[PreSemanticValidationException, Unit] = isPreSemanticValid(modifier, reader, settings) @@ -36,11 +44,13 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings } else if (!syntacticValidation) { logger.info(s"Modifier ${modifier.encodedId} is syntactically invalid.") context.parent ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) + context.parent ! InvalidModifier(modifierId) } else { preSemanticValidation.leftMap { case IllegalHeight(error) => logger.info(s"Modifier ${modifier.encodedId} is invalid cause: $error.") context.parent ! BanPeer(remote, PreSemanticInvalidModifier(error)) + context.parent ! InvalidModifier(modifierId) } } } From 12a0f5c53e2154cc85362cbf1c8d05b7d4e6a465 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 14:35:24 +0300 Subject: [PATCH 006/177] added inner impl of nvh --- .../scala/encry/nvg/IntermediaryNVH.scala | 9 +- .../scala/encry/nvg/ModifiersValidator.scala | 11 +- src/main/scala/encry/nvg/NodeViewHolder.scala | 462 +++++++++++++++++- .../scala/encry/view/NodeViewHolder.scala | 324 ------------ 4 files changed, 467 insertions(+), 339 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 994fc943dd..f1af14f4b0 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -3,15 +3,17 @@ package encry.nvg import akka.actor.{ Actor, ActorRef } import akka.routing.BalancingPool import com.typesafe.scalalogging.StrictLogging -import encry.network.DownloadedModifiersValidator.ModifiersForValidating +import encry.network.BlackList.BanReason.CorruptedSerializedBytes +import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.ModifierForValidation import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.settings.EncryAppSettings import encry.view.history.HistoryReader import org.encryfoundation.common.utils.TaggedTypes.ModifierId -class IntermediaryNVH(settings: EncryAppSettings) extends Actor with StrictLogging { +class IntermediaryNVH(settings: EncryAppSettings, intermediaryNetwork: ActorRef) extends Actor with StrictLogging { val networkMessagesProcessor: ActorRef = context.actorOf(NetworkMessagesProcessor.props, name = "Network-messages-processor") @@ -37,7 +39,8 @@ class IntermediaryNVH(settings: EncryAppSettings) extends Actor with StrictLoggi case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! newReader - case _ => + case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg + case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg } } diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index f16346b79d..512620a119 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -45,14 +45,13 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings logger.info(s"Modifier ${modifier.encodedId} is syntactically invalid.") context.parent ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) context.parent ! InvalidModifier(modifierId) - } else { + } else preSemanticValidation.leftMap { case IllegalHeight(error) => logger.info(s"Modifier ${modifier.encodedId} is invalid cause: $error.") context.parent ! BanPeer(remote, PreSemanticInvalidModifier(error)) context.parent ! InvalidModifier(modifierId) } - } } } @@ -64,14 +63,12 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings modifier match { case header: Header => val bestHeaderHeight: Int = historyReader.getBestHeaderHeight - Either.cond( - bestHeaderHeight - settings.constants.MaxRollbackDepth <= header.height, - (), + if (bestHeaderHeight - settings.constants.MaxRollbackDepth <= header.height) ().asRight + else IllegalHeight( s"Height of received header is ${header.height}. Current best header height is $bestHeaderHeight. " + s"Max possible received header's height is ${bestHeaderHeight - settings.constants.MaxRollbackDepth}." - ) - ) + ).asLeft case _: Payload => ().asRight[PreSemanticValidationException] } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 4ad1757d18..f83295b81d 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -1,15 +1,467 @@ package encry.nvg -import akka.actor.{Actor, Props} -import encry.view.history.{History, HistoryReader} +import java.io.File -class NodeViewHolder extends Actor { - override def receive: Receive = ??? +import akka.actor.{ Actor, ActorRef, Props } +import com.typesafe.scalalogging.StrictLogging +import encry.EncryApp +import encry.EncryApp.system +import encry.api.http.DataHolderForApi +import encry.consensus.HistoryConsensus.ProgressInfo +import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.network.DeliveryManager.FullBlockChainIsSynced +import encry.network.NodeViewSynchronizer.ReceivableMessages.{ + ChangedHistory, + ChangedState, + RollbackFailed, + RollbackSucceed, + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification +} +import encry.nvg.NodeViewHolder.NodeView +import encry.settings.EncryAppSettings +import encry.stats.StatsSender.{ + BestHeaderInChain, + EndOfApplyingModifier, + HeightStatistics, + ModifierAppendedToHistory, + ModifierAppendedToState, + StartApplyingModifier, + StateUpdating, + TransactionsInBlock +} +import encry.utils.CoreTaggedTypes.VersionTag +import encry.utils.NetworkTimeProvider +import encry.view.ModifiersCache +import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError +import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote +import encry.view.NodeViewHolder.{ DownloadRequest, UpdateInformation } +import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId +import encry.view.fast.sync.SnapshotHolder.{ + HeaderChainIsSynced, + RequiredManifestHeightAndId, + SnapshotChunk, + TreeChunks +} +import encry.view.history.{ History, HistoryReader } +import encry.view.mempool.MemoryPool.RolledBackTransactions +import encry.view.state.UtxoState +import encry.view.state.avlTree.AvlTree +import encry.view.wallet.EncryWallet +import io.iohk.iodb.ByteArrayWrapper +import org.apache.commons.io.FileUtils +import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId } + +import scala.collection.{ mutable, IndexedSeq, Seq } +import scala.concurrent.ExecutionContextExecutor +import scala.util.{ Failure, Success, Try } + +class NodeViewHolder( + settings: EncryAppSettings, + timeProvider: NetworkTimeProvider, + influxRef: Option[ActorRef] +) extends Actor + with StrictLogging + with AutoCloseable { + + import context.dispatcher + + var nodeView: NodeView = restoreState().getOrElse(genesisState) + + var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] + + override def receive: Receive = { + case ModifierFromRemote(modifier) => + } + + //todo refactor loop + def computeApplications(): Unit = { + val mods = ModifiersCache.popCandidate(nodeView.history) + if (mods.nonEmpty) { + logger.info(s"mods: ${mods.map(mod => Algos.encode(mod.id))}") + mods.foreach(mod => pmodModify(mod)) + computeApplications() + } else Unit + } + + //todo replace with reader + def updateNodeView( + updatedHistory: Option[History] = None, + updatedState: Option[UtxoState] = None, + updatedVault: Option[EncryWallet] = None + ): Unit = { + val newNodeView: NodeView = NodeView( + updatedHistory.getOrElse(nodeView.history), + updatedState.getOrElse(nodeView.state), + updatedVault.getOrElse(nodeView.wallet) + ) + if (updatedHistory.nonEmpty) context.parent ! ChangedHistory(newNodeView.history) + if (updatedState.nonEmpty) context.parent ! ChangedState(newNodeView.state) + nodeView = newNodeView + } + + def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = None): Unit = + pi.toDownload.foreach { + case (tid, id) => + if (tid != Transaction.modifierTypeId) + logger.info( + s"NVH trigger sending DownloadRequest to NVSH with type: $tid " + + s"for modifier: ${Algos.encode(id)}. PrevMod is: ${previousModifier.map(Algos.encode)}." + ) + if ((nodeView.history.isFullChainSynced && tid == Payload.modifierTypeId) || tid != Payload.modifierTypeId) + system.actorSelection("/user/nodeViewSynchronizer") ! DownloadRequest(tid, id, previousModifier) + else + logger.info( + s"Ignore sending request for payload (${Algos.encode(id)}) from nvh because of nodeView.history.isFullChainSynced = false" + ) + } + + def trimChainSuffix(suffix: IndexedSeq[PersistentModifier], + rollbackPoint: ModifierId): IndexedSeq[PersistentModifier] = { + val idx: Int = suffix.indexWhere(_.id.sameElements(rollbackPoint)) + if (idx == -1) IndexedSeq() else suffix.drop(idx) + } + + @scala.annotation.tailrec + private def updateState(history: History, + state: UtxoState, + progressInfo: ProgressInfo, + suffixApplied: IndexedSeq[PersistentModifier], + isLocallyGenerated: Boolean = false): (History, UtxoState, Seq[PersistentModifier]) = { + logger.info(s"\nStarting updating state in updateState function!") + if (!isLocallyGenerated) progressInfo.toApply.foreach { + case header: Header => requestDownloads(progressInfo, Some(header.id)) + case _ => requestDownloads(progressInfo, None) + } + val branchingPointOpt: Option[VersionTag] = progressInfo.branchPoint.map(VersionTag !@@ _) + val (stateToApplyTry: Try[UtxoState], suffixTrimmed: IndexedSeq[PersistentModifier] @unchecked) = + if (progressInfo.chainSwitchingNeeded) { + branchingPointOpt.map { branchPoint => + if (!state.version.sameElements(branchPoint)) { + val branchPointHeight = history.getHeaderById(ModifierId !@@ branchPoint).get.height + val additionalBlocks = (state.safePointHeight + 1 to branchPointHeight).foldLeft(List.empty[Block]) { + case (blocks, height) => + val headerAtHeight = history.getBestHeaderAtHeight(height).get + val blockAtHeight = history.getBlockByHeader(headerAtHeight).get + blocks :+ blockAtHeight + } + context.system.actorSelection("/user/miner") ! DisableMining + state.rollbackTo(branchPoint, additionalBlocks) -> trimChainSuffix(suffixApplied, + ModifierId !@@ branchPoint) + } else Success(state) -> IndexedSeq() + }.getOrElse(Failure(new Exception("Trying to rollback when branchPoint is empty."))) + } else Success(state) -> suffixApplied + stateToApplyTry match { + case Success(stateToApply) => + context.system.eventStream.publish(RollbackSucceed(branchingPointOpt)) + val u0: UpdateInformation = UpdateInformation(history, stateToApply, None, None, suffixTrimmed) + val uf: UpdateInformation = progressInfo.toApply.foldLeft(u0) { + case (u, modToApply) => + val saveRootNodesFlag = (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2 + if (u.failedMod.isEmpty) u.state.applyModifier(modToApply, saveRootNodesFlag) match { + case Right(stateAfterApply) => + influxRef.foreach( + ref => + modToApply match { + case b: Block if history.isFullChainSynced => ref ! TransactionsInBlock(b.payload.txs.size) + case _ => + } + ) + val newHis: History = history.reportModifierIsValid(modToApply) + //dataHolder ! DataHolderForApi.BlockAndHeaderInfo(newHis.getBestHeader, newHis.getBestBlock) + modToApply match { + case header: Header => + val requiredHeight: Int = header.height - settings.constants.MaxRollbackDepth + if (requiredHeight % settings.constants.SnapshotCreationHeight == 0) { + newHis.lastAvailableManifestHeight = requiredHeight + logger.info(s"heightOfLastAvailablePayloadForRequest -> ${newHis.lastAvailableManifestHeight}") + } + case _ => + } + newHis.getHeaderOfBestBlock.foreach { header: Header => + val potentialManifestId: Array[Byte] = Algos.hash(stateAfterApply.tree.rootHash ++ header.id) + val isManifestExists: Boolean = potentialManifestIds.exists(_.sameElements(potentialManifestId)) + val isCorrectCreationHeight: Boolean = + header.height % settings.constants.SnapshotCreationHeight == 0 + val isGenesisHeader: Boolean = header.height == settings.constants.GenesisHeight + if (settings.snapshotSettings.enableSnapshotCreation && newHis.isFullChainSynced && + !isManifestExists && isCorrectCreationHeight && !isGenesisHeader) { + val startTime = System.currentTimeMillis() + logger.info(s"Start chunks creation for new snapshot") + import encry.view.state.avlTree.utils.implicits.Instances._ + val chunks: List[SnapshotChunk] = + AvlTree.getChunks( + stateAfterApply.tree.rootNode, + currentChunkHeight = settings.snapshotSettings.chunkDepth, + stateAfterApply.tree.avlStorage + ) + system.actorSelection("/user/nodeViewSynchronizer") ! TreeChunks(chunks, potentialManifestId) + potentialManifestIds = ManifestId @@ potentialManifestId :: potentialManifestIds + logger.info( + s"State tree successfully processed for snapshot. " + + s"Processing time is: ${(System.currentTimeMillis() - startTime) / 1000}s." + ) + } + } + if (settings.node.mining && progressInfo.chainSwitchingNeeded) + context.system.actorSelection("/user/miner") ! StartMining + context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) + if (newHis.getBestHeaderId.exists( + bestHeaderId => + newHis.getBestBlockId.exists(bId => ByteArrayWrapper(bId) == ByteArrayWrapper(bestHeaderId)) + )) newHis.isFullChainSynced = true + influxRef.foreach { ref => + logger.info(s"send info 2. about ${newHis.getBestHeaderHeight} | ${newHis.getBestBlockHeight}") + ref ! HeightStatistics(newHis.getBestHeaderHeight, stateAfterApply.height) + val isBlock: Boolean = modToApply match { + case _: Block => true + case _: Payload => true + case _ => false + } + if (isBlock) ref ! ModifierAppendedToState(success = true) + } + UpdateInformation(newHis, stateAfterApply, None, None, u.suffix :+ modToApply) + case Left(e) => + logger.info(s"Application to state failed cause $e") + val (newHis: History, newProgressInfo: ProgressInfo) = + history.reportModifierIsInvalid(modToApply) + context.system.eventStream.publish(SemanticallyFailedModification(modToApply, e)) + UpdateInformation(newHis, u.state, Some(modToApply), Some(newProgressInfo), u.suffix) + } else u + } + uf.failedMod match { + case Some(_) => + uf.history.updateIdsForSyncInfo() + updateState(uf.history, uf.state, uf.alternativeProgressInfo.get, uf.suffix, isLocallyGenerated) + case None => (uf.history, uf.state, uf.suffix) + } + case Failure(e) => + context.system.eventStream.publish(RollbackFailed(branchingPointOpt)) + EncryApp.forceStopApplication(500, s"Rollback failed: $e") + } + } + + def pmodModify(pmod: PersistentModifier, isLocallyGenerated: Boolean = false): Unit = + if (!nodeView.history.isModifierDefined(pmod.id)) { + logger.debug( + s"\nStarting to apply modifier ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to history." + ) + val startAppHistory = System.currentTimeMillis() + if (settings.influxDB.isDefined) + context.system + .actorSelection("user/statsSender") ! + StartApplyingModifier(pmod.id, pmod.modifierTypeId, System.currentTimeMillis()) + nodeView.history.append(pmod) match { + case Right((historyBeforeStUpdate, progressInfo)) => + logger.info( + s"Successfully applied modifier ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to history." + ) + logger.debug( + s"Time of applying to history SUCCESS is: ${System.currentTimeMillis() - startAppHistory}. modId is: ${pmod.encodedId}" + ) + if (pmod.modifierTypeId == Header.modifierTypeId) historyBeforeStUpdate.updateIdsForSyncInfo() + influxRef.foreach { ref => + ref ! EndOfApplyingModifier(pmod.id) + val isHeader: Boolean = pmod match { + case _: Header => true + case _: Payload => false + } + ref ! ModifierAppendedToHistory(isHeader, success = true) + } + if (historyBeforeStUpdate.fastSyncInProgress.fastSyncVal && pmod.modifierTypeId == Payload.modifierTypeId && + historyBeforeStUpdate.getBestBlockHeight >= historyBeforeStUpdate.lastAvailableManifestHeight) { + logger.info(s"nodeView.history.getBestBlockHeight ${historyBeforeStUpdate.getBestBlockHeight}") + logger.info( + s"nodeView.history.heightOfLastAvailablePayloadForRequest ${historyBeforeStUpdate.lastAvailableManifestHeight}" + ) + historyBeforeStUpdate.getBestHeaderAtHeight(historyBeforeStUpdate.lastAvailableManifestHeight).foreach { + h => + system.actorSelection("/user/nodeViewSynchronizer") ! RequiredManifestHeightAndId( + historyBeforeStUpdate.lastAvailableManifestHeight, + Algos.hash(h.stateRoot ++ h.id) + ) + } + } + logger.info( + s"Going to apply modifications ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to the state: $progressInfo" + ) + if (progressInfo.toApply.nonEmpty) { + logger.info( + s"\n progress info non empty. To apply: ${progressInfo.toApply.map(mod => Algos.encode(mod.id))}" + ) + val startPoint: Long = System.currentTimeMillis() + val (newHistory: History, newState: UtxoState, blocksApplied: Seq[PersistentModifier]) = + updateState(historyBeforeStUpdate, nodeView.state, progressInfo, IndexedSeq(), isLocallyGenerated) + if (newHistory.isHeadersChainSynced) + system.actorSelection("/user/nodeViewSynchronizer") ! HeaderChainIsSynced + if (settings.influxDB.isDefined) + context.actorSelection("/user/statsSender") ! StateUpdating(System.currentTimeMillis() - startPoint) + sendUpdatedInfoToMemoryPool(progressInfo.toRemove) + if (progressInfo.chainSwitchingNeeded) + nodeView.wallet.rollback(VersionTag !@@ progressInfo.branchPoint.get).get + blocksApplied.foreach(nodeView.wallet.scanPersistent) + logger.debug(s"\nPersistent modifier ${pmod.encodedId} applied successfully") + if (settings.influxDB.isDefined) + newHistory.getBestHeader.foreach( + header => context.actorSelection("/user/statsSender") ! BestHeaderInChain(header) + ) + if (newHistory.isFullChainSynced) { + logger.debug(s"\nblockchain is synced on nvh on height ${newHistory.getBestHeaderHeight}!") + ModifiersCache.setChainSynced() + system.actorSelection("/user/nodeViewSynchronizer") ! FullBlockChainIsSynced + system.actorSelection("/user/miner") ! FullBlockChainIsSynced + } + updateNodeView(Some(newHistory), Some(newState), Some(nodeView.wallet)) + } else { + influxRef.foreach { ref => + logger.info( + s"send info 3. about ${historyBeforeStUpdate.getBestHeaderHeight} | ${historyBeforeStUpdate.getBestBlockHeight}" + ) + ref ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height) + } + if (!isLocallyGenerated) requestDownloads(progressInfo, Some(pmod.id)) + context.system.eventStream.publish(SemanticallySuccessfulModifier(pmod)) + logger.info(s"\nProgress info is empty") + updateNodeView(updatedHistory = Some(historyBeforeStUpdate)) + } + case Left(e) => + logger.debug( + s"\nCan`t apply persistent modifier (id: ${pmod.encodedId}, contents: $pmod)" + + s" to history caused $e" + ) + context.system.eventStream + .publish(SyntacticallyFailedModification(pmod, List(HistoryApplyError(e.getMessage)))) + } + } else logger.info(s"\nTrying to apply modifier ${pmod.encodedId} that's already in history.") + + def key(id: ModifierId): mutable.WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) + + def sendUpdatedInfoToMemoryPool(toRemove: Seq[PersistentModifier]): Unit = { + val rolledBackTxs: IndexedSeq[Transaction] = toRemove + .flatMap(extractTransactions) + .toIndexedSeq + //if (rolledBackTxs.nonEmpty) memoryPoolRef ! RolledBackTransactions(rolledBackTxs) + } + + def extractTransactions(mod: PersistentModifier): Seq[Transaction] = mod match { + case b: Block => b.payload.txs + case p: Payload => p.txs + case _ => Seq.empty[Transaction] + } + + def genesisState: NodeView = { + val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdir() + val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() + assert(stateDir.listFiles().isEmpty, s"Genesis directory $stateDir should always be empty.") + val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) + val history: History = History.readOrGenerate(settings, timeProvider) + val wallet: EncryWallet = + EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) + NodeView(history, state, wallet) + } + + def restoreState(influxRef: Option[ActorRef] = None): Option[NodeView] = + if (History.getHistoryIndexDir(settings).listFiles.nonEmpty) + try { + val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdirs() + val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() + val history: History = History.readOrGenerate(settings, timeProvider) + val wallet: EncryWallet = + EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) + val state: UtxoState = restoreConsistentState( + UtxoState.create(stateDir, rootsDir, settings, influxRef), + history, + influxRef + ) + history.updateIdsForSyncInfo() + logger.info(s"History best block height: ${history.getBestBlockHeight}") + logger.info(s"History best header height: ${history.getBestHeaderHeight}") + Some(NodeView(history, state, wallet)) + } catch { + case ex: Throwable => + logger.info(s"${ex.getMessage} during state restore. Recover from Modifiers holder!") + new File(settings.directory).listFiles.foreach(dir => FileUtils.cleanDirectory(dir)) + Some(genesisState) + } else { + None + } + + def getRecreatedState( + version: Option[VersionTag] = None, + digest: Option[ADDigest] = None, + influxRef: Option[ActorRef] + ): UtxoState = { + val dir: File = UtxoState.getStateDir(settings) + dir.mkdirs() + dir.listFiles.foreach(_.delete()) + val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdirs() + val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() + UtxoState.create(stateDir, rootsDir, settings, influxRef) + } + + def restoreConsistentState( + stateIn: UtxoState, + history: History, + influxRefActor: Option[ActorRef] + ): UtxoState = + (stateIn.version, history.getBestBlock, stateIn, stateIn.safePointHeight) match { + case (stateId, None, _, _) if stateId sameElements Array.emptyByteArray => + logger.info(s"State and history are both empty on startup") + stateIn + case (_, None, _, _) => + logger.info( + s"State and history are inconsistent." + + s" History is empty on startup, rollback state to genesis." + ) + getRecreatedState(influxRef = influxRefActor) + case (_, Some(historyBestBlock), state: UtxoState, safePointHeight) => + val headerAtSafePointHeight = history.getBestHeaderAtHeight(safePointHeight) + val (rollbackId, newChain) = history.getChainToHeader(headerAtSafePointHeight, historyBestBlock.header) + logger.info( + s"State and history are inconsistent." + + s" Going to rollback to ${rollbackId.map(Algos.encode)} and " + + s"apply ${newChain.length} modifiers" + ) + val additionalBlocks = + (state.safePointHeight + 1 to historyBestBlock.header.height).foldLeft(List.empty[Block]) { + case (blocks, height) => + val headerAtHeight = history.getBestHeaderAtHeight(height).get + val blockAtHeight = history.getBlockByHeader(headerAtHeight).get + blocks :+ blockAtHeight + } + logger.info(s"Qty of additional blocks: ${additionalBlocks.length}") + rollbackId + .map(_ => state.restore(additionalBlocks).get) + .getOrElse(getRecreatedState(influxRef = influxRefActor)) + } + + override def close(): Unit = { + nodeView.history.close() + nodeView.state.close() + nodeView.wallet.close() + } } object NodeViewHolder { final case class UpdateHistoryReader(history: HistoryReader) extends AnyVal - def props: Props = Props(new NodeViewHolder) + final case class NodeView(history: History, state: UtxoState, wallet: EncryWallet) + + def props( + settings: EncryAppSettings, + timeProvider: NetworkTimeProvider, + influxRef: Option[ActorRef] + ): Props = Props(new NodeViewHolder(settings, timeProvider, influxRef)) } diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index 9099a4dc59..404f9a52ae 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -51,9 +51,6 @@ class NodeViewHolder(memoryPoolRef: ActorRef, implicit val exCon: ExecutionContextExecutor = context.dispatcher - case class NodeView(history: History, state: UtxoState, wallet: EncryWallet) - - var nodeView: NodeView = restoreState().getOrElse(genesisState(influxRef)) context.system.actorSelection("/user/nodeViewSynchronizer") ! ChangedHistory(nodeView.history) dataHolder ! UpdatedHistory(nodeView.history) @@ -167,327 +164,6 @@ class NodeViewHolder(memoryPoolRef: ActorRef, case msg => logger.error(s"Got strange message on nvh: $msg") } - //todo refactor loop - def computeApplications(): Unit = { - val mods = ModifiersCache.popCandidate(nodeView.history) - if (mods.nonEmpty) { - logger.info(s"mods: ${mods.map(mod => Algos.encode(mod.id))}") - mods.foreach(mod => pmodModify(mod)) - computeApplications() - } - else Unit - } - - def key(id: ModifierId): mutable.WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) - - def updateNodeView(updatedHistory: Option[History] = None, - updatedState: Option[UtxoState] = None, - updatedVault: Option[EncryWallet] = None): Unit = { - val newNodeView: NodeView = NodeView(updatedHistory.getOrElse(nodeView.history), - updatedState.getOrElse(nodeView.state), - updatedVault.getOrElse(nodeView.wallet)) - if (updatedHistory.nonEmpty) { - system.actorSelection("/user/nodeViewSynchronizer") ! ChangedHistory(newNodeView.history) - context.system.eventStream.publish(ChangedHistory(newNodeView.history)) - } - if (updatedState.nonEmpty) context.system.eventStream.publish(ChangedState(newNodeView.state)) - nodeView = newNodeView - } - - def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = None): Unit = - pi.toDownload.foreach { case (tid, id) => - if (tid != Transaction.modifierTypeId) logger.info(s"NVH trigger sending DownloadRequest to NVSH with type: $tid " + - s"for modifier: ${Algos.encode(id)}. PrevMod is: ${previousModifier.map(Algos.encode)}.") - if ((nodeView.history.isFullChainSynced && tid == Payload.modifierTypeId) || tid != Payload.modifierTypeId) - system.actorSelection("/user/nodeViewSynchronizer")! DownloadRequest(tid, id, previousModifier) - else logger.info(s"Ignore sending request for payload (${Algos.encode(id)}) from nvh because of nodeView.history.isFullChainSynced = false") - } - - def trimChainSuffix(suffix: IndexedSeq[PersistentModifier], rollbackPoint: ModifierId): - IndexedSeq[PersistentModifier] = { - val idx: Int = suffix.indexWhere(_.id.sameElements(rollbackPoint)) - if (idx == -1) IndexedSeq() else suffix.drop(idx) - } - - @scala.annotation.tailrec - private def updateState(history: History, - state: UtxoState, - progressInfo: ProgressInfo, - suffixApplied: IndexedSeq[PersistentModifier], - isLocallyGenerated: Boolean = false): - (History, UtxoState, Seq[PersistentModifier]) = { - logger.info(s"\nStarting updating state in updateState function!") - if (!isLocallyGenerated) progressInfo.toApply.foreach { - case header: Header => requestDownloads(progressInfo, Some(header.id)) - case _ => requestDownloads(progressInfo, None) - } - val branchingPointOpt: Option[VersionTag] = progressInfo.branchPoint.map(VersionTag !@@ _) - val (stateToApplyTry: Try[UtxoState], suffixTrimmed: IndexedSeq[PersistentModifier]@unchecked) = - if (progressInfo.chainSwitchingNeeded) { - branchingPointOpt.map { branchPoint => - if (!state.version.sameElements(branchPoint)) { - val branchPointHeight = history.getHeaderById(ModifierId !@@ branchPoint).get.height - val additionalBlocks = (state.safePointHeight + 1 to branchPointHeight).foldLeft(List.empty[Block]){ - case (blocks, height) => - val headerAtHeight = history.getBestHeaderAtHeight(height).get - val blockAtHeight = history.getBlockByHeader(headerAtHeight).get - blocks :+ blockAtHeight - } - context.system.actorSelection("/user/miner") ! DisableMining - state.rollbackTo(branchPoint, additionalBlocks) -> trimChainSuffix(suffixApplied, ModifierId !@@ branchPoint) - } else Success(state) -> IndexedSeq() - }.getOrElse(Failure(new Exception("Trying to rollback when branchPoint is empty."))) - } else Success(state) -> suffixApplied - stateToApplyTry match { - case Success(stateToApply) => - context.system.eventStream.publish(RollbackSucceed(branchingPointOpt)) - val u0: UpdateInformation = UpdateInformation(history, stateToApply, None, None, suffixTrimmed) - val uf: UpdateInformation = progressInfo.toApply.foldLeft(u0) { case (u, modToApply) => - val saveRootNodesFlag = (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < encrySettings.constants.MaxRollbackDepth * 2 - if (u.failedMod.isEmpty) u.state.applyModifier(modToApply, saveRootNodesFlag) match { - case Right(stateAfterApply) => - influxRef.foreach(ref => modToApply match { - case b: Block if history.isFullChainSynced => ref ! TransactionsInBlock(b.payload.txs.size) - case _ => - }) - val newHis: History = history.reportModifierIsValid(modToApply) - dataHolder ! DataHolderForApi.BlockAndHeaderInfo(newHis.getBestHeader, newHis.getBestBlock) - modToApply match { - case header: Header => - val requiredHeight: Int = header.height - encrySettings.constants.MaxRollbackDepth - if (requiredHeight % encrySettings.constants.SnapshotCreationHeight == 0) { - newHis.lastAvailableManifestHeight = requiredHeight - logger.info(s"heightOfLastAvailablePayloadForRequest -> ${newHis.lastAvailableManifestHeight}") - } - case _ => - } - newHis.getHeaderOfBestBlock.foreach { header: Header => - val potentialManifestId: Array[Byte] = Algos.hash(stateAfterApply.tree.rootHash ++ header.id) - val isManifestExists: Boolean = potentialManifestIds.exists(_.sameElements(potentialManifestId)) - val isCorrectCreationHeight: Boolean = - header.height % encrySettings.constants.SnapshotCreationHeight == 0 - val isGenesisHeader: Boolean = header.height == encrySettings.constants.GenesisHeight - if (encrySettings.snapshotSettings.enableSnapshotCreation && newHis.isFullChainSynced && - !isManifestExists && isCorrectCreationHeight && !isGenesisHeader) { - val startTime = System.currentTimeMillis() - logger.info(s"Start chunks creation for new snapshot") - import encry.view.state.avlTree.utils.implicits.Instances._ - val chunks: List[SnapshotChunk] = - AvlTree.getChunks( - stateAfterApply.tree.rootNode, - currentChunkHeight = encrySettings.snapshotSettings.chunkDepth, - stateAfterApply.tree.avlStorage - ) - system.actorSelection("/user/nodeViewSynchronizer") ! TreeChunks(chunks, potentialManifestId) - potentialManifestIds = ManifestId @@ potentialManifestId :: potentialManifestIds - logger.info(s"State tree successfully processed for snapshot. " + - s"Processing time is: ${(System.currentTimeMillis() - startTime) / 1000}s.") - } - } - if (encrySettings.node.mining && progressInfo.chainSwitchingNeeded) context.system.actorSelection("/user/miner") ! StartMining - context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) - if (newHis.getBestHeaderId.exists(bestHeaderId => - newHis.getBestBlockId.exists(bId => ByteArrayWrapper(bId) == ByteArrayWrapper(bestHeaderId)) - )) newHis.isFullChainSynced = true - influxRef.foreach { ref => - logger.info(s"send info 2. about ${newHis.getBestHeaderHeight} | ${newHis.getBestBlockHeight}") - ref ! HeightStatistics(newHis.getBestHeaderHeight, stateAfterApply.height) - val isBlock: Boolean = modToApply match { - case _: Block => true - case _: Payload => true - case _ => false - } - if (isBlock) ref ! ModifierAppendedToState(success = true) - } - UpdateInformation(newHis, stateAfterApply, None, None, u.suffix :+ modToApply) - case Left(e) => - logger.info(s"Application to state failed cause $e") - val (newHis: History, newProgressInfo: ProgressInfo) = - history.reportModifierIsInvalid(modToApply) - context.system.eventStream.publish(SemanticallyFailedModification(modToApply, e)) - UpdateInformation(newHis, u.state, Some(modToApply), Some(newProgressInfo), u.suffix) - } else u - } - uf.failedMod match { - case Some(_) => - uf.history.updateIdsForSyncInfo() - updateState(uf.history, uf.state, uf.alternativeProgressInfo.get, uf.suffix, isLocallyGenerated) - case None => (uf.history, uf.state, uf.suffix) - } - case Failure(e) => - context.system.eventStream.publish(RollbackFailed(branchingPointOpt)) - EncryApp.forceStopApplication(500, s"Rollback failed: $e") - } - } - - def pmodModify(pmod: PersistentModifier, isLocallyGenerated: Boolean = false): Unit = - if (!nodeView.history.isModifierDefined(pmod.id)) { - logger.debug(s"\nStarting to apply modifier ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to history.") - val startAppHistory = System.currentTimeMillis() - if (encrySettings.influxDB.isDefined) context.system - .actorSelection("user/statsSender") ! - StartApplyingModifier(pmod.id, pmod.modifierTypeId, System.currentTimeMillis()) - nodeView.history.append(pmod) match { - case Right((historyBeforeStUpdate, progressInfo)) => - logger.info(s"Successfully applied modifier ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to history.") - logger.debug(s"Time of applying to history SUCCESS is: ${System.currentTimeMillis() - startAppHistory}. modId is: ${pmod.encodedId}") - if (pmod.modifierTypeId == Header.modifierTypeId) historyBeforeStUpdate.updateIdsForSyncInfo() - influxRef.foreach { ref => - ref ! EndOfApplyingModifier(pmod.id) - val isHeader: Boolean = pmod match { - case _: Header => true - case _: Payload => false - } - ref ! ModifierAppendedToHistory(isHeader, success = true) - } - if (historyBeforeStUpdate.fastSyncInProgress.fastSyncVal && pmod.modifierTypeId == Payload.modifierTypeId && - historyBeforeStUpdate.getBestBlockHeight >= historyBeforeStUpdate.lastAvailableManifestHeight) { - logger.info(s"nodeView.history.getBestBlockHeight ${historyBeforeStUpdate.getBestBlockHeight}") - logger.info(s"nodeView.history.heightOfLastAvailablePayloadForRequest ${historyBeforeStUpdate.lastAvailableManifestHeight}") - historyBeforeStUpdate.getBestHeaderAtHeight(historyBeforeStUpdate.lastAvailableManifestHeight) - .foreach { h => - system.actorSelection("/user/nodeViewSynchronizer") ! RequiredManifestHeightAndId( - historyBeforeStUpdate.lastAvailableManifestHeight, - Algos.hash(h.stateRoot ++ h.id) - ) - } - } - logger.info(s"Going to apply modifications ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to the state: $progressInfo") - if (progressInfo.toApply.nonEmpty) { - logger.info(s"\n progress info non empty. To apply: ${progressInfo.toApply.map(mod => Algos.encode(mod.id))}") - val startPoint: Long = System.currentTimeMillis() - val (newHistory: History, newState: UtxoState, blocksApplied: Seq[PersistentModifier]) = - updateState(historyBeforeStUpdate, nodeView.state, progressInfo, IndexedSeq(), isLocallyGenerated) - if (newHistory.isHeadersChainSynced) system.actorSelection("/user/nodeViewSynchronizer") ! HeaderChainIsSynced - if (encrySettings.influxDB.isDefined) - context.actorSelection("/user/statsSender") ! StateUpdating(System.currentTimeMillis() - startPoint) - sendUpdatedInfoToMemoryPool(progressInfo.toRemove) - if (progressInfo.chainSwitchingNeeded) - nodeView.wallet.rollback(VersionTag !@@ progressInfo.branchPoint.get).get - blocksApplied.foreach(nodeView.wallet.scanPersistent) - logger.debug(s"\nPersistent modifier ${pmod.encodedId} applied successfully") - if (encrySettings.influxDB.isDefined) newHistory.getBestHeader.foreach(header => - context.actorSelection("/user/statsSender") ! BestHeaderInChain(header)) - if (newHistory.isFullChainSynced) { - logger.debug(s"\nblockchain is synced on nvh on height ${newHistory.getBestHeaderHeight}!") - ModifiersCache.setChainSynced() - system.actorSelection("/user/nodeViewSynchronizer") ! FullBlockChainIsSynced - system.actorSelection("/user/miner") ! FullBlockChainIsSynced - } - updateNodeView(Some(newHistory), Some(newState), Some(nodeView.wallet)) - } else { - influxRef.foreach { ref => - logger.info(s"send info 3. about ${historyBeforeStUpdate.getBestHeaderHeight} | ${historyBeforeStUpdate.getBestBlockHeight}") - ref ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height) - } - if (!isLocallyGenerated) requestDownloads(progressInfo, Some(pmod.id)) - context.system.eventStream.publish(SemanticallySuccessfulModifier(pmod)) - logger.info(s"\nProgress info is empty") - updateNodeView(updatedHistory = Some(historyBeforeStUpdate)) - } - case Left(e) => - logger.debug(s"\nCan`t apply persistent modifier (id: ${pmod.encodedId}, contents: $pmod)" + - s" to history caused $e") - context.system.eventStream.publish(SyntacticallyFailedModification(pmod, List(HistoryApplyError(e.getMessage)))) - } - } else logger.info(s"\nTrying to apply modifier ${pmod.encodedId} that's already in history.") - - def sendUpdatedInfoToMemoryPool(toRemove: Seq[PersistentModifier]): Unit = { - val rolledBackTxs: IndexedSeq[Transaction] = toRemove - .flatMap(extractTransactions) - .toIndexedSeq - if (rolledBackTxs.nonEmpty) - memoryPoolRef ! RolledBackTransactions(rolledBackTxs) - } - - def extractTransactions(mod: PersistentModifier): Seq[Transaction] = mod match { - case b: Block => b.payload.txs - case p: Payload => p.txs - case _ => Seq.empty[Transaction] - } - - def genesisState(influxRef: Option[ActorRef] = None): NodeView = { - val stateDir: File = UtxoState.getStateDir(encrySettings) - stateDir.mkdir() - val rootsDir: File = UtxoState.getRootsDir(encrySettings) - rootsDir.mkdir() - assert(stateDir.listFiles().isEmpty, s"Genesis directory $stateDir should always be empty.") - val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, encrySettings, influxRef) - val history: History = History.readOrGenerate(encrySettings, timeProvider) - val wallet: EncryWallet = - EncryWallet.readOrGenerate(EncryWallet.getWalletDir(encrySettings), EncryWallet.getKeysDir(encrySettings), encrySettings) - NodeView(history, state, wallet) - } - - def restoreState(influxRef: Option[ActorRef] = None): Option[NodeView] = if (History.getHistoryIndexDir(encrySettings).listFiles.nonEmpty) - try { - val stateDir: File = UtxoState.getStateDir(encrySettings) - stateDir.mkdirs() - val rootsDir: File = UtxoState.getRootsDir(encrySettings) - rootsDir.mkdir() - val history: History = History.readOrGenerate(encrySettings, timeProvider) - val wallet: EncryWallet = - EncryWallet.readOrGenerate(EncryWallet.getWalletDir(encrySettings), EncryWallet.getKeysDir(encrySettings), encrySettings) - val state: UtxoState = restoreConsistentState( - UtxoState.create(stateDir, rootsDir, encrySettings, influxRef), history, influxRef - ) - history.updateIdsForSyncInfo() - logger.info(s"History best block height: ${history.getBestBlockHeight}") - logger.info(s"History best header height: ${history.getBestHeaderHeight}") - Some(NodeView(history, state, wallet)) - } catch { - case ex: Throwable => - logger.info(s"${ex.getMessage} during state restore. Recover from Modifiers holder!") - new File(encrySettings.directory).listFiles.foreach(dir => FileUtils.cleanDirectory(dir)) - Some(genesisState(influxRef)) - } else { - None - } - - def getRecreatedState(version: Option[VersionTag] = None, - digest: Option[ADDigest] = None, - influxRef: Option[ActorRef]): UtxoState = { - val dir: File = UtxoState.getStateDir(encrySettings) - dir.mkdirs() - dir.listFiles.foreach(_.delete()) - val stateDir: File = UtxoState.getStateDir(encrySettings) - stateDir.mkdirs() - val rootsDir: File = UtxoState.getRootsDir(encrySettings) - rootsDir.mkdir() - UtxoState.create(stateDir, rootsDir, encrySettings, influxRef) - } - - def restoreConsistentState(stateIn: UtxoState, history: History, influxRefActor: Option[ActorRef]): UtxoState = - (stateIn.version, history.getBestBlock, stateIn, stateIn.safePointHeight) match { - case (stateId, None, _, _) if stateId sameElements Array.emptyByteArray => - logger.info(s"State and history are both empty on startup") - stateIn - case (_, None, _, _) => - logger.info(s"State and history are inconsistent." + - s" History is empty on startup, rollback state to genesis.") - getRecreatedState(influxRef = influxRefActor) - case (_, Some(historyBestBlock), state: UtxoState, safePointHeight) => - val headerAtSafePointHeight = history.getBestHeaderAtHeight(safePointHeight) - val (rollbackId, newChain) = history.getChainToHeader(headerAtSafePointHeight, historyBestBlock.header) - logger.info(s"State and history are inconsistent." + - s" Going to rollback to ${rollbackId.map(Algos.encode)} and " + - s"apply ${newChain.length} modifiers") - val additionalBlocks = (state.safePointHeight + 1 to historyBestBlock.header.height).foldLeft(List.empty[Block]){ - case (blocks, height) => - val headerAtHeight = history.getBestHeaderAtHeight(height).get - val blockAtHeight = history.getBlockByHeader(headerAtHeight).get - blocks :+ blockAtHeight - } - logger.info(s"Qty of additional blocks: ${additionalBlocks.length}") - rollbackId.map(_ => state.restore(additionalBlocks).get) - .getOrElse(getRecreatedState(influxRef = influxRefActor)) - } - - override def close(): Unit = { - nodeView.history.close() - nodeView.state.close() - nodeView.wallet.close() - } } object NodeViewHolder { From 35dc809c44f957e7d025506114aa2abd03d78601 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 14:43:52 +0300 Subject: [PATCH 007/177] added full-filled receive function --- src/main/scala/encry/nvg/NodeViewHolder.scala | 100 ++++++++++++------ .../scala/encry/view/NodeViewHolder.scala | 52 +-------- 2 files changed, 68 insertions(+), 84 deletions(-) diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index f83295b81d..194cb16c6f 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -6,46 +6,27 @@ import akka.actor.{ Actor, ActorRef, Props } import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.EncryApp.system -import encry.api.http.DataHolderForApi import encry.consensus.HistoryConsensus.ProgressInfo import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.NodeViewSynchronizer.ReceivableMessages.{ - ChangedHistory, - ChangedState, - RollbackFailed, - RollbackSucceed, - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification -} +import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.NodeViewHolder.NodeView import encry.settings.EncryAppSettings -import encry.stats.StatsSender.{ - BestHeaderInChain, - EndOfApplyingModifier, - HeightStatistics, - ModifierAppendedToHistory, - ModifierAppendedToState, - StartApplyingModifier, - StateUpdating, - TransactionsInBlock -} +import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.ModifiersCache import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError -import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote +import encry.view.NodeViewHolder.ReceivableMessages.{ + CreateAccountManagerFromSeed, + LocallyGeneratedModifier, + ModifierFromRemote +} import encry.view.NodeViewHolder.{ DownloadRequest, UpdateInformation } import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId -import encry.view.fast.sync.SnapshotHolder.{ - HeaderChainIsSynced, - RequiredManifestHeightAndId, - SnapshotChunk, - TreeChunks -} -import encry.view.history.{ History, HistoryReader } -import encry.view.mempool.MemoryPool.RolledBackTransactions +import encry.view.fast.sync.SnapshotHolder._ +import encry.view.history.storage.HistoryStorage +import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet @@ -58,7 +39,6 @@ import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId } import scala.collection.{ mutable, IndexedSeq, Seq } -import scala.concurrent.ExecutionContextExecutor import scala.util.{ Failure, Success, Try } class NodeViewHolder( @@ -69,14 +49,68 @@ class NodeViewHolder( with StrictLogging with AutoCloseable { - import context.dispatcher - var nodeView: NodeView = restoreState().getOrElse(genesisState) var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] override def receive: Receive = { - case ModifierFromRemote(modifier) => + case ModifierFromRemote(modifier: PersistentModifier) => + val isInHistory: Boolean = nodeView.history.isModifierDefined(modifier.id) + val isInCache: Boolean = ModifiersCache.contains(key(modifier.id)) + if (isInHistory || isInCache) + logger.info( + s"Modifier ${modifier.encodedId} can't be placed into the cache cause:" + + s" contains in cache: $isInCache, contains in history: $isInHistory" + ) + else ModifiersCache.put(key(modifier.id), modifier, nodeView.history) + computeApplications() + + case LocallyGeneratedModifier(modifier: PersistentModifier) => + val startTime: Long = System.currentTimeMillis() + logger.info(s"Got locally generated modifier ${modifier.encodedId} of type ${modifier.modifierTypeId}") + modifier match { + case block: Block => + pmodModify(block.header, isLocallyGenerated = true) + pmodModify(block.payload, isLocallyGenerated = true) + } + logger.debug( + s"Processing time of modifier ${modifier.encodedId} is ${(System.currentTimeMillis() - startTime) / 1000}s." + ) + + case FastSyncFinished(state, wallet) => + logger.info(s"Node view holder got message FastSyncDoneAt. Started state replacing.") + nodeView.state.tree.avlStorage.close() + nodeView.wallet.close() + FileUtils.deleteDirectory(new File(s"${settings.directory}/tmpDirState")) + FileUtils.deleteDirectory(new File(s"${settings.directory}/keysTmp")) + FileUtils.deleteDirectory(new File(s"${settings.directory}/walletTmp")) + logger.info(s"Updated best block in fast sync mod. Updated state height.") + val newHistory = new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { + override val settings: EncryAppSettings = settings + override var isFullChainSynced: Boolean = settings.node.offlineGeneration + override val timeProvider: NetworkTimeProvider = EncryApp.timeProvider + override val historyStorage: HistoryStorage = nodeView.history.historyStorage + } + newHistory.fastSyncInProgress.fastSyncVal = false + newHistory.blockDownloadProcessor.updateMinimalBlockHeightVar( + nodeView.history.blockDownloadProcessor.minimalBlockHeight + ) + newHistory.isHeadersChainSyncedVar = true + updateNodeView( + updatedHistory = Some(newHistory), + updatedState = Some(state), + updatedVault = Some(wallet) + ) + system.actorSelection("/user/nodeViewSynchronizer") ! FastSyncDone + logger.info(s"Fast sync finished successfully!") + + case CreateAccountManagerFromSeed(seed) => + val newAccount = nodeView.wallet.addAccount(seed, settings.wallet.map(_.password).get, nodeView.state) + updateNodeView(updatedVault = newAccount.toOption) + sender() ! newAccount + + case RemoveRedundantManifestIds => potentialManifestIds = List.empty + } //todo refactor loop diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index 404f9a52ae..9f3a060876 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -84,57 +84,7 @@ class NodeViewHolder(memoryPoolRef: ActorRef, var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] override def receive: Receive = { - case CreateAccountManagerFromSeed(seed) => - val newAccount = nodeView.wallet.addAccount(seed, encrySettings.wallet.map(_.password).get, nodeView.state) - updateNodeView(updatedVault = newAccount.toOption) - sender() ! newAccount - case FastSyncFinished(state, wallet) => - logger.info(s"Node view holder got message FastSyncDoneAt. Started state replacing.") - nodeView.state.tree.avlStorage.close() - nodeView.wallet.close() - FileUtils.deleteDirectory(new File(s"${encrySettings.directory}/tmpDirState")) - FileUtils.deleteDirectory(new File(s"${encrySettings.directory}/keysTmp")) - FileUtils.deleteDirectory(new File(s"${encrySettings.directory}/walletTmp")) - logger.info(s"Updated best block in fast sync mod. Updated state height.") - val newHistory = new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { - override val settings: EncryAppSettings = encrySettings - override var isFullChainSynced: Boolean = settings.node.offlineGeneration - override val timeProvider: NetworkTimeProvider = EncryApp.timeProvider - override val historyStorage: HistoryStorage = nodeView.history.historyStorage - } - newHistory.fastSyncInProgress.fastSyncVal = false - newHistory.blockDownloadProcessor.updateMinimalBlockHeightVar(nodeView.history.blockDownloadProcessor.minimalBlockHeight) - newHistory.isHeadersChainSyncedVar = true - updateNodeView( - updatedHistory = Some(newHistory), - updatedState = Some(state), - updatedVault = Some(wallet) - ) - system.actorSelection("/user/nodeViewSynchronizer") ! FastSyncDone - logger.info(s"Fast sync finished successfully!") - case RemoveRedundantManifestIds => potentialManifestIds = List.empty - case ModifierFromRemote(mod) => - val isInHistory: Boolean = nodeView.history.isModifierDefined(mod.id) - val isInCache: Boolean = ModifiersCache.contains(key(mod.id)) - if (isInHistory || isInCache) - logger.info(s"Received modifier of type: ${mod.modifierTypeId} ${Algos.encode(mod.id)} " + - s"can't be placed into cache cause of: inCache: ${!isInCache}.") - else ModifiersCache.put(key(mod.id), mod, nodeView.history) - computeApplications() - - case lm: LocallyGeneratedModifier => - logger.debug(s"Start processing LocallyGeneratedModifier message on NVH.") - val startTime = System.currentTimeMillis() - logger.info(s"Got locally generated modifier ${lm.pmod.encodedId} of type ${lm.pmod.modifierTypeId}") - lm.pmod match { - case block: Block => - pmodModify(block.header, isLocallyGenerated = true) - pmodModify(block.payload, isLocallyGenerated = true) - case anyMod => - pmodModify(anyMod, isLocallyGenerated = true) - } - logger.debug(s"Time processing of msg LocallyGeneratedModifier with mod of type ${lm.pmod.modifierTypeId}:" + - s" with id: ${Algos.encode(lm.pmod.id)} -> ${System.currentTimeMillis() - startTime}") + case GetDataFromCurrentView(f) => f(CurrentView(nodeView.history, nodeView.state, nodeView.wallet)) match { From 6abbb600dc781faa3bfe4d85251d9180e6b79e66 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 3 Mar 2020 14:46:40 +0300 Subject: [PATCH 008/177] add some additions to new peers keeper --- .../encry/network/NetworkController.scala | 2 +- .../scala/encry/network/NetworkRouter.scala | 21 +++--- src/main/scala/encry/network/PK.scala | 71 ++++++++++++++++++- .../scala/encry/network/PeersKeeper.scala | 10 +-- .../network/ConnectWithNewPeerTests.scala | 18 ++--- 5 files changed, 94 insertions(+), 28 deletions(-) diff --git a/src/main/scala/encry/network/NetworkController.scala b/src/main/scala/encry/network/NetworkController.scala index af2d242cac..92e43451d4 100755 --- a/src/main/scala/encry/network/NetworkController.scala +++ b/src/main/scala/encry/network/NetworkController.scala @@ -93,7 +93,7 @@ class NetworkController(networkSettings: NetworkSettings, logger.info(s"Network controller got 'Connected' message from: $remote. " + s"Trying to set stable connection with remote... " + s"Local TCP endpoint is: $localAddress.") - peersKeeper ! VerifyConnection(remote, sender()) + peersKeeper ! NewConnection(remote, sender()) case ConnectionVerified(remote, remoteConnection, connectionType) => logger.info(s"Network controller got approvement for stable connection with: $remote. Starting interaction process...") diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index a1ba21cf58..4602df1311 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -12,11 +12,12 @@ import encry.network.Messages.MessageToNetwork import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} -import encry.network.PeersKeeper.{BanPeer, ConnectionStopped, ConnectionVerified, HandshakedDone, OutgoingConnectionFailed, PeerForConnection, VerifyConnection} -import encry.settings.NetworkSettings +import encry.network.PeersKeeper.{BanPeer, ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection, OutgoingConnectionFailed, PeerForConnection} +import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage -class NetworkRouter(settings: NetworkSettings) extends Actor with StrictLogging { +class NetworkRouter(settings: NetworkSettings, + blackListSettings: BlackListSettings) extends Actor with StrictLogging { import context.system @@ -24,7 +25,7 @@ class NetworkRouter(settings: NetworkSettings) extends Actor with StrictLogging IO(Tcp) ! Bind(self, settings.bindAddress, options = KeepAlive(true) :: Nil, pullMode = false) - val peersKeeper = context.system.actorOf(PK.props(), "peersKeeper") + val peersKeeper = context.system.actorOf(PK.props(settings, blackListSettings), "peersKeeper") val deliveryManager = context.system.actorOf(DM.props(), "deliveryManager") val externalSocketAddress: Option[InetSocketAddress] = settings.declaredAddress @@ -70,7 +71,12 @@ class NetworkRouter(settings: NetworkSettings) extends Actor with StrictLogging logger.info(s"Network router got 'Connected' message from: $remote. " + s"Trying to set stable connection with remote... " + s"Local TCP endpoint is: $localAddress.") - peersKeeper ! VerifyConnection(remote, sender()) + peersKeeper ! NewConnection(remote, sender()) + + case HandshakedDone(remote) => + logger.info(s"Network controller got approvement from peer handler about successful handshake. " + + s"Sending to peerKeeper connected peer.") + peersKeeper ! HandshakedDone(remote) case ConnectionVerified(remote, remoteConnection, connectionType) => logger.info(s"Network controller got approvement for stable connection with: $remote. Starting interaction process...") @@ -80,11 +86,6 @@ class NetworkRouter(settings: NetworkSettings) extends Actor with StrictLogging ) peerConnectionHandler ! StartInteraction - case HandshakedDone(remote) => - logger.info(s"Network controller got approvement from peer handler about successful handshake. " + - s"Sending to peerKeeper connected peer.") - peersKeeper ! HandshakedDone(remote) - case ConnectionStopped(peer) => logger.info(s"Network controller got signal about breaking connection with: $peer. " + s"Sending to peerKeeper actual information.") diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 7adda9e5db..304caf540e 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -2,13 +2,19 @@ package encry.network import java.net.{InetAddress, InetSocketAddress} -import akka.actor.{Actor, Props} +import akka.actor.{Actor, ActorRef, Props} +import com.typesafe.scalalogging.StrictLogging +import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction +import encry.network.PeerConnectionHandler.{Incoming, Outgoing} +import encry.network.PeersKeeper.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} import encry.settings.{BlackListSettings, NetworkSettings} import scala.util.Try class PK(networkSettings: NetworkSettings, - blacklistSettings: BlackListSettings) extends Actor { + blacklistSettings: BlackListSettings) extends Actor with StrictLogging { + + val connectWithOnlyKnownPeers: Boolean = networkSettings.connectOnlyWithKnownPeers.getOrElse(true) var connectedPeers: ConnectedPeersCollection = ConnectedPeersCollection() @@ -17,7 +23,66 @@ class PK(networkSettings: NetworkSettings, var knownPeers: Set[InetAddress] = networkSettings.knownPeers .collect { case peer: InetSocketAddress if !isSelf(peer) => peer.getAddress }.toSet - override def receive: Receive = ??? + var outgoingConnections: Set[InetSocketAddress] = Set.empty + + var awaitingHandshakeConnections: Set[InetSocketAddress] = Set.empty + + var peersForConnection: Map[InetSocketAddress, Int] = networkSettings.knownPeers + .collect { case peer: InetSocketAddress if !isSelf(peer) => peer -> 0 }.toMap + + override def receive: Receive = { + case NewConnection(remote, remoteConnection) if connectedPeers.size < networkSettings.maxConnections && !isSelf(remote) => + logger.info(s"Peers keeper got request for verifying the connection with remote: $remote. " + + s"Remote InetSocketAddress is: $remote. Remote InetAddress is ${remote.getAddress}. " + + s"Current known peers: ${knownPeers.mkString(",")}") + val notConnectedYet: Boolean = !connectedPeers.contains(remote) + val notBannedPeer: Boolean = !blackList.contains(remote.getAddress) + if (notConnectedYet && notBannedPeer) { + logger.info(s"Peer: $remote is available to setup connect with.") + if (outgoingConnections.contains(remote)) { + logger.info(s"Got outgoing connection.") + outgoingConnections -= remote + sender() ! ConnectionVerified(remote, remoteConnection, Outgoing) + } + else if (connectWithOnlyKnownPeers && knownPeers.contains(remote.getAddress)) { + logger.info(s"connectWithOnlyKnownPeers - true, but connected peer is contained in known peers collection.") + sender() ! ConnectionVerified(remote, remoteConnection, Incoming) + } + else if (connectWithOnlyKnownPeers) + logger.info(s"Got incoming connection but we can connect only with known peers.") + else { + logger.info(s"Got new incoming connection. Sending to network controller approvement for connect.") + sender() ! ConnectionVerified(remote, remoteConnection, Incoming) + } + } else logger.info(s"Connection for requested peer: $remote is unavailable cause of:" + + s" Didn't banned: $notBannedPeer, Didn't connected: $notConnectedYet.") + + case NewConnection(remote, _) => + logger.info(s"Peers keeper got request for verifying the connection but current number of max connection is " + + s"bigger than possible or isSelf: ${isSelf(remote)}.") + + case HandshakedDone(connectedPeer) => + logger.info(s"Peers keeper got approvement about finishing a handshake." + + s" Initializing new peer: ${connectedPeer.socketAddress}") + connectedPeers = connectedPeers.initializePeer(connectedPeer) + logger.info(s"Remove ${connectedPeer.socketAddress} from awaitingHandshakeConnections collection. Current is: " + + s"${awaitingHandshakeConnections.mkString(",")}.") + awaitingHandshakeConnections -= connectedPeer.socketAddress + peersForConnection = peersForConnection.updated(connectedPeer.socketAddress, 0) + logger.info(s"Adding new peer: ${connectedPeer.socketAddress} to available collection." + + s" Current collection is: ${peersForConnection.keys.mkString(",")}.") + + case ConnectionStopped(peer) => + logger.info(s"Connection stopped for: $peer.") + awaitingHandshakeConnections -= peer + connectedPeers = connectedPeers.removePeer(peer) + if (blackList.contains(peer.getAddress)) { + peersForConnection -= peer + logger.info(s"Peer: $peer removed from availablePeers cause of it has been banned. " + + s"Current is: ${peersForConnection.mkString(",")}.") + } + + } def isSelf(address: InetSocketAddress): Boolean = Try(address == networkSettings.bindAddress || networkSettings.declaredAddress.contains(address) || diff --git a/src/main/scala/encry/network/PeersKeeper.scala b/src/main/scala/encry/network/PeersKeeper.scala index 584b0b225e..8bdf174b69 100644 --- a/src/main/scala/encry/network/PeersKeeper.scala +++ b/src/main/scala/encry/network/PeersKeeper.scala @@ -108,7 +108,7 @@ class PeersKeeper(settings: EncryAppSettings, case RequestPeerForConnection => logger.info(s"Got request for a new connection but current number of connection is max: ${connectedPeers.size}.") - case VerifyConnection(remote, remoteConnection) if connectedPeers.size < settings.network.maxConnections && !isSelf(remote) => + case NewConnection(remote, remoteConnection) if connectedPeers.size < settings.network.maxConnections && !isSelf(remote) => logger.info(s"Peers keeper got request for verifying the connection with remote: $remote. " + s"Remote InetSocketAddress is: $remote. Remote InetAddress is ${remote.getAddress}. " + s"Current known peers: ${knownPeers.mkString(",")}") @@ -134,7 +134,7 @@ class PeersKeeper(settings: EncryAppSettings, } else logger.info(s"Connection for requested peer: $remote is unavailable cause of:" + s" Didn't banned: $notBannedPeer, Didn't connected: $notConnectedYet.") - case VerifyConnection(remote, remoteConnection) => + case NewConnection(remote, remoteConnection) => logger.info(s"Peers keeper got request for verifying the connection but current number of max connection is " + s"bigger than possible or isSelf: ${isSelf(remote)}.") @@ -310,8 +310,8 @@ object PeersKeeper { sealed trait PeerCommandHelper - final case class VerifyConnection(peer: InetSocketAddress, - remoteConnection: ActorRef) + final case class NewConnection(peer: InetSocketAddress, + remoteConnection: ActorRef) final case class ConnectionVerified(peer: InetSocketAddress, remoteConnection: ActorRef, @@ -353,7 +353,7 @@ object PeersKeeper { case AccumulatedPeersStatistic(_) => 1 case BanPeer(_, _) => 1 case SendLocalSyncInfo => 1 - case VerifyConnection(_, _) => 2 + case NewConnection(_, _) => 2 case HandshakedDone(_) => 2 case ConnectionStopped(_) => 2 case OutgoingConnectionFailed(_) => 2 diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index c7b4e905c6..bdb2126515 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -219,7 +219,7 @@ class ConnectWithNewPeerTests extends WordSpecLike peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true - networkController.send(peersKeeper, VerifyConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) + networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) networkController.expectMsg( ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) @@ -239,7 +239,7 @@ class ConnectWithNewPeerTests extends WordSpecLike peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true - networkController.send(peersKeeper, VerifyConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) + networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) networkController.expectMsg( ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) @@ -260,7 +260,7 @@ class ConnectWithNewPeerTests extends WordSpecLike peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true - networkController.send(peersKeeper, VerifyConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) + networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) networkController.expectMsg( ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) @@ -275,7 +275,7 @@ class ConnectWithNewPeerTests extends WordSpecLike val remoteAddress: InetSocketAddress = testNetSettings.network.knownPeers.head val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) - networkController.send(peersKeeper, VerifyConnection(remoteAddress, remoteConnectionTestProbe.ref)) + networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) networkController.expectMsg(ConnectionVerified(remoteAddress, remoteConnectionTestProbe.ref, Incoming)) peersKeeper.stop() } @@ -290,7 +290,7 @@ class ConnectWithNewPeerTests extends WordSpecLike "test-peer", Some(remoteAddress), System.currentTimeMillis())) peersKeeper ! BanPeer(connectedPeer, SyntacticallyInvalidPersistentModifier) - networkController.send(peersKeeper, VerifyConnection(remoteAddress, remoteConnectionTestProbe.ref)) + networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) networkController.expectNoMsg() peersKeeper.stop() } @@ -305,7 +305,7 @@ class ConnectWithNewPeerTests extends WordSpecLike "test-peer", Some(remoteAddress), System.currentTimeMillis())) peersKeeper ! HandshakedDone(connectedPeer) - networkController.send(peersKeeper, VerifyConnection(remoteAddress, remoteConnectionTestProbe.ref)) + networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) networkController.expectNoMsg() peersKeeper.stop() } @@ -315,7 +315,7 @@ class ConnectWithNewPeerTests extends WordSpecLike val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.99", 9001) val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - networkController.send(peersKeeper, VerifyConnection(remoteAddress, remoteConnectionTestProbe.ref)) + networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) networkController.expectNoMsg() peersKeeper.stop() } @@ -324,7 +324,7 @@ class ConnectWithNewPeerTests extends WordSpecLike val remoteConnectionTestProbe: TestProbe = TestProbe() val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) - networkController.send(peersKeeper, VerifyConnection( + networkController.send(peersKeeper, NewConnection( new InetSocketAddress("0.0.0.0", 9001), remoteConnectionTestProbe.ref)) networkController.expectNoMsg() peersKeeper.stop() @@ -336,7 +336,7 @@ class ConnectWithNewPeerTests extends WordSpecLike peersKeeper ! RequestPeerForConnection peersKeeper.underlyingActor.outgoingConnections.contains(knowPeersSettings.network.knownPeers.head) shouldBe true - networkController.send(peersKeeper, VerifyConnection(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref)) + networkController.send(peersKeeper, NewConnection(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref)) networkController.expectMsg( ConnectionVerified(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref, Outgoing)) } From aa18607864e585f62d79044a7f26edc689d11a16 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 15:07:57 +0300 Subject: [PATCH 009/177] nvh some cleanups --- .../scala/encry/nvg/IntermediaryNVH.scala | 2 + .../scala/encry/nvg/ModifiersValidator.scala | 7 +- src/main/scala/encry/nvg/NodeViewHolder.scala | 65 ++++++++++--------- 3 files changed, 41 insertions(+), 33 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index f1af14f4b0..887dd813cb 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -10,6 +10,7 @@ import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.ModifierForValidation import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.settings.EncryAppSettings +import encry.view.fast.sync.SnapshotHolder.FastSyncDone import encry.view.history.HistoryReader import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -41,6 +42,7 @@ class IntermediaryNVH(settings: EncryAppSettings, intermediaryNetwork: ActorRef) networkMessagesProcessor ! newReader case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg + case msg @ FastSyncDone => networkMessagesProcessor ! msg } } diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index 512620a119..48ce5b076e 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -15,9 +15,8 @@ import encry.network.BlackList.BanReason.{ import encry.network.DownloadedModifiersValidator.InvalidModifier import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeer -import encry.nvg.ModifiersValidator.ModifierForValidation +import encry.nvg.ModifiersValidator.{ ModifierForValidation, ValidatedModifier } import encry.settings.EncryAppSettings -import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Header, HeaderProtoSerializer, Payload, PayloadProtoSerializer } @@ -40,7 +39,7 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings isSyntacticallyValid(modifier, settings.constants.ModifierIdSize) if (preSemanticValidation.isRight && syntacticValidation) { logger.debug(s"Modifier ${modifier.encodedId} is valid.") - nodeViewHolderRef ! ModifierFromRemote(modifier) + nodeViewHolderRef ! ValidatedModifier(modifier) } else if (!syntacticValidation) { logger.info(s"Modifier ${modifier.encodedId} is syntactically invalid.") context.parent ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) @@ -100,6 +99,8 @@ object ModifiersValidator { remote: ConnectedPeer ) + final case class ValidatedModifier(modifier: PersistentModifier) extends AnyVal + def props(nodeViewHolderRef: ActorRef, settings: EncryAppSettings): Props = Props(new ModifiersValidator(nodeViewHolderRef, settings)) } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 194cb16c6f..378d770626 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -1,15 +1,13 @@ package encry.nvg import java.io.File - import akka.actor.{ Actor, ActorRef, Props } import com.typesafe.scalalogging.StrictLogging -import encry.EncryApp -import encry.EncryApp.system import encry.consensus.HistoryConsensus.ProgressInfo import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages._ +import encry.nvg.ModifiersValidator.ValidatedModifier import encry.nvg.NodeViewHolder.NodeView import encry.settings.EncryAppSettings import encry.stats.StatsSender._ @@ -17,11 +15,7 @@ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.ModifiersCache import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError -import encry.view.NodeViewHolder.ReceivableMessages.{ - CreateAccountManagerFromSeed, - LocallyGeneratedModifier, - ModifierFromRemote -} +import encry.view.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } import encry.view.NodeViewHolder.{ DownloadRequest, UpdateInformation } import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ @@ -37,13 +31,12 @@ import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId } - import scala.collection.{ mutable, IndexedSeq, Seq } import scala.util.{ Failure, Success, Try } class NodeViewHolder( settings: EncryAppSettings, - timeProvider: NetworkTimeProvider, + ntp: NetworkTimeProvider, influxRef: Option[ActorRef] ) extends Actor with StrictLogging @@ -54,41 +47,48 @@ class NodeViewHolder( var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] override def receive: Receive = { - case ModifierFromRemote(modifier: PersistentModifier) => - val isInHistory: Boolean = nodeView.history.isModifierDefined(modifier.id) - val isInCache: Boolean = ModifiersCache.contains(key(modifier.id)) + case ValidatedModifier(modifier: PersistentModifier) => + val startTime: Long = System.currentTimeMillis() + val wrappedKey: mutable.WrappedArray.ofByte = key(modifier.id) + val isInHistory: Boolean = nodeView.history.isModifierDefined(modifier.id) + val isInCache: Boolean = ModifiersCache.contains(wrappedKey) if (isInHistory || isInCache) logger.info( - s"Modifier ${modifier.encodedId} can't be placed into the cache cause:" + - s" contains in cache: $isInCache, contains in history: $isInHistory" + s"Modifier ${modifier.encodedId} can't be placed into the cache cause: " + + s"contains in cache: $isInCache, contains in history: $isInHistory." ) - else ModifiersCache.put(key(modifier.id), modifier, nodeView.history) + else ModifiersCache.put(wrappedKey, modifier, nodeView.history) computeApplications() + logger.debug( + s"Time of processing validated modifier with id: ${modifier.encodedId} " + + s"is: ${(System.currentTimeMillis() - startTime) / 1000}s." + ) case LocallyGeneratedModifier(modifier: PersistentModifier) => val startTime: Long = System.currentTimeMillis() - logger.info(s"Got locally generated modifier ${modifier.encodedId} of type ${modifier.modifierTypeId}") + logger.info(s"Got locally generated modifier ${modifier.encodedId}.") modifier match { case block: Block => pmodModify(block.header, isLocallyGenerated = true) pmodModify(block.payload, isLocallyGenerated = true) } logger.debug( - s"Processing time of modifier ${modifier.encodedId} is ${(System.currentTimeMillis() - startTime) / 1000}s." + s"Time of process locally generated modifier with id: ${modifier.encodedId} " + + s"is ${(System.currentTimeMillis() - startTime) / 1000}s." ) case FastSyncFinished(state, wallet) => - logger.info(s"Node view holder got message FastSyncDoneAt. Started state replacing.") + val startTime: Long = System.currentTimeMillis() + logger.info(s"Node view holder got a signal about finishing fast sync process.") nodeView.state.tree.avlStorage.close() nodeView.wallet.close() FileUtils.deleteDirectory(new File(s"${settings.directory}/tmpDirState")) FileUtils.deleteDirectory(new File(s"${settings.directory}/keysTmp")) FileUtils.deleteDirectory(new File(s"${settings.directory}/walletTmp")) - logger.info(s"Updated best block in fast sync mod. Updated state height.") val newHistory = new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { override val settings: EncryAppSettings = settings override var isFullChainSynced: Boolean = settings.node.offlineGeneration - override val timeProvider: NetworkTimeProvider = EncryApp.timeProvider + override val timeProvider: NetworkTimeProvider = ntp override val historyStorage: HistoryStorage = nodeView.history.historyStorage } newHistory.fastSyncInProgress.fastSyncVal = false @@ -101,24 +101,29 @@ class NodeViewHolder( updatedState = Some(state), updatedVault = Some(wallet) ) - system.actorSelection("/user/nodeViewSynchronizer") ! FastSyncDone - logger.info(s"Fast sync finished successfully!") + context.parent ! FastSyncDone + logger.debug( + s"Time of processing fast sync done message is: ${(System.currentTimeMillis() - startTime) / 1000}s." + ) case CreateAccountManagerFromSeed(seed) => - val newAccount = nodeView.wallet.addAccount(seed, settings.wallet.map(_.password).get, nodeView.state) + val newAccount: Either[String, EncryWallet] = + nodeView.wallet.addAccount(seed, settings.wallet.map(_.password).get, nodeView.state) updateNodeView(updatedVault = newAccount.toOption) sender() ! newAccount - case RemoveRedundantManifestIds => potentialManifestIds = List.empty + case RemoveRedundantManifestIds => + potentialManifestIds = List.empty } //todo refactor loop def computeApplications(): Unit = { - val mods = ModifiersCache.popCandidate(nodeView.history) - if (mods.nonEmpty) { + val modifiers: List[PersistentModifier] = ModifiersCache.popCandidate(nodeView.history) + if (modifiers.nonEmpty) { + logger.info(s"Got new modifiers in compute application ${modifiers.map(_.encodedId)}.") logger.info(s"mods: ${mods.map(mod => Algos.encode(mod.id))}") - mods.foreach(mod => pmodModify(mod)) + modifiers.foreach(mod => pmodModify(mod)) computeApplications() } else Unit } @@ -395,7 +400,7 @@ class NodeViewHolder( rootsDir.mkdir() assert(stateDir.listFiles().isEmpty, s"Genesis directory $stateDir should always be empty.") val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) - val history: History = History.readOrGenerate(settings, timeProvider) + val history: History = History.readOrGenerate(settings, ntp) val wallet: EncryWallet = EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) NodeView(history, state, wallet) @@ -408,7 +413,7 @@ class NodeViewHolder( stateDir.mkdirs() val rootsDir: File = UtxoState.getRootsDir(settings) rootsDir.mkdir() - val history: History = History.readOrGenerate(settings, timeProvider) + val history: History = History.readOrGenerate(settings, ntp) val wallet: EncryWallet = EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) val state: UtxoState = restoreConsistentState( From 25622bb78bb92b80afcf4b814e94b60db69d0482 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 15:48:54 +0300 Subject: [PATCH 010/177] nvh improved --- .../scala/encry/nvg/IntermediaryNVH.scala | 23 ++- src/main/scala/encry/nvg/NodeViewHolder.scala | 191 ++++++++---------- 2 files changed, 108 insertions(+), 106 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 887dd813cb..8701259d2e 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -3,14 +3,22 @@ package encry.nvg import akka.actor.{ Actor, ActorRef } import akka.routing.BalancingPool import com.typesafe.scalalogging.StrictLogging +import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.BlackList.BanReason.CorruptedSerializedBytes +import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.ModifierForValidation import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.settings.EncryAppSettings -import encry.view.fast.sync.SnapshotHolder.FastSyncDone +import encry.view.NodeViewHolder.DownloadRequest +import encry.view.fast.sync.SnapshotHolder.{ + FastSyncDone, + HeaderChainIsSynced, + RequiredManifestHeightAndId, + TreeChunks +} import encry.view.history.HistoryReader import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -40,9 +48,16 @@ class IntermediaryNVH(settings: EncryAppSettings, intermediaryNetwork: ActorRef) case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! newReader - case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg - case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg - case msg @ FastSyncDone => networkMessagesProcessor ! msg + case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg + case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg + case msg @ FastSyncDone => networkMessagesProcessor ! msg + case msg @ DownloadRequest(_, _, _) => networkMessagesProcessor ! msg + case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync + case msg @ TreeChunks(_, _) => //+ to fast sync + case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg + case msg @ FullBlockChainIsSynced => networkMessagesProcessor ! msg //+ to miner + case msg @ DisableMining => //+ to miner + case msg @ StartMining => //+ to miner } } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 378d770626..8b9d90f0a2 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -1,10 +1,13 @@ package encry.nvg import java.io.File -import akka.actor.{ Actor, ActorRef, Props } + +import cats.syntax.option._ +import akka.actor.{Actor, ActorRef, Props} import com.typesafe.scalalogging.StrictLogging +import encry.EncryApp import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.local.miner.Miner.{DisableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier @@ -15,24 +18,25 @@ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.ModifiersCache import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError -import encry.view.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.view.NodeViewHolder.{ DownloadRequest, UpdateInformation } +import encry.view.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} +import encry.view.NodeViewHolder.{DownloadRequest, UpdateInformation} import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.storage.HistoryStorage -import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } +import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId } -import scala.collection.{ mutable, IndexedSeq, Seq } -import scala.util.{ Failure, Success, Try } +import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} + +import scala.collection.{IndexedSeq, Seq, mutable} +import scala.util.{Failure, Success, Try} class NodeViewHolder( settings: EncryAppSettings, @@ -69,8 +73,8 @@ class NodeViewHolder( logger.info(s"Got locally generated modifier ${modifier.encodedId}.") modifier match { case block: Block => - pmodModify(block.header, isLocallyGenerated = true) - pmodModify(block.payload, isLocallyGenerated = true) + applyModfier(block.header, isLocallyGenerated = true) + applyModfier(block.payload, isLocallyGenerated = true) } logger.debug( s"Time of process locally generated modifier with id: ${modifier.encodedId} " + @@ -97,9 +101,9 @@ class NodeViewHolder( ) newHistory.isHeadersChainSyncedVar = true updateNodeView( - updatedHistory = Some(newHistory), - updatedState = Some(state), - updatedVault = Some(wallet) + updatedHistory = newHistory.some, + updatedState = state.some, + updatedVault = wallet.some ) context.parent ! FastSyncDone logger.debug( @@ -122,8 +126,7 @@ class NodeViewHolder( val modifiers: List[PersistentModifier] = ModifiersCache.popCandidate(nodeView.history) if (modifiers.nonEmpty) { logger.info(s"Got new modifiers in compute application ${modifiers.map(_.encodedId)}.") - logger.info(s"mods: ${mods.map(mod => Algos.encode(mod.id))}") - modifiers.foreach(mod => pmodModify(mod)) + modifiers.foreach(applyModfier(_)) computeApplications() } else Unit } @@ -146,35 +149,35 @@ class NodeViewHolder( def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = None): Unit = pi.toDownload.foreach { - case (tid, id) => - if (tid != Transaction.modifierTypeId) - logger.info( - s"NVH trigger sending DownloadRequest to NVSH with type: $tid " + - s"for modifier: ${Algos.encode(id)}. PrevMod is: ${previousModifier.map(Algos.encode)}." - ) + case (tid: ModifierTypeId, id: ModifierId) => + logger.info( + s"Node view holder created download request for modifier ${Algos.encode(id)} of type $tid." + + s" Previous modifier is ${previousModifier.map(Algos.encode)}." + ) if ((nodeView.history.isFullChainSynced && tid == Payload.modifierTypeId) || tid != Payload.modifierTypeId) - system.actorSelection("/user/nodeViewSynchronizer") ! DownloadRequest(tid, id, previousModifier) - else - logger.info( - s"Ignore sending request for payload (${Algos.encode(id)}) from nvh because of nodeView.history.isFullChainSynced = false" - ) + context.parent ! DownloadRequest(tid, id, previousModifier) + else logger.info(s"Ignore sending request for payload ${Algos.encode(id)} because full chain is not synced.") } - def trimChainSuffix(suffix: IndexedSeq[PersistentModifier], - rollbackPoint: ModifierId): IndexedSeq[PersistentModifier] = { + def trimChainSuffix( + suffix: IndexedSeq[PersistentModifier], + rollbackPoint: ModifierId + ): IndexedSeq[PersistentModifier] = { val idx: Int = suffix.indexWhere(_.id.sameElements(rollbackPoint)) - if (idx == -1) IndexedSeq() else suffix.drop(idx) + if (idx == -1) IndexedSeq.empty else suffix.drop(idx) } @scala.annotation.tailrec - private def updateState(history: History, - state: UtxoState, - progressInfo: ProgressInfo, - suffixApplied: IndexedSeq[PersistentModifier], - isLocallyGenerated: Boolean = false): (History, UtxoState, Seq[PersistentModifier]) = { - logger.info(s"\nStarting updating state in updateState function!") + private def updateState( + history: History, + state: UtxoState, + progressInfo: ProgressInfo, + suffixApplied: IndexedSeq[PersistentModifier], + isLocallyGenerated: Boolean = false + ): (History, UtxoState, Seq[PersistentModifier]) = { + logger.info(s"Starting updating state in updateState function!") if (!isLocallyGenerated) progressInfo.toApply.foreach { - case header: Header => requestDownloads(progressInfo, Some(header.id)) + case header: Header => requestDownloads(progressInfo, header.id.some) case _ => requestDownloads(progressInfo, None) } val branchingPointOpt: Option[VersionTag] = progressInfo.branchPoint.map(VersionTag !@@ _) @@ -189,10 +192,10 @@ class NodeViewHolder( val blockAtHeight = history.getBlockByHeader(headerAtHeight).get blocks :+ blockAtHeight } - context.system.actorSelection("/user/miner") ! DisableMining + context.parent ! DisableMining state.rollbackTo(branchPoint, additionalBlocks) -> trimChainSuffix(suffixApplied, ModifierId !@@ branchPoint) - } else Success(state) -> IndexedSeq() + } else Success(state) -> IndexedSeq.empty }.getOrElse(Failure(new Exception("Trying to rollback when branchPoint is empty."))) } else Success(state) -> suffixApplied stateToApplyTry match { @@ -201,7 +204,8 @@ class NodeViewHolder( val u0: UpdateInformation = UpdateInformation(history, stateToApply, None, None, suffixTrimmed) val uf: UpdateInformation = progressInfo.toApply.foldLeft(u0) { case (u, modToApply) => - val saveRootNodesFlag = (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2 + val saveRootNodesFlag: Boolean = + (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2 if (u.failedMod.isEmpty) u.state.applyModifier(modToApply, saveRootNodesFlag) match { case Right(stateAfterApply) => influxRef.foreach( @@ -239,7 +243,7 @@ class NodeViewHolder( currentChunkHeight = settings.snapshotSettings.chunkDepth, stateAfterApply.tree.avlStorage ) - system.actorSelection("/user/nodeViewSynchronizer") ! TreeChunks(chunks, potentialManifestId) + context.parent ! TreeChunks(chunks, potentialManifestId) potentialManifestIds = ManifestId @@ potentialManifestId :: potentialManifestIds logger.info( s"State tree successfully processed for snapshot. " + @@ -248,7 +252,7 @@ class NodeViewHolder( } } if (settings.node.mining && progressInfo.chainSwitchingNeeded) - context.system.actorSelection("/user/miner") ! StartMining + context.parent ! StartMining context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) if (newHis.getBestHeaderId.exists( bestHeaderId => @@ -270,7 +274,7 @@ class NodeViewHolder( val (newHis: History, newProgressInfo: ProgressInfo) = history.reportModifierIsInvalid(modToApply) context.system.eventStream.publish(SemanticallyFailedModification(modToApply, e)) - UpdateInformation(newHis, u.state, Some(modToApply), Some(newProgressInfo), u.suffix) + UpdateInformation(newHis, u.state, modToApply.some, newProgressInfo.some, u.suffix) } else u } uf.failedMod match { @@ -285,98 +289,81 @@ class NodeViewHolder( } } - def pmodModify(pmod: PersistentModifier, isLocallyGenerated: Boolean = false): Unit = - if (!nodeView.history.isModifierDefined(pmod.id)) { + def applyModfier(modifier: PersistentModifier, isLocallyGenerated: Boolean = false): Unit = + if (!nodeView.history.isModifierDefined(modifier.id)) { logger.debug( - s"\nStarting to apply modifier ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to history." + s"Start modifier ${modifier.encodedId} application of type ${modifier.modifierTypeId} to the history." ) - val startAppHistory = System.currentTimeMillis() - if (settings.influxDB.isDefined) - context.system - .actorSelection("user/statsSender") ! - StartApplyingModifier(pmod.id, pmod.modifierTypeId, System.currentTimeMillis()) - nodeView.history.append(pmod) match { + val startApplicationToTheHistory: Long = System.currentTimeMillis() + influxRef.foreach(_ ! StartApplyingModifier(modifier.id, modifier.modifierTypeId, System.currentTimeMillis())) + nodeView.history.append(modifier) match { case Right((historyBeforeStUpdate, progressInfo)) => logger.info( - s"Successfully applied modifier ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to history." - ) - logger.debug( - s"Time of applying to history SUCCESS is: ${System.currentTimeMillis() - startAppHistory}. modId is: ${pmod.encodedId}" + s"Successfully applied modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} to the history. " + + s"Time of applying is: ${(System.currentTimeMillis() - startApplicationToTheHistory) / 1000}s." ) - if (pmod.modifierTypeId == Header.modifierTypeId) historyBeforeStUpdate.updateIdsForSyncInfo() - influxRef.foreach { ref => - ref ! EndOfApplyingModifier(pmod.id) - val isHeader: Boolean = pmod match { + if (modifier.modifierTypeId == Header.modifierTypeId) historyBeforeStUpdate.updateIdsForSyncInfo() + influxRef.foreach { ref: ActorRef => + ref ! EndOfApplyingModifier(modifier.id) + ref ! ModifierAppendedToHistory(modifier match { case _: Header => true case _: Payload => false - } - ref ! ModifierAppendedToHistory(isHeader, success = true) + }, success = true) } - if (historyBeforeStUpdate.fastSyncInProgress.fastSyncVal && pmod.modifierTypeId == Payload.modifierTypeId && + if (historyBeforeStUpdate.fastSyncInProgress.fastSyncVal && + modifier.modifierTypeId == Payload.modifierTypeId && historyBeforeStUpdate.getBestBlockHeight >= historyBeforeStUpdate.lastAvailableManifestHeight) { - logger.info(s"nodeView.history.getBestBlockHeight ${historyBeforeStUpdate.getBestBlockHeight}") logger.info( - s"nodeView.history.heightOfLastAvailablePayloadForRequest ${historyBeforeStUpdate.lastAvailableManifestHeight}" + s"getBestBlockHeight ${historyBeforeStUpdate.getBestBlockHeight}. " + + s"heightOfLastAvailablePayloadForRequest ${historyBeforeStUpdate.lastAvailableManifestHeight}" ) historyBeforeStUpdate.getBestHeaderAtHeight(historyBeforeStUpdate.lastAvailableManifestHeight).foreach { - h => - system.actorSelection("/user/nodeViewSynchronizer") ! RequiredManifestHeightAndId( + h: Header => + context.parent ! RequiredManifestHeightAndId( historyBeforeStUpdate.lastAvailableManifestHeight, Algos.hash(h.stateRoot ++ h.id) ) } } logger.info( - s"Going to apply modifications ${pmod.encodedId} of type ${pmod.modifierTypeId} on nodeViewHolder to the state: $progressInfo" + s"Going to apply modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} to the state. " + + s"Progress info is: $progressInfo." ) if (progressInfo.toApply.nonEmpty) { - logger.info( - s"\n progress info non empty. To apply: ${progressInfo.toApply.map(mod => Algos.encode(mod.id))}" - ) val startPoint: Long = System.currentTimeMillis() + logger.info(s"Progress info is non empty. To apply is: ${progressInfo.toApply.map(_.encodedId)}.") val (newHistory: History, newState: UtxoState, blocksApplied: Seq[PersistentModifier]) = updateState(historyBeforeStUpdate, nodeView.state, progressInfo, IndexedSeq(), isLocallyGenerated) - if (newHistory.isHeadersChainSynced) - system.actorSelection("/user/nodeViewSynchronizer") ! HeaderChainIsSynced - if (settings.influxDB.isDefined) - context.actorSelection("/user/statsSender") ! StateUpdating(System.currentTimeMillis() - startPoint) + if (newHistory.isHeadersChainSynced) context.parent ! HeaderChainIsSynced + influxRef.foreach(_ ! StateUpdating(System.currentTimeMillis() - startPoint)) sendUpdatedInfoToMemoryPool(progressInfo.toRemove) if (progressInfo.chainSwitchingNeeded) nodeView.wallet.rollback(VersionTag !@@ progressInfo.branchPoint.get).get blocksApplied.foreach(nodeView.wallet.scanPersistent) - logger.debug(s"\nPersistent modifier ${pmod.encodedId} applied successfully") - if (settings.influxDB.isDefined) - newHistory.getBestHeader.foreach( - header => context.actorSelection("/user/statsSender") ! BestHeaderInChain(header) - ) + logger.debug(s"Persistent modifier ${modifier.encodedId} was applied successfully.") + for { + ref <- influxRef + header <- newHistory.getBestHeader + } yield ref ! BestHeaderInChain(header) if (newHistory.isFullChainSynced) { - logger.debug(s"\nblockchain is synced on nvh on height ${newHistory.getBestHeaderHeight}!") + logger.debug(s"BlockChain is synced on nvh at the height ${newHistory.getBestHeaderHeight}.") ModifiersCache.setChainSynced() - system.actorSelection("/user/nodeViewSynchronizer") ! FullBlockChainIsSynced - system.actorSelection("/user/miner") ! FullBlockChainIsSynced + context.parent ! FullBlockChainIsSynced } - updateNodeView(Some(newHistory), Some(newState), Some(nodeView.wallet)) + updateNodeView(newHistory.some, newState.some, nodeView.wallet.some) } else { - influxRef.foreach { ref => - logger.info( - s"send info 3. about ${historyBeforeStUpdate.getBestHeaderHeight} | ${historyBeforeStUpdate.getBestBlockHeight}" - ) - ref ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height) - } - if (!isLocallyGenerated) requestDownloads(progressInfo, Some(pmod.id)) - context.system.eventStream.publish(SemanticallySuccessfulModifier(pmod)) - logger.info(s"\nProgress info is empty") - updateNodeView(updatedHistory = Some(historyBeforeStUpdate)) + logger.info(s"Progress info is empty.") + influxRef.foreach(_ ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height)) + if (!isLocallyGenerated) requestDownloads(progressInfo, modifier.id.some) + context.system.eventStream.publish(SemanticallySuccessfulModifier(modifier)) + updateNodeView(updatedHistory = historyBeforeStUpdate.some) } case Left(e) => - logger.debug( - s"\nCan`t apply persistent modifier (id: ${pmod.encodedId}, contents: $pmod)" + - s" to history caused $e" - ) + logger.debug(s"Can't apply modifier ${modifier.encodedId}, contents: $modifier to history cause $e.") context.system.eventStream - .publish(SyntacticallyFailedModification(pmod, List(HistoryApplyError(e.getMessage)))) + .publish(SyntacticallyFailedModification(modifier, List(HistoryApplyError(e.getMessage)))) } - } else logger.info(s"\nTrying to apply modifier ${pmod.encodedId} that's already in history.") + } else logger.info(s"Trying to apply modifier ${modifier.encodedId} that's already in history.") def key(id: ModifierId): mutable.WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) @@ -424,12 +411,12 @@ class NodeViewHolder( history.updateIdsForSyncInfo() logger.info(s"History best block height: ${history.getBestBlockHeight}") logger.info(s"History best header height: ${history.getBestHeaderHeight}") - Some(NodeView(history, state, wallet)) + NodeView(history, state, wallet).some } catch { case ex: Throwable => logger.info(s"${ex.getMessage} during state restore. Recover from Modifiers holder!") new File(settings.directory).listFiles.foreach(dir => FileUtils.cleanDirectory(dir)) - Some(genesisState) + genesisState.some } else { None } From f52462d5ce99cc99c14e16113f23f27967b73b9b Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 15:50:20 +0300 Subject: [PATCH 011/177] added apply to nvh --- src/main/scala/encry/nvg/IntermediaryNVH.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 8701259d2e..8a94b2d588 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -12,6 +12,7 @@ import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.ModifierForValidation import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.settings.EncryAppSettings +import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.DownloadRequest import encry.view.fast.sync.SnapshotHolder.{ FastSyncDone, @@ -22,12 +23,18 @@ import encry.view.fast.sync.SnapshotHolder.{ import encry.view.history.HistoryReader import org.encryfoundation.common.utils.TaggedTypes.ModifierId -class IntermediaryNVH(settings: EncryAppSettings, intermediaryNetwork: ActorRef) extends Actor with StrictLogging { +class IntermediaryNVH( + settings: EncryAppSettings, + intermediaryNetwork: ActorRef, + timeProvider: NetworkTimeProvider, + influxRef: Option[ActorRef] +) extends Actor + with StrictLogging { val networkMessagesProcessor: ActorRef = context.actorOf(NetworkMessagesProcessor.props, name = "Network-messages-processor") val nodeViewHolder: ActorRef = - context.actorOf(NodeViewHolder.props, name = "Node-view-holder") + context.actorOf(NodeViewHolder.props(settings, timeProvider, influxRef), name = "Node-view-holder") val modifiersValidatorRouter: ActorRef = context.actorOf( BalancingPool(5) From ddb10b41d452c4979292ac62fdb655d68225d09c Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 15:55:28 +0300 Subject: [PATCH 012/177] more additions --- .../scala/encry/nvg/IntermediaryNVH.scala | 17 +++++++------ src/main/scala/encry/nvg/NodeViewHolder.scala | 24 ++++++++++--------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 8a94b2d588..9bdd31a8e6 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -1,12 +1,13 @@ package encry.nvg -import akka.actor.{ Actor, ActorRef } +import akka.actor.{Actor, ActorRef} import akka.routing.BalancingPool import com.typesafe.scalalogging.StrictLogging -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.api.http.DataHolderForApi.BlockAndHeaderInfo +import encry.local.miner.Miner.{DisableMining, StartMining} import encry.network.BlackList.BanReason.CorruptedSerializedBytes import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } +import encry.network.DownloadedModifiersValidator.{InvalidModifier, ModifiersForValidating} import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.ModifierForValidation @@ -14,13 +15,9 @@ import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.DownloadRequest -import encry.view.fast.sync.SnapshotHolder.{ - FastSyncDone, - HeaderChainIsSynced, - RequiredManifestHeightAndId, - TreeChunks -} +import encry.view.fast.sync.SnapshotHolder.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks} import encry.view.history.HistoryReader +import encry.view.mempool.MemoryPool.RolledBackTransactions import org.encryfoundation.common.utils.TaggedTypes.ModifierId class IntermediaryNVH( @@ -65,6 +62,8 @@ class IntermediaryNVH( case msg @ FullBlockChainIsSynced => networkMessagesProcessor ! msg //+ to miner case msg @ DisableMining => //+ to miner case msg @ StartMining => //+ to miner + case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder + case msg @ RolledBackTransactions(_) => //+ to memory pool } } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 8b9d90f0a2..2c825fec9f 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -3,11 +3,12 @@ package encry.nvg import java.io.File import cats.syntax.option._ -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import com.typesafe.scalalogging.StrictLogging import encry.EncryApp +import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{DisableMining, StartMining} +import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier @@ -18,25 +19,26 @@ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.ModifiersCache import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError -import encry.view.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} -import encry.view.NodeViewHolder.{DownloadRequest, UpdateInformation} +import encry.view.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } +import encry.view.NodeViewHolder.{ DownloadRequest, UpdateInformation } import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.storage.HistoryStorage -import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} +import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } +import encry.view.mempool.MemoryPool.RolledBackTransactions import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } -import scala.collection.{IndexedSeq, Seq, mutable} -import scala.util.{Failure, Success, Try} +import scala.collection.{ mutable, IndexedSeq, Seq } +import scala.util.{ Failure, Success, Try } class NodeViewHolder( settings: EncryAppSettings, @@ -216,7 +218,7 @@ class NodeViewHolder( } ) val newHis: History = history.reportModifierIsValid(modToApply) - //dataHolder ! DataHolderForApi.BlockAndHeaderInfo(newHis.getBestHeader, newHis.getBestBlock) + context.parent ! BlockAndHeaderInfo(newHis.getBestHeader, newHis.getBestBlock) modToApply match { case header: Header => val requiredHeight: Int = header.height - settings.constants.MaxRollbackDepth @@ -371,7 +373,7 @@ class NodeViewHolder( val rolledBackTxs: IndexedSeq[Transaction] = toRemove .flatMap(extractTransactions) .toIndexedSeq - //if (rolledBackTxs.nonEmpty) memoryPoolRef ! RolledBackTransactions(rolledBackTxs) + if (rolledBackTxs.nonEmpty) context.parent ! RolledBackTransactions(rolledBackTxs) } def extractTransactions(mod: PersistentModifier): Seq[Transaction] = mod match { From 5e446d4e26991bfffd83188321413193cfbf9aa4 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 3 Mar 2020 16:10:57 +0300 Subject: [PATCH 013/177] add new pipelines to message builder --- .../scala/encry/network/MessageBuilder.scala | 25 ++++++++++++++++--- src/main/scala/encry/network/Messages.scala | 9 ++++--- .../scala/encry/network/NetworkRouter.scala | 19 ++++---------- src/main/scala/encry/network/PK.scala | 11 +++++--- .../scala/encry/network/PeersKeeper.scala | 19 ++++++++------ 5 files changed, 50 insertions(+), 33 deletions(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 5efffac15a..c1156ba322 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -1,13 +1,18 @@ package encry.network +import java.net.InetSocketAddress + import akka.actor.{Actor, ActorRef, Props} import akka.pattern._ import akka.util.Timeout -import encry.network.MessageBuilder.GetPeers +import encry.network.MessageBuilder.{GetPeerInfo, GetPeers} import encry.network.Messages.MessageToNetwork -import encry.network.Messages.MessageToNetwork.{RequestFromLocal, SendSyncInfo} +import encry.network.Messages.MessageToNetwork.{RequestFromLocal, ResponseFromLocal, SendSyncInfo} +import encry.network.PeerConnectionHandler.ConnectedPeer +import org.encryfoundation.common.network.BasicMessagesRepo.{ModifiersNetworkMessage, RequestModifiersNetworkMessage} import scala.concurrent.duration._ +import scala.util.Try case class MessageBuilder(msg: MessageToNetwork, peersKeeper: ActorRef) extends Actor { @@ -17,9 +22,20 @@ case class MessageBuilder(msg: MessageToNetwork, peersKeeper: ActorRef) extends override def receive: Receive = { case RequestFromLocal(peer, modTypeId, modsIds) => + Try { + (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].map { peer => + peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) + } + } case SendSyncInfo(syncInfo) => - (peersKeeper ? GetPeers).mapTo[List[ActorRef]].map { peers => - peers.foreach(_ ! syncInfo) + (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => + peers.foreach(_.handlerRef ! syncInfo) + } + case ResponseFromLocal(peer, modTypeId, modsIds) => + Try { + (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].map { peer => + peer.handlerRef ! ModifiersNetworkMessage(modTypeId -> modsIds) + } } } } @@ -27,6 +43,7 @@ case class MessageBuilder(msg: MessageToNetwork, peersKeeper: ActorRef) extends object MessageBuilder { case object GetPeers + case class GetPeerInfo(peerIp: InetSocketAddress) def props(msg: MessageToNetwork, peersKeeper: ActorRef): Props = Props(new MessageBuilder(msg, peersKeeper)) } \ No newline at end of file diff --git a/src/main/scala/encry/network/Messages.scala b/src/main/scala/encry/network/Messages.scala index 193e887081..0a57f29f32 100644 --- a/src/main/scala/encry/network/Messages.scala +++ b/src/main/scala/encry/network/Messages.scala @@ -1,6 +1,6 @@ package encry.network -import encry.network.PeerConnectionHandler.ConnectedPeer +import java.net.InetSocketAddress import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} @@ -8,9 +8,12 @@ object Messages { sealed trait MessageToNetwork object MessageToNetwork { - final case class RequestFromLocal(source: ConnectedPeer, + final case class RequestFromLocal(source: InetSocketAddress, modifierTypeId: ModifierTypeId, - modifierIds: Seq[ModifierId]) extends MessageToNetwork + modifierIds: List[ModifierId]) extends MessageToNetwork final case class SendSyncInfo(syncInfo: SyncInfo) extends MessageToNetwork + final case class ResponseFromLocal(source: InetSocketAddress, + modifierTypeId: ModifierTypeId, + modifiers: Map[ModifierId, Array[Byte]]) extends MessageToNetwork } } diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 4602df1311..41f7629a7b 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -12,7 +12,8 @@ import encry.network.Messages.MessageToNetwork import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} -import encry.network.PeersKeeper.{BanPeer, ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection, OutgoingConnectionFailed, PeerForConnection} +import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionVerified, NewConnection, OutgoingConnectionFailed} +import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConnection} import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage @@ -53,7 +54,7 @@ class NetworkRouter(settings: NetworkSettings, case MessageFromNetwork(message, Some(remote)) => peersKeeper ! BanPeer(remote, InvalidNetworkMessage(message.messageName)) logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") - case msg: MessageToNetwork => context.system.actorOf(MessageBuilder.props(msg), "peersKeeper") + case msg: MessageToNetwork => context.system.actorOf(MessageBuilder.props(msg, peersKeeper), "peersKeeper") } def peersLogic: Receive = { @@ -66,18 +67,11 @@ class NetworkRouter(settings: NetworkSettings, Some(settings.connectionTimeout), pullMode = true ) - case Connected(remote, localAddress) => - logger.info(s"Network router got 'Connected' message from: $remote. " + + logger.info(s"Network controller got 'Connected' message from: $remote. " + s"Trying to set stable connection with remote... " + s"Local TCP endpoint is: $localAddress.") peersKeeper ! NewConnection(remote, sender()) - - case HandshakedDone(remote) => - logger.info(s"Network controller got approvement from peer handler about successful handshake. " + - s"Sending to peerKeeper connected peer.") - peersKeeper ! HandshakedDone(remote) - case ConnectionVerified(remote, remoteConnection, connectionType) => logger.info(s"Network controller got approvement for stable connection with: $remote. Starting interaction process...") val peerConnectionHandler: ActorRef = context.actorOf( @@ -86,10 +80,7 @@ class NetworkRouter(settings: NetworkSettings, ) peerConnectionHandler ! StartInteraction - case ConnectionStopped(peer) => - logger.info(s"Network controller got signal about breaking connection with: $peer. " + - s"Sending to peerKeeper actual information.") - peersKeeper ! ConnectionStopped(peer) + case msg: ConnectionStatusMessages => peersKeeper ! msg case CommandFailed(connect: Connect) => logger.info(s"Failed to connect to: ${connect.remoteAddress}.") diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 304caf540e..d0d13ef2cb 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -2,11 +2,11 @@ package encry.network import java.net.{InetAddress, InetSocketAddress} -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{Actor, Props} import com.typesafe.scalalogging.StrictLogging -import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction +import encry.network.MessageBuilder.{GetPeerInfo, GetPeers} import encry.network.PeerConnectionHandler.{Incoming, Outgoing} -import encry.network.PeersKeeper.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} +import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} import encry.settings.{BlackListSettings, NetworkSettings} import scala.util.Try @@ -81,7 +81,10 @@ class PK(networkSettings: NetworkSettings, logger.info(s"Peer: $peer removed from availablePeers cause of it has been banned. " + s"Current is: ${peersForConnection.mkString(",")}.") } - + case GetPeers => sender() ! connectedPeers.getAll.map(_._2.connectedPeer) + case GetPeerInfo(peerIp) => connectedPeers.getAll.find(_._1 == peerIp).map { + case (_, info) => sender() ! info.connectedPeer + } } def isSelf(address: InetSocketAddress): Boolean = Try(address == networkSettings.bindAddress || diff --git a/src/main/scala/encry/network/PeersKeeper.scala b/src/main/scala/encry/network/PeersKeeper.scala index 8bdf174b69..7a421d15c7 100644 --- a/src/main/scala/encry/network/PeersKeeper.scala +++ b/src/main/scala/encry/network/PeersKeeper.scala @@ -310,18 +310,21 @@ object PeersKeeper { sealed trait PeerCommandHelper - final case class NewConnection(peer: InetSocketAddress, - remoteConnection: ActorRef) + sealed trait ConnectionStatusMessages + object ConnectionStatusMessages { + final case class NewConnection(peer: InetSocketAddress, + remoteConnection: ActorRef) extends ConnectionStatusMessages - final case class ConnectionVerified(peer: InetSocketAddress, - remoteConnection: ActorRef, - ct: ConnectionType) + final case class ConnectionVerified(peer: InetSocketAddress, + remoteConnection: ActorRef, + ct: ConnectionType) extends ConnectionStatusMessages - final case class OutgoingConnectionFailed(peer: InetSocketAddress) + final case class OutgoingConnectionFailed(peer: InetSocketAddress) extends ConnectionStatusMessages - final case class HandshakedDone(peer: ConnectedPeer) + final case class HandshakedDone(peer: ConnectedPeer) extends ConnectionStatusMessages - final case class ConnectionStopped(peer: InetSocketAddress) + final case class ConnectionStopped(peer: InetSocketAddress) extends ConnectionStatusMessages + } case object RequestPeerForConnection From 5deac158ccf84bfd2bb50cfc9998af4206fc83e3 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 16:17:37 +0300 Subject: [PATCH 014/177] nvh more improvements --- .../scala/encry/nvg/IntermediaryNVH.scala | 16 ++- src/main/scala/encry/nvg/NodeViewHolder.scala | 44 ++++++- .../scala/encry/view/NodeViewHolder.scala | 122 ++++-------------- 3 files changed, 76 insertions(+), 106 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 9bdd31a8e6..c498667274 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -1,13 +1,12 @@ package encry.nvg -import akka.actor.{Actor, ActorRef} +import akka.actor.{ Actor, ActorRef } import akka.routing.BalancingPool import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo -import encry.local.miner.Miner.{DisableMining, StartMining} -import encry.network.BlackList.BanReason.CorruptedSerializedBytes +import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.DownloadedModifiersValidator.{InvalidModifier, ModifiersForValidating} +import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.ModifierForValidation @@ -15,7 +14,12 @@ import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.DownloadRequest -import encry.view.fast.sync.SnapshotHolder.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks} +import encry.view.fast.sync.SnapshotHolder.{ + FastSyncDone, + HeaderChainIsSynced, + RequiredManifestHeightAndId, + TreeChunks +} import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -63,7 +67,7 @@ class IntermediaryNVH( case msg @ DisableMining => //+ to miner case msg @ StartMining => //+ to miner case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder - case msg @ RolledBackTransactions(_) => //+ to memory pool + case msg @ RolledBackTransactions(_) => //+ to memory pool } } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 2c825fec9f..baadd2da87 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -2,8 +2,8 @@ package encry.nvg import java.io.File -import cats.syntax.option._ import akka.actor.{ Actor, ActorRef, Props } +import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo @@ -12,15 +12,14 @@ import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.NodeView +import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } +import encry.nvg.NodeViewHolder.{ DownloadRequest, NodeView, UpdateInformation } import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.ModifiersCache import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError -import encry.view.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.view.NodeViewHolder.{ DownloadRequest, UpdateInformation } import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.storage.HistoryStorage @@ -38,6 +37,7 @@ import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } import scala.collection.{ mutable, IndexedSeq, Seq } +import scala.concurrent.duration._ import scala.util.{ Failure, Success, Try } class NodeViewHolder( @@ -48,10 +48,27 @@ class NodeViewHolder( with StrictLogging with AutoCloseable { + import context.dispatcher + var nodeView: NodeView = restoreState().getOrElse(genesisState) var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] + context.parent ! BlockAndHeaderInfo(nodeView.history.getBestHeader, nodeView.history.getBestBlock) + + context.system.scheduler.schedule(1.seconds, 10.seconds) { + logger.info( + s"History best header id is: ${nodeView.history.getBestHeaderId.map(Algos.encode)}.\n " + + s"History best header height is: ${nodeView.history.getBestHeaderHeight}.\n " + + s"History best block id is: ${nodeView.history.getBestBlockId.map(Algos.encode)}.\n " + + s"History best block height is: ${nodeView.history.getBestBlockHeight}.\n " + + s"History best block header is: ${nodeView.history.getHeaderOfBestBlock.map(_.encodedId)}.\n " + + s"State height is: ${nodeView.state.height}.\n " + + s"Cache size is: ${ModifiersCache.size}.\n " + + s"Cache elements are: ${ModifiersCache.cache.keys.toList.map(key => Algos.encode(key.toArray)).mkString(",")}." + ) + } + override def receive: Receive = { case ValidatedModifier(modifier: PersistentModifier) => val startTime: Long = System.currentTimeMillis() @@ -487,6 +504,25 @@ object NodeViewHolder { final case class NodeView(history: History, state: UtxoState, wallet: EncryWallet) + object ReceivableMessages { + final case class CreateAccountManagerFromSeed(seed: String) extends AnyVal + final case class LocallyGeneratedModifier(pmod: PersistentModifier) extends AnyVal + } + + final case class DownloadRequest( + modifierTypeId: ModifierTypeId, + modifierId: ModifierId, + previousModifier: Option[ModifierId] = None + ) extends NodeViewHolderEvent + + final case class UpdateInformation( + history: History, + state: UtxoState, + failedMod: Option[PersistentModifier], + alternativeProgressInfo: Option[ProgressInfo], + suffix: IndexedSeq[PersistentModifier] + ) + def props( settings: EncryAppSettings, timeProvider: NetworkTimeProvider, diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index 9f3a060876..514fb93335 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -1,95 +1,34 @@ package encry.view -import java.io.File - -import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props} -import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} -import akka.pattern._ +import akka.actor.{ Actor, ActorRef, ActorSystem, PoisonPill, Props } +import akka.dispatch.{ PriorityGenerator, UnboundedStablePriorityMailbox } import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.EncryApp -import encry.EncryApp.{system, timeProvider} -import encry.api.http.DataHolderForApi -import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{DisableMining, StartMining} -import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ -import encry.network.PeerConnectionHandler.ConnectedPeer import encry.settings.EncryAppSettings -import encry.stats.StatsSender._ -import encry.utils.CoreTaggedTypes.VersionTag -import encry.utils.NetworkTimeProvider -import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError -import encry.view.NodeViewHolder.ReceivableMessages._ import encry.view.NodeViewHolder._ -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId -import encry.view.fast.sync.SnapshotHolder._ -import encry.view.history.storage.HistoryStorage -import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor} -import encry.view.mempool.MemoryPool.RolledBackTransactions -import encry.view.state.UtxoState -import encry.view.state.avlTree.AvlTree -import encry.view.wallet.EncryWallet -import io.iohk.iodb.ByteArrayWrapper -import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history._ import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.ModifierId -import scala.collection.{IndexedSeq, Seq, mutable} -import scala.concurrent.duration._ -import scala.concurrent.{ExecutionContextExecutor, Future} -import scala.util.{Failure, Success, Try} +import scala.collection.Seq +import scala.concurrent.Future class NodeViewHolder(memoryPoolRef: ActorRef, influxRef: Option[ActorRef], dataHolder: ActorRef, - encrySettings: EncryAppSettings) extends Actor with StrictLogging with AutoCloseable { - - implicit val exCon: ExecutionContextExecutor = context.dispatcher - - context.system.actorSelection("/user/nodeViewSynchronizer") ! ChangedHistory(nodeView.history) - - dataHolder ! UpdatedHistory(nodeView.history) - dataHolder ! ChangedState(nodeView.state) - dataHolder ! DataHolderForApi.BlockAndHeaderInfo(nodeView.history.getBestHeader, nodeView.history.getBestBlock) - - influxRef.foreach(ref => context.system.scheduler.schedule(5.second, 5.second) { - logger.info(s"send info. about ${nodeView.history.getBestHeaderHeight} | ${nodeView.history.getBestBlockHeight} | " + - s"${nodeView.state.height} -> best header id ${nodeView.history.getBestHeader.map(_.encodedId)} ->" + - s" best block id ${nodeView.history.getBestBlock.map(_.encodedId)}" + - s" Best header at best block height ${nodeView.history.getBestBlock.flatMap(b => - nodeView.history.getBestHeaderAtHeight(b.header.height) - ).map(l => l.encodedId -> Algos.encode(l.payloadId))}") - }) - - override def preStart(): Unit = logger.info(s"Node view holder started.") - - override def preRestart(reason: Throwable, message: Option[Any]): Unit = { - reason.printStackTrace() - System.exit(100) - } - - context.system.scheduler.schedule(1.seconds, 10.seconds)(logger.info(s"Modifiers cache from NVH: " + - s"${ModifiersCache.size}. Elems: ${ModifiersCache.cache.keys.map(key => Algos.encode(key.toArray)).mkString(",")}")) - - override def postStop(): Unit = { - logger.warn(s"Stopping NodeViewHolder...") - nodeView.history.closeStorage() - } - - var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] + encrySettings: EncryAppSettings) + extends Actor + with StrictLogging + with AutoCloseable { override def receive: Receive = { - - case GetDataFromCurrentView(f) => f(CurrentView(nodeView.history, nodeView.state, nodeView.wallet)) match { case resultFuture: Future[_] => resultFuture.pipeTo(sender()) - case result => sender() ! result + case result => sender() ! result } case GetNodeViewChanges(history, state, _) => @@ -100,38 +39,34 @@ class NodeViewHolder(memoryPoolRef: ActorRef, logger.info(s"Start processing CompareViews message on NVH.") val startTime = System.currentTimeMillis() val ids: Seq[ModifierId] = modifierTypeId match { - case _ => modifierIds - .filterNot(mid => nodeView.history.isModifierDefined(mid) || ModifiersCache.contains(key(mid))) + case _ => + modifierIds + .filterNot(mid => nodeView.history.isModifierDefined(mid) || ModifiersCache.contains(key(mid))) } - if (modifierTypeId != Transaction.modifierTypeId) logger.debug(s"Got compare view message on NVH from ${peer.socketAddress}." + - s" Type of requesting modifiers is: $modifierTypeId. Requesting ids size are: ${ids.size}." + - s" Sending RequestFromLocal with ids to $sender." + - s"\n Requesting ids are: ${ids.map(Algos.encode).mkString(",")}.") + if (modifierTypeId != Transaction.modifierTypeId) + logger.debug( + s"Got compare view message on NVH from ${peer.socketAddress}." + + s" Type of requesting modifiers is: $modifierTypeId. Requesting ids size are: ${ids.size}." + + s" Sending RequestFromLocal with ids to $sender." + + s"\n Requesting ids are: ${ids.map(Algos.encode).mkString(",")}." + ) if (ids.nonEmpty && (modifierTypeId == Header.modifierTypeId || (nodeView.history.isHeadersChainSynced && modifierTypeId == Payload.modifierTypeId))) sender() ! RequestFromLocal(peer, modifierTypeId, ids) - logger.debug(s"Time processing of msg CompareViews from $sender with modTypeId $modifierTypeId: ${System.currentTimeMillis() - startTime}") + logger.debug( + s"Time processing of msg CompareViews from $sender with modTypeId $modifierTypeId: ${System.currentTimeMillis() - startTime}" + ) case SemanticallySuccessfulModifier(_) => - case msg => logger.error(s"Got strange message on nvh: $msg") + case msg => logger.error(s"Got strange message on nvh: $msg") } } object NodeViewHolder { - final case class DownloadRequest(modifierTypeId: ModifierTypeId, - modifierId: ModifierId, - previousModifier: Option[ModifierId] = None) extends NodeViewHolderEvent - case class CurrentView[HIS, MS, VL](history: HIS, state: MS, vault: VL) - case class UpdateInformation(history: History, - state: UtxoState, - failedMod: Option[PersistentModifier], - alternativeProgressInfo: Option[ProgressInfo], - suffix: IndexedSeq[PersistentModifier]) object ReceivableMessages { - case class CreateAccountManagerFromSeed(seed: String) case class GetNodeViewChanges(history: Boolean, state: Boolean, vault: Boolean) @@ -141,15 +76,10 @@ object NodeViewHolder { case class CompareViews(source: ConnectedPeer, modifierTypeId: ModifierTypeId, modifierIds: Seq[ModifierId]) - final case class ModifierFromRemote(serializedModifiers: PersistentModifier) - - case class LocallyGeneratedModifier(pmod: PersistentModifier) - } class NodeViewHolderPriorityQueue(settings: ActorSystem.Settings, config: Config) - extends UnboundedStablePriorityMailbox( - PriorityGenerator { + extends UnboundedStablePriorityMailbox(PriorityGenerator { case CompareViews(_, _, _) => 0 case PoisonPill => 2 @@ -162,4 +92,4 @@ object NodeViewHolder { dataHolder: ActorRef, settings: EncryAppSettings): Props = Props(new NodeViewHolder(memoryPoolRef, influxRef, dataHolder, settings)) -} \ No newline at end of file +} From a220f71878f36e27b880a2c4f2a60fe627d95df1 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 16:17:58 +0300 Subject: [PATCH 015/177] nvh more improvements --- src/main/scala/encry/nvg/NodeViewHolder.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index baadd2da87..cb679fed5d 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -35,7 +35,6 @@ import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } - import scala.collection.{ mutable, IndexedSeq, Seq } import scala.concurrent.duration._ import scala.util.{ Failure, Success, Try } From b394be40b06083d7d6f015c3d110e7a4eaa3f316 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 3 Mar 2020 16:23:09 +0300 Subject: [PATCH 016/177] add new pipelines to message builder --- src/main/scala/encry/network/MessageBuilder.scala | 8 ++++++-- src/main/scala/encry/network/Messages.scala | 2 ++ src/main/scala/encry/network/NetworkRouter.scala | 3 +++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index c1156ba322..4df0ef55a9 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -7,9 +7,9 @@ import akka.pattern._ import akka.util.Timeout import encry.network.MessageBuilder.{GetPeerInfo, GetPeers} import encry.network.Messages.MessageToNetwork -import encry.network.Messages.MessageToNetwork.{RequestFromLocal, ResponseFromLocal, SendSyncInfo} +import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} import encry.network.PeerConnectionHandler.ConnectedPeer -import org.encryfoundation.common.network.BasicMessagesRepo.{ModifiersNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, RequestModifiersNetworkMessage} import scala.concurrent.duration._ import scala.util.Try @@ -37,6 +37,10 @@ case class MessageBuilder(msg: MessageToNetwork, peersKeeper: ActorRef) extends peer.handlerRef ! ModifiersNetworkMessage(modTypeId -> modsIds) } } + case BroadcastModifier(modTypeId, modInfo) => + (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => + peers.foreach(_.handlerRef ! InvNetworkMessage(modTypeId -> List(modInfo))) + } } } diff --git a/src/main/scala/encry/network/Messages.scala b/src/main/scala/encry/network/Messages.scala index 0a57f29f32..faf0078240 100644 --- a/src/main/scala/encry/network/Messages.scala +++ b/src/main/scala/encry/network/Messages.scala @@ -15,5 +15,7 @@ object Messages { final case class ResponseFromLocal(source: InetSocketAddress, modifierTypeId: ModifierTypeId, modifiers: Map[ModifierId, Array[Byte]]) extends MessageToNetwork + final case class BroadcastModifier(modifierTypeId: ModifierTypeId, + modifierId: ModifierId) extends MessageToNetwork } } diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 41f7629a7b..7268d69107 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -16,6 +16,7 @@ import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionVerified, N import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConnection} import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage +import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} class NetworkRouter(settings: NetworkSettings, blackListSettings: BlackListSettings) extends Actor with StrictLogging { @@ -101,5 +102,7 @@ class NetworkRouter(settings: NetworkSettings, object NetworkRouter { + case class ModifierFromNetwork(modTypeId: ModifierTypeId, modId: ModifierId, modBytes: Array[Byte]) + def props(settings: NetworkSettings): Props = Props(new NetworkRouter(settings)) } From 4647145a79f0f23bed2658a3a3b44e85e887da5f Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 3 Mar 2020 16:26:00 +0300 Subject: [PATCH 017/177] change ModifierFromNetwork --- src/main/scala/encry/network/NetworkRouter.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 7268d69107..df413f773b 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -102,7 +102,10 @@ class NetworkRouter(settings: NetworkSettings, object NetworkRouter { - case class ModifierFromNetwork(modTypeId: ModifierTypeId, modId: ModifierId, modBytes: Array[Byte]) + case class ModifierFromNetwork(source: InetSocketAddress, + modTypeId: ModifierTypeId, + modId: ModifierId, + modBytes: Array[Byte]) def props(settings: NetworkSettings): Props = Props(new NetworkRouter(settings)) } From 867402bad9d29c7d1b4a2f1e2d9a22356b007ca3 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 16:56:45 +0300 Subject: [PATCH 018/177] more nvh improvements --- .../scala/encry/nvg/IntermediaryNVH.scala | 39 +++-- src/main/scala/encry/nvg/NodeViewHolder.scala | 159 +++++++++--------- src/main/scala/encry/stats/StatsSender.scala | 17 +- 3 files changed, 109 insertions(+), 106 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index c498667274..3e3517b746 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -8,12 +8,18 @@ import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.NodeViewSynchronizer.ReceivableMessages.{ + RollbackFailed, + RollbackSucceed, + SemanticallyFailedModification, + SemanticallySuccessfulModifier +} import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.ModifierForValidation -import encry.nvg.NodeViewHolder.UpdateHistoryReader +import encry.nvg.NodeViewHolder.{ DownloadRequest, UpdateHistoryReader } import encry.settings.EncryAppSettings +import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider -import encry.view.NodeViewHolder.DownloadRequest import encry.view.fast.sync.SnapshotHolder.{ FastSyncDone, HeaderChainIsSynced, @@ -56,18 +62,23 @@ class IntermediaryNVH( case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! newReader - case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg - case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg - case msg @ FastSyncDone => networkMessagesProcessor ! msg - case msg @ DownloadRequest(_, _, _) => networkMessagesProcessor ! msg - case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync - case msg @ TreeChunks(_, _) => //+ to fast sync - case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg - case msg @ FullBlockChainIsSynced => networkMessagesProcessor ! msg //+ to miner - case msg @ DisableMining => //+ to miner - case msg @ StartMining => //+ to miner - case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder - case msg @ RolledBackTransactions(_) => //+ to memory pool + case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg + case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg + case msg @ FastSyncDone => networkMessagesProcessor ! msg + case msg @ DownloadRequest(_, _, _) => networkMessagesProcessor ! msg + case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync + case msg @ TreeChunks(_, _) => //+ to fast sync + case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg + case msg @ FullBlockChainIsSynced => networkMessagesProcessor ! msg //+ to miner + case msg @ DisableMining => //+ to miner + case msg @ StartMining => //+ to miner + case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder + case msg @ RolledBackTransactions(_) => //+ to memory pool + case msg: StatsSenderMessage => //+ to stats sender + case msg @ RollbackSucceed(_) => + case msg @ RollbackFailed(_) => + case msg @ SemanticallySuccessfulModifier(_) => + case msg @ SemanticallyFailedModification(_, _) => } } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index cb679fed5d..da205eb964 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -19,6 +19,7 @@ import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.ModifiersCache +import encry.view.NodeViewErrors.ModifierApplyError import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ @@ -35,6 +36,7 @@ import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } + import scala.collection.{ mutable, IndexedSeq, Seq } import scala.concurrent.duration._ import scala.util.{ Failure, Success, Try } @@ -71,7 +73,7 @@ class NodeViewHolder( override def receive: Receive = { case ValidatedModifier(modifier: PersistentModifier) => val startTime: Long = System.currentTimeMillis() - val wrappedKey: mutable.WrappedArray.ofByte = key(modifier.id) + val wrappedKey: mutable.WrappedArray.ofByte = toKey(modifier.id) val isInHistory: Boolean = nodeView.history.isModifierDefined(modifier.id) val isInCache: Boolean = ModifiersCache.contains(wrappedKey) if (isInHistory || isInCache) @@ -91,17 +93,17 @@ class NodeViewHolder( logger.info(s"Got locally generated modifier ${modifier.encodedId}.") modifier match { case block: Block => - applyModfier(block.header, isLocallyGenerated = true) - applyModfier(block.payload, isLocallyGenerated = true) + applyModifier(block.header, isLocallyGenerated = true) + applyModifier(block.payload, isLocallyGenerated = true) } logger.debug( - s"Time of process locally generated modifier with id: ${modifier.encodedId} " + + s"Time of processing locally generated modifier with id: ${modifier.encodedId} " + s"is ${(System.currentTimeMillis() - startTime) / 1000}s." ) - case FastSyncFinished(state, wallet) => + case FastSyncFinished(state: UtxoState, wallet: EncryWallet) => val startTime: Long = System.currentTimeMillis() - logger.info(s"Node view holder got a signal about finishing fast sync process.") + logger.info(s"Got a signal about finishing fast sync process.") nodeView.state.tree.avlStorage.close() nodeView.wallet.close() FileUtils.deleteDirectory(new File(s"${settings.directory}/tmpDirState")) @@ -124,9 +126,9 @@ class NodeViewHolder( updatedVault = wallet.some ) context.parent ! FastSyncDone - logger.debug( - s"Time of processing fast sync done message is: ${(System.currentTimeMillis() - startTime) / 1000}s." - ) + logger.debug(s"Time of processing FastSyncDone message is: ${(System.currentTimeMillis() - startTime) / 1000}s.") + + case RemoveRedundantManifestIds => potentialManifestIds = List.empty case CreateAccountManagerFromSeed(seed) => val newAccount: Either[String, EncryWallet] = @@ -134,26 +136,23 @@ class NodeViewHolder( updateNodeView(updatedVault = newAccount.toOption) sender() ! newAccount - case RemoveRedundantManifestIds => - potentialManifestIds = List.empty - } //todo refactor loop def computeApplications(): Unit = { val modifiers: List[PersistentModifier] = ModifiersCache.popCandidate(nodeView.history) if (modifiers.nonEmpty) { - logger.info(s"Got new modifiers in compute application ${modifiers.map(_.encodedId)}.") - modifiers.foreach(applyModfier(_)) + logger.info(s"Got new modifiers in compute application function: ${modifiers.map(_.encodedId)}.") + modifiers.foreach(applyModifier(_)) computeApplications() - } else Unit + } else () } - //todo replace with reader + //todo replace outgoing message with history reader def updateNodeView( - updatedHistory: Option[History] = None, - updatedState: Option[UtxoState] = None, - updatedVault: Option[EncryWallet] = None + updatedHistory: Option[History] = none, + updatedState: Option[UtxoState] = none, + updatedVault: Option[EncryWallet] = none ): Unit = { val newNodeView: NodeView = NodeView( updatedHistory.getOrElse(nodeView.history), @@ -165,16 +164,19 @@ class NodeViewHolder( nodeView = newNodeView } - def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = None): Unit = + def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = none): Unit = pi.toDownload.foreach { case (tid: ModifierTypeId, id: ModifierId) => logger.info( - s"Node view holder created download request for modifier ${Algos.encode(id)} of type $tid." + - s" Previous modifier is ${previousModifier.map(Algos.encode)}." + s"Node view holder created download request for modifier ${Algos.encode(id)} of type $tid. " + + s"Previous modifier is ${previousModifier.map(Algos.encode)}." ) - if ((nodeView.history.isFullChainSynced && tid == Payload.modifierTypeId) || tid != Payload.modifierTypeId) + if (tid != Payload.modifierTypeId || (nodeView.history.isFullChainSynced && tid == Payload.modifierTypeId)) context.parent ! DownloadRequest(tid, id, previousModifier) - else logger.info(s"Ignore sending request for payload ${Algos.encode(id)} because full chain is not synced.") + else + logger.info( + s"Ignore sending download request for modifier ${Algos.encode(id)} because full chain is not synced." + ) } def trimChainSuffix( @@ -196,20 +198,22 @@ class NodeViewHolder( logger.info(s"Starting updating state in updateState function!") if (!isLocallyGenerated) progressInfo.toApply.foreach { case header: Header => requestDownloads(progressInfo, header.id.some) - case _ => requestDownloads(progressInfo, None) + case _ => requestDownloads(progressInfo, none) } val branchingPointOpt: Option[VersionTag] = progressInfo.branchPoint.map(VersionTag !@@ _) val (stateToApplyTry: Try[UtxoState], suffixTrimmed: IndexedSeq[PersistentModifier] @unchecked) = if (progressInfo.chainSwitchingNeeded) { - branchingPointOpt.map { branchPoint => + branchingPointOpt.map { branchPoint: VersionTag => if (!state.version.sameElements(branchPoint)) { - val branchPointHeight = history.getHeaderById(ModifierId !@@ branchPoint).get.height - val additionalBlocks = (state.safePointHeight + 1 to branchPointHeight).foldLeft(List.empty[Block]) { - case (blocks, height) => - val headerAtHeight = history.getBestHeaderAtHeight(height).get - val blockAtHeight = history.getBlockByHeader(headerAtHeight).get - blocks :+ blockAtHeight - } + val branchPointHeight: Int = history.getHeaderById(ModifierId !@@ branchPoint).get.height + val additionalBlocks: List[Block] = + (state.safePointHeight + 1 to branchPointHeight).foldLeft(List.empty[Block]) { + case (blocks: List[Block], height: Int) => + //todo get best header id instead of best header + val headerAtHeight: Header = history.getBestHeaderAtHeight(height).get + val blockAtHeight: Block = history.getBlockByHeader(headerAtHeight).get + blocks :+ blockAtHeight + } context.parent ! DisableMining state.rollbackTo(branchPoint, additionalBlocks) -> trimChainSuffix(suffixApplied, ModifierId !@@ branchPoint) @@ -217,22 +221,19 @@ class NodeViewHolder( }.getOrElse(Failure(new Exception("Trying to rollback when branchPoint is empty."))) } else Success(state) -> suffixApplied stateToApplyTry match { - case Success(stateToApply) => - context.system.eventStream.publish(RollbackSucceed(branchingPointOpt)) - val u0: UpdateInformation = UpdateInformation(history, stateToApply, None, None, suffixTrimmed) + case Success(stateToApply: UtxoState) => + context.parent ! RollbackSucceed(branchingPointOpt) + val u0: UpdateInformation = UpdateInformation(history, stateToApply, none, none, suffixTrimmed) val uf: UpdateInformation = progressInfo.toApply.foldLeft(u0) { - case (u, modToApply) => + case (u: UpdateInformation, modToApply: PersistentModifier) => val saveRootNodesFlag: Boolean = (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2 if (u.failedMod.isEmpty) u.state.applyModifier(modToApply, saveRootNodesFlag) match { case Right(stateAfterApply) => - influxRef.foreach( - ref => - modToApply match { - case b: Block if history.isFullChainSynced => ref ! TransactionsInBlock(b.payload.txs.size) - case _ => - } - ) + modToApply match { + case b: Block if history.isFullChainSynced => context.parent ! TransactionsInBlock(b.payload.txs.size) + case _ => + } val newHis: History = history.reportModifierIsValid(modToApply) context.parent ! BlockAndHeaderInfo(newHis.getBestHeader, newHis.getBestBlock) modToApply match { @@ -271,27 +272,23 @@ class NodeViewHolder( } if (settings.node.mining && progressInfo.chainSwitchingNeeded) context.parent ! StartMining - context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) + context.parent ! SemanticallySuccessfulModifier(modToApply) if (newHis.getBestHeaderId.exists( bestHeaderId => newHis.getBestBlockId.exists(bId => ByteArrayWrapper(bId) == ByteArrayWrapper(bestHeaderId)) )) newHis.isFullChainSynced = true - influxRef.foreach { ref => - logger.info(s"send info 2. about ${newHis.getBestHeaderHeight} | ${newHis.getBestBlockHeight}") - ref ! HeightStatistics(newHis.getBestHeaderHeight, stateAfterApply.height) - val isBlock: Boolean = modToApply match { - case _: Block => true - case _: Payload => true - case _ => false - } - if (isBlock) ref ! ModifierAppendedToState(success = true) - } - UpdateInformation(newHis, stateAfterApply, None, None, u.suffix :+ modToApply) - case Left(e) => + context.parent ! HeightStatistics(newHis.getBestHeaderHeight, stateAfterApply.height) + if (modToApply match { + case _: Block => true + case _: Payload => true + case _ => false + }) context.parent ! ModifierAppendedToState(success = true) + UpdateInformation(newHis, stateAfterApply, none, none, u.suffix :+ modToApply) + case Left(e: List[ModifierApplyError]) => logger.info(s"Application to state failed cause $e") val (newHis: History, newProgressInfo: ProgressInfo) = history.reportModifierIsInvalid(modToApply) - context.system.eventStream.publish(SemanticallyFailedModification(modToApply, e)) + context.parent ! SemanticallyFailedModification(modToApply, e) UpdateInformation(newHis, u.state, modToApply.some, newProgressInfo.some, u.suffix) } else u } @@ -302,18 +299,18 @@ class NodeViewHolder( case None => (uf.history, uf.state, uf.suffix) } case Failure(e) => - context.system.eventStream.publish(RollbackFailed(branchingPointOpt)) + context.parent ! RollbackFailed(branchingPointOpt) EncryApp.forceStopApplication(500, s"Rollback failed: $e") } } - def applyModfier(modifier: PersistentModifier, isLocallyGenerated: Boolean = false): Unit = + def applyModifier(modifier: PersistentModifier, isLocallyGenerated: Boolean = false): Unit = if (!nodeView.history.isModifierDefined(modifier.id)) { logger.debug( s"Start modifier ${modifier.encodedId} application of type ${modifier.modifierTypeId} to the history." ) val startApplicationToTheHistory: Long = System.currentTimeMillis() - influxRef.foreach(_ ! StartApplyingModifier(modifier.id, modifier.modifierTypeId, System.currentTimeMillis())) + context.parent ! StartApplyingModifier(modifier.id, modifier.modifierTypeId, System.currentTimeMillis()) nodeView.history.append(modifier) match { case Right((historyBeforeStUpdate, progressInfo)) => logger.info( @@ -321,13 +318,11 @@ class NodeViewHolder( s"Time of applying is: ${(System.currentTimeMillis() - startApplicationToTheHistory) / 1000}s." ) if (modifier.modifierTypeId == Header.modifierTypeId) historyBeforeStUpdate.updateIdsForSyncInfo() - influxRef.foreach { ref: ActorRef => - ref ! EndOfApplyingModifier(modifier.id) - ref ! ModifierAppendedToHistory(modifier match { - case _: Header => true - case _: Payload => false - }, success = true) - } + context.parent ! EndOfApplyingModifier(modifier.id) + context.parent ! ModifierAppendedToHistory(modifier match { + case _: Header => true + case _: Payload => false + }, success = true) if (historyBeforeStUpdate.fastSyncInProgress.fastSyncVal && modifier.modifierTypeId == Payload.modifierTypeId && historyBeforeStUpdate.getBestBlockHeight >= historyBeforeStUpdate.lastAvailableManifestHeight) { @@ -353,16 +348,13 @@ class NodeViewHolder( val (newHistory: History, newState: UtxoState, blocksApplied: Seq[PersistentModifier]) = updateState(historyBeforeStUpdate, nodeView.state, progressInfo, IndexedSeq(), isLocallyGenerated) if (newHistory.isHeadersChainSynced) context.parent ! HeaderChainIsSynced - influxRef.foreach(_ ! StateUpdating(System.currentTimeMillis() - startPoint)) + context.parent ! StateUpdating(System.currentTimeMillis() - startPoint) sendUpdatedInfoToMemoryPool(progressInfo.toRemove) if (progressInfo.chainSwitchingNeeded) nodeView.wallet.rollback(VersionTag !@@ progressInfo.branchPoint.get).get blocksApplied.foreach(nodeView.wallet.scanPersistent) logger.debug(s"Persistent modifier ${modifier.encodedId} was applied successfully.") - for { - ref <- influxRef - header <- newHistory.getBestHeader - } yield ref ! BestHeaderInChain(header) + newHistory.getBestHeader.foreach(context.parent ! BestHeaderInChain(_)) if (newHistory.isFullChainSynced) { logger.debug(s"BlockChain is synced on nvh at the height ${newHistory.getBestHeaderHeight}.") ModifiersCache.setChainSynced() @@ -371,19 +363,18 @@ class NodeViewHolder( updateNodeView(newHistory.some, newState.some, nodeView.wallet.some) } else { logger.info(s"Progress info is empty.") - influxRef.foreach(_ ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height)) + context.parent ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height) if (!isLocallyGenerated) requestDownloads(progressInfo, modifier.id.some) - context.system.eventStream.publish(SemanticallySuccessfulModifier(modifier)) + context.parent ! SemanticallySuccessfulModifier(modifier) updateNodeView(updatedHistory = historyBeforeStUpdate.some) } - case Left(e) => + case Left(e: Throwable) => logger.debug(s"Can't apply modifier ${modifier.encodedId}, contents: $modifier to history cause $e.") - context.system.eventStream - .publish(SyntacticallyFailedModification(modifier, List(HistoryApplyError(e.getMessage)))) + context.parent ! SyntacticallyFailedModification(modifier, List(HistoryApplyError(e.getMessage))) } } else logger.info(s"Trying to apply modifier ${modifier.encodedId} that's already in history.") - def key(id: ModifierId): mutable.WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) + def toKey(id: ModifierId): mutable.WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) def sendUpdatedInfoToMemoryPool(toRemove: Seq[PersistentModifier]): Unit = { val rolledBackTxs: IndexedSeq[Transaction] = toRemove @@ -411,7 +402,7 @@ class NodeViewHolder( NodeView(history, state, wallet) } - def restoreState(influxRef: Option[ActorRef] = None): Option[NodeView] = + def restoreState(influxRef: Option[ActorRef] = none): Option[NodeView] = if (History.getHistoryIndexDir(settings).listFiles.nonEmpty) try { val stateDir: File = UtxoState.getStateDir(settings) @@ -436,12 +427,12 @@ class NodeViewHolder( new File(settings.directory).listFiles.foreach(dir => FileUtils.cleanDirectory(dir)) genesisState.some } else { - None + none } def getRecreatedState( - version: Option[VersionTag] = None, - digest: Option[ADDigest] = None, + version: Option[VersionTag] = none, + digest: Option[ADDigest] = none, influxRef: Option[ActorRef] ): UtxoState = { val dir: File = UtxoState.getStateDir(settings) @@ -511,7 +502,7 @@ object NodeViewHolder { final case class DownloadRequest( modifierTypeId: ModifierTypeId, modifierId: ModifierId, - previousModifier: Option[ModifierId] = None + previousModifier: Option[ModifierId] = none ) extends NodeViewHolderEvent final case class UpdateInformation( diff --git a/src/main/scala/encry/stats/StatsSender.scala b/src/main/scala/encry/stats/StatsSender.scala index f14af85b0b..deae07515f 100644 --- a/src/main/scala/encry/stats/StatsSender.scala +++ b/src/main/scala/encry/stats/StatsSender.scala @@ -147,16 +147,17 @@ class StatsSender(influxDBSettings: InfluxDBSettings, networkSettings: NetworkSe } object StatsSender { - final case class BestHeaderInChain(bestHeader: Header) extends AnyVal - final case class HeightStatistics(bestHeaderHeight: Int, bestBlockHeight: Int) - final case class TransactionsInBlock(txsNum: Int) extends AnyVal - final case class ModifierAppendedToHistory(isHeader: Boolean, success: Boolean) - final case class ModifierAppendedToState(success: Boolean) extends AnyVal + sealed trait StatsSenderMessage + final case class TransactionsInBlock(txsNum: Int) extends StatsSenderMessage + final case class BestHeaderInChain(bestHeader: Header) extends StatsSenderMessage + final case class HeightStatistics(bestHeaderHeight: Int, bestBlockHeight: Int) extends StatsSenderMessage + final case class ModifierAppendedToHistory(isHeader: Boolean, success: Boolean) extends StatsSenderMessage + final case class ModifierAppendedToState(success: Boolean) extends StatsSenderMessage final case class InfoAboutTransactionsFromMiner(qty: Int) extends AnyVal - final case class EndOfApplyingModifier(modifierId: ModifierId) extends AnyVal - final case class StateUpdating(time: Long) extends AnyVal + final case class EndOfApplyingModifier(modifierId: ModifierId) extends StatsSenderMessage + final case class StateUpdating(time: Long) extends StatsSenderMessage final case class SleepTime(time: Long) extends AnyVal - final case class StartApplyingModifier(modifierId: ModifierId, modifierTypeId: ModifierTypeId, startTime: Long) + final case class StartApplyingModifier(modifierId: ModifierId, modifierTypeId: ModifierTypeId, startTime: Long) extends StatsSenderMessage final case class MiningEnd(blockHeader: Header, workerIdx: Int, workersQty: Int) final case class MiningTime(time: Long) extends AnyVal final case class SendDownloadRequest(modifierTypeId: ModifierTypeId, modifiers: Seq[ModifierId]) From d7d88c816b525707f0ba806675516e22734700f1 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 16:57:30 +0300 Subject: [PATCH 019/177] more nvh improvements --- src/main/scala/encry/nvg/IntermediaryNVH.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 3e3517b746..ea83e0e8f4 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -82,4 +82,6 @@ class IntermediaryNVH( } } -object IntermediaryNVH {} +object IntermediaryNVH { + +} From 35e695b1ef92f6d069c6d112bbb6e1c149dc4091 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 17:11:38 +0300 Subject: [PATCH 020/177] changed incoming ModifierFromNetwork message --- src/main/scala/encry/network/PeersKeeper.scala | 2 +- src/main/scala/encry/nvg/IntermediaryNVH.scala | 15 ++++++--------- src/main/scala/encry/nvg/ModifiersValidator.scala | 5 +++-- src/main/scala/encry/view/NodeViewHolder.scala | 5 +++-- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/main/scala/encry/network/PeersKeeper.scala b/src/main/scala/encry/network/PeersKeeper.scala index 7a421d15c7..a6e380fc9e 100644 --- a/src/main/scala/encry/network/PeersKeeper.scala +++ b/src/main/scala/encry/network/PeersKeeper.scala @@ -339,7 +339,7 @@ object PeersKeeper { final case class UpdatedPeersCollection(peers: Map[InetSocketAddress, (ConnectedPeer, HistoryComparisonResult, PeersPriorityStatus)]) - final case class BanPeer(peer: ConnectedPeer, reason: BanReason) + final case class BanPeer(peer: InetSocketAddress, reason: BanReason) final case class BanPeerFromAPI(peer: InetSocketAddress, reason: BanReason) extends PeerCommandHelper diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index ea83e0e8f4..53a0907504 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -8,6 +8,7 @@ import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.NetworkRouter.ModifierFromNetwork import encry.network.NodeViewSynchronizer.ReceivableMessages.{ RollbackFailed, RollbackSucceed, @@ -28,6 +29,7 @@ import encry.view.fast.sync.SnapshotHolder.{ } import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions +import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId class IntermediaryNVH( @@ -52,12 +54,9 @@ class IntermediaryNVH( var historyReader: HistoryReader = HistoryReader.empty override def receive: Receive = { - case ModifiersForValidating(remote, typeId, modifiers) => - logger.info(s"Got ${modifiers.size} modifiers of type $typeId for validation.") - modifiers.foreach { - case (id: ModifierId, bytes: Array[Byte]) => - modifiersValidatorRouter ! ModifierForValidation(historyReader, id, typeId, bytes, remote) - } + case ModifierFromNetwork(remote, typeId, modifierId, modifierBytes) => + logger.info(s"Got modifier ${Algos.encode(modifierId)} of type $typeId from $remote for validation.") + modifiersValidatorRouter ! ModifierForValidation(historyReader, modifierId, typeId, modifierBytes, remote) case msg @ DataFromPeer(_, _) => networkMessagesProcessor ! msg case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader @@ -82,6 +81,4 @@ class IntermediaryNVH( } } -object IntermediaryNVH { - -} +object IntermediaryNVH {} diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index 48ce5b076e..215f1ec2a2 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -1,5 +1,7 @@ package encry.nvg +import java.net.InetSocketAddress + import HeaderProto.HeaderProtoMessage import PayloadProto.PayloadProtoMessage import akka.actor.{ Actor, ActorRef, Props } @@ -13,7 +15,6 @@ import encry.network.BlackList.BanReason.{ SyntacticallyInvalidPersistentModifier } import encry.network.DownloadedModifiersValidator.InvalidModifier -import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.{ ModifierForValidation, ValidatedModifier } import encry.settings.EncryAppSettings @@ -96,7 +97,7 @@ object ModifiersValidator { modifierId: ModifierId, modifierTypeId: ModifierTypeId, modifierBytes: Array[Byte], - remote: ConnectedPeer + remote: InetSocketAddress ) final case class ValidatedModifier(modifier: PersistentModifier) extends AnyVal diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index 514fb93335..67671fb731 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -1,12 +1,13 @@ package encry.view -import akka.actor.{ Actor, ActorRef, ActorSystem, PoisonPill, Props } -import akka.dispatch.{ PriorityGenerator, UnboundedStablePriorityMailbox } +import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props} +import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.settings.EncryAppSettings +import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import encry.view.NodeViewHolder._ import org.encryfoundation.common.modifiers.history._ import org.encryfoundation.common.modifiers.mempool.transaction.Transaction From f50ae42c1865b15a65bea3ca443d63efc8964af7 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 3 Mar 2020 17:33:08 +0300 Subject: [PATCH 021/177] add dm --- src/main/scala/encry/network/DM.scala | 48 ++++++++++++++++++- .../scala/encry/network/NetworkRouter.scala | 2 +- src/main/scala/encry/network/PK.scala | 16 ++++++- 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 2133fe0742..f92510536b 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -1,11 +1,55 @@ package encry.network +import java.net.InetSocketAddress + import akka.actor.{Actor, Props} +import com.typesafe.scalalogging.StrictLogging +import encry.network.DM.{AwaitingRequest, RequestSent} +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.NetworkRouter.ModifierFromNetwork +import encry.network.NodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier +import encry.settings.NetworkSettings +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} + +import scala.collection.mutable + +case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging { + + import context.dispatcher + + type ModifierIdAsKey = scala.collection.mutable.WrappedArray.ofByte -class DM extends Actor { - override def receive: Receive = ??? + var expectedModifiers: Set[ModifierIdAsKey] = Set.empty + var receivedModifier: Set[ModifierIdAsKey] = Set.empty + + override def receive: Receive = { + case RequestSent(peer, modTypeId, modId) => + expectedModifiers += toKey(modId) + context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! + AwaitingRequest(peer, modTypeId, modId, 1) + ) + case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts <= networkSettings.maxDeliveryChecks => + if (expectedModifiers.contains(toKey(modId))) context.parent ! RequestFromLocal(peer, modTypeId, List(modId)) + case AwaitingRequest(peer, _, modId, _) => + logger.info(s"Stop requesting modifier ${Algos.encode(modId)} from peer $peer") + case ModifierFromNetwork(source, modTypeId, modId, modBytes) => + if (expectedModifiers.contains(toKey(modId))) { + expectedModifiers -= toKey(modId) + receivedModifier += toKey(modId) + context.parent ! ModifierFromNetwork(source, modTypeId, modId, modBytes) + } else logger.info(s"Peer $source sent spam mod of type $modTypeId and id ${Algos.encode(modId)}") + case SemanticallySuccessfulModifier(mod) => receivedModifier -= toKey(mod.id) + } + + def toKey(id: ModifierId): ModifierIdAsKey = new mutable.WrappedArray.ofByte(id) } object DM { + + case class AwaitingRequest(peer: InetSocketAddress, modTypeId: ModifierTypeId, modId: ModifierId, attempts: Int) + case class RequestSent(peer: InetSocketAddress, modTypeId: ModifierTypeId, modId: ModifierId) + + def props(): Props = ??? } diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index df413f773b..1b6ab8e4cb 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -53,7 +53,7 @@ class NetworkRouter(settings: NetworkSettings, logger.debug(s"Got ${message.messageName} on the NetworkController.") findHandler(message, message.NetworkMessageTypeID, remote, messagesHandlers) case MessageFromNetwork(message, Some(remote)) => - peersKeeper ! BanPeer(remote, InvalidNetworkMessage(message.messageName)) + peersKeeper ! BanPeer(remote.socketAddress, InvalidNetworkMessage(message.messageName)) logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") case msg: MessageToNetwork => context.system.actorOf(MessageBuilder.props(msg, peersKeeper), "peersKeeper") } diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index d0d13ef2cb..fd67067aed 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -5,7 +5,9 @@ import java.net.{InetAddress, InetSocketAddress} import akka.actor.{Actor, Props} import com.typesafe.scalalogging.StrictLogging import encry.network.MessageBuilder.{GetPeerInfo, GetPeers} +import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection import encry.network.PeerConnectionHandler.{Incoming, Outgoing} +import encry.network.PeersKeeper.{BanPeer, BanPeerFromAPI} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} import encry.settings.{BlackListSettings, NetworkSettings} @@ -30,7 +32,7 @@ class PK(networkSettings: NetworkSettings, var peersForConnection: Map[InetSocketAddress, Int] = networkSettings.knownPeers .collect { case peer: InetSocketAddress if !isSelf(peer) => peer -> 0 }.toMap - override def receive: Receive = { + override def receive: Receive = banPeersLogic orElse { case NewConnection(remote, remoteConnection) if connectedPeers.size < networkSettings.maxConnections && !isSelf(remote) => logger.info(s"Peers keeper got request for verifying the connection with remote: $remote. " + s"Remote InetSocketAddress is: $remote. Remote InetAddress is ${remote.getAddress}. " + @@ -60,7 +62,6 @@ class PK(networkSettings: NetworkSettings, case NewConnection(remote, _) => logger.info(s"Peers keeper got request for verifying the connection but current number of max connection is " + s"bigger than possible or isSelf: ${isSelf(remote)}.") - case HandshakedDone(connectedPeer) => logger.info(s"Peers keeper got approvement about finishing a handshake." + s" Initializing new peer: ${connectedPeer.socketAddress}") @@ -87,6 +88,17 @@ class PK(networkSettings: NetworkSettings, } } + def banPeersLogic: Receive = { + case BanPeer(peer, reason) => + logger.info(s"Banning peer: ${peer} for $reason.") + blackList = blackList.banPeer(reason, peer.getAddress) + connectedPeers.getAll.find(_._1 == peer).map(_._2.connectedPeer.handlerRef ! CloseConnection) + + case BanPeerFromAPI(peer, reason) => + logger.info(s"Got msg from API... Removing peer: $peer, reason: $reason") + blackList = blackList.banPeer(reason, peer.getAddress) + } + def isSelf(address: InetSocketAddress): Boolean = Try(address == networkSettings.bindAddress || networkSettings.declaredAddress.contains(address) || InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) || From 06503d4c42131c6d7a82426186bc6e2d9df6c9a0 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 17:36:14 +0300 Subject: [PATCH 022/177] added processing of sync info and inv messages --- .../encry/network/NetworkController.scala | 2 +- .../encry/network/NodeViewSynchronizer.scala | 18 +---- .../scala/encry/nvg/IntermediaryNVH.scala | 9 ++- .../encry/nvg/NetworkMessagesProcessor.scala | 65 ++++++++++++++++++- src/main/scala/encry/nvg/NodeViewHolder.scala | 6 +- .../scala/encry/view/NodeViewHolder.scala | 2 +- .../encry/view/history/HistoryReader.scala | 22 +++++++ 7 files changed, 99 insertions(+), 25 deletions(-) diff --git a/src/main/scala/encry/network/NetworkController.scala b/src/main/scala/encry/network/NetworkController.scala index 92e43451d4..0213394830 100755 --- a/src/main/scala/encry/network/NetworkController.scala +++ b/src/main/scala/encry/network/NetworkController.scala @@ -138,7 +138,7 @@ object NetworkController { object ReceivableMessages { - case class DataFromPeer(message: NetworkMessage, source: ConnectedPeer) + case class DataFromPeer(message: NetworkMessage, source: InetSocketAddress) case class RegisterMessagesHandler(types: Seq[(Byte, String)], handler: ActorRef) } diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index 931768507c..c3e37bf98d 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -113,17 +113,7 @@ class NodeViewSynchronizer(influxRef: Option[ActorRef], case _ => //Do nothing } case DataFromPeer(message, remote) => message match { - case SyncInfoNetworkMessage(syncInfo) => Option(history) match { - case Some(historyReader) => - val ext: Seq[ModifierId] = historyReader.continuationIds(syncInfo, settings.network.syncPacketLength) - val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) - logger.info(s"Comparison with $remote having starting points ${idsToString(syncInfo.startingPoints)}. " + - s"Comparison result is $comparison. Sending extension of length ${ext.length}.") - if (!(ext.nonEmpty || comparison != Younger)) logger.warn("Extension is empty while comparison is younger") - deliveryManager ! OtherNodeSyncingStatus(remote, comparison, Some(ext.map(h => Header.modifierTypeId -> h))) - peersKeeper ! OtherNodeSyncingStatus(remote, comparison, Some(ext.map(h => Header.modifierTypeId -> h))) - case _ => - } + case RequestModifiersNetworkMessage((typeId, requestedIds)) if chainSynced || settings.node.offlineGeneration => val modifiersFromCache: Map[ModifierId, Array[Byte]] = requestedIds .flatMap(id => modifiersRequestCache @@ -170,9 +160,7 @@ class NodeViewSynchronizer(influxRef: Option[ActorRef], case InvNetworkMessage(invData) if invData._1 == Transaction.modifierTypeId => logger.debug(s"Get inv with tx: ${invData._2.map(Algos.encode).mkString(",")}, but " + s"chainSynced is $chainSynced and canProcessTransactions is $canProcessTransactions.") - case InvNetworkMessage(invData) if invData._1 == Payload.modifierTypeId && !history.isFullChainSynced => - logger.info(s"Got inv message with payloads: ${invData._2.map(Algos.encode).mkString(",")}. " + - s"But full chain is not synced. Ignore them.") + case InvNetworkMessage(invData) => logger.debug(s"Got inv message on NodeViewSynchronizer from ${remote.socketAddress} with modifiers of type:" + s" $invData._1. Size of inv is: ${invData._2.size}. Sending CompareViews to NVH. " + @@ -264,7 +252,7 @@ object NodeViewSynchronizer { case object SendLocalSyncInfo - final case class OtherNodeSyncingStatus(remote: ConnectedPeer, + final case class OtherNodeSyncingStatus(remote: InetSocketAddress, status: encry.consensus.HistoryConsensus.HistoryComparisonResult, extension: Option[Seq[(ModifierTypeId, ModifierId)]]) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 53a0907504..1f62613ee5 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -6,10 +6,12 @@ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } +import encry.network.DownloadedModifiersValidator.InvalidModifier +import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NetworkRouter.ModifierFromNetwork import encry.network.NodeViewSynchronizer.ReceivableMessages.{ + OtherNodeSyncingStatus, RollbackFailed, RollbackSucceed, SemanticallyFailedModification, @@ -30,7 +32,6 @@ import encry.view.fast.sync.SnapshotHolder.{ import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.ModifierId class IntermediaryNVH( settings: EncryAppSettings, @@ -65,6 +66,8 @@ class IntermediaryNVH( case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg case msg @ FastSyncDone => networkMessagesProcessor ! msg case msg @ DownloadRequest(_, _, _) => networkMessagesProcessor ! msg + case msg @ OtherNodeSyncingStatus(_, _, _) => networkMessagesProcessor ! msg + case msg @ RequestFromLocal(_, _, _) => networkMessagesProcessor ! msg case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync case msg @ TreeChunks(_, _) => //+ to fast sync case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg @@ -73,7 +76,7 @@ class IntermediaryNVH( case msg @ StartMining => //+ to miner case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder case msg @ RolledBackTransactions(_) => //+ to memory pool - case msg: StatsSenderMessage => //+ to stats sender + case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) case msg @ RollbackSucceed(_) => case msg @ RollbackFailed(_) => case msg @ SemanticallySuccessfulModifier(_) => diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 6cb0c78b59..f5038bf53d 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -1,9 +1,70 @@ package encry.nvg import akka.actor.{ Actor, Props } +import com.typesafe.scalalogging.StrictLogging +import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus +import encry.settings.EncryAppSettings +import encry.utils.Utils.idsToString +import encry.view.ModifiersCache +import encry.view.history.HistoryReader +import org.encryfoundation.common.modifiers.history.{ Header, Payload } +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.network.BasicMessagesRepo.{ + InvNetworkMessage, + RequestModifiersNetworkMessage, + SyncInfoNetworkMessage +} +import org.encryfoundation.common.network.SyncInfo +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } + +import scala.collection.Seq + +class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { + + var historyReader: HistoryReader = HistoryReader.empty + + override def receive: Receive = { + case DataFromPeer(message, remote) => + message match { + case SyncInfoNetworkMessage(syncInfo: SyncInfo) => + val ext: Seq[ModifierId] = historyReader.continuationIds(syncInfo, settings.network.syncPacketLength) + val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) + logger.info( + s"Comparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}. " + + s"Comparison result is $comparison. Sending extension of length ${ext.length}." + ) + if (!(ext.nonEmpty || comparison != Younger)) logger.warn("Extension is empty while comparison is younger") + context.parent ! OtherNodeSyncingStatus(remote, comparison, Some(ext.map(h => Header.modifierTypeId -> h))) + + case InvNetworkMessage(invData) if invData._1 == Payload.modifierTypeId && !historyReader.isFullChainSynced => + logger.info( + s"Got inv message with payloads: ${invData._2.map(Algos.encode).mkString(",")}. " + + s"But full chain is not synced. Ignore them." + ) + + case InvNetworkMessage(invData) => + val startTime: Long = System.currentTimeMillis() + val ids: Seq[ModifierId] = invData._2.filterNot( + (mid: ModifierId) => + historyReader.isModifierDefined(mid) || ModifiersCache.contains(NodeViewHolder.toKey(mid)) + ) + logger.info( + s"Got inv message from $remote. Type of nested ids is: ${invData._1}. " + + s"Ids for request are: ${ids.map(Algos.encode).mkString(",")}." + ) + if (ids.nonEmpty && + (invData._1 == Header.modifierTypeId || + (historyReader.isHeadersChainSyncedVar && invData._1 == Payload.modifierTypeId))) + sender() ! RequestFromLocal(remote, invData._1, ids.toList) + logger.info(s"Time of processing inv message is: ${(System.currentTimeMillis() - startTime) / 1000}s.") -class NetworkMessagesProcessor extends Actor { - override def receive: Receive = ??? + case RequestModifiersNetworkMessage(data) => + } + } } object NetworkMessagesProcessor { diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index da205eb964..1250d39c9e 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -73,7 +73,7 @@ class NodeViewHolder( override def receive: Receive = { case ValidatedModifier(modifier: PersistentModifier) => val startTime: Long = System.currentTimeMillis() - val wrappedKey: mutable.WrappedArray.ofByte = toKey(modifier.id) + val wrappedKey: mutable.WrappedArray.ofByte = NodeViewHolder.toKey(modifier.id) val isInHistory: Boolean = nodeView.history.isModifierDefined(modifier.id) val isInCache: Boolean = ModifiersCache.contains(wrappedKey) if (isInHistory || isInCache) @@ -374,8 +374,6 @@ class NodeViewHolder( } } else logger.info(s"Trying to apply modifier ${modifier.encodedId} that's already in history.") - def toKey(id: ModifierId): mutable.WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) - def sendUpdatedInfoToMemoryPool(toRemove: Seq[PersistentModifier]): Unit = { val rolledBackTxs: IndexedSeq[Transaction] = toRemove .flatMap(extractTransactions) @@ -490,6 +488,8 @@ class NodeViewHolder( object NodeViewHolder { + def toKey(id: ModifierId): mutable.WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) + final case class UpdateHistoryReader(history: HistoryReader) extends AnyVal final case class NodeView(history: History, state: UtxoState, wallet: EncryWallet) diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index 67671fb731..9e3ead575b 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -7,7 +7,7 @@ import com.typesafe.scalalogging.StrictLogging import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.settings.EncryAppSettings -import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView +import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetDataFromCurrentView} import encry.view.NodeViewHolder._ import org.encryfoundation.common.modifiers.history._ import org.encryfoundation.common.modifiers.mempool.transaction.Transaction diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 7388d4af38..78e3ff6fb9 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -1,17 +1,39 @@ package encry.view.history +import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Older} +import org.encryfoundation.common.network.SyncInfo +import org.encryfoundation.common.utils.TaggedTypes.ModifierId + trait HistoryReader { def getBestHeaderHeight: Int + def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] + + def compare(si: SyncInfo): HistoryComparisonResult + + var isFullChainSynced: Boolean + + var isHeadersChainSyncedVar: Boolean = false + + def isModifierDefined(id: ModifierId): Boolean + } object HistoryReader { def empty: HistoryReader = new HistoryReader { + def isModifierDefined(id: ModifierId): Boolean = true def getBestHeaderHeight = 0 + def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty + var isFullChainSynced: Boolean = true + def compare(si: SyncInfo): HistoryComparisonResult = Older } def apply(): HistoryReader = new HistoryReader { + def isModifierDefined(id: ModifierId): Boolean = true def getBestHeaderHeight = 1 + def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty + def compare(si: SyncInfo): HistoryComparisonResult = Older + var isFullChainSynced: Boolean = true } } \ No newline at end of file From f37daf885a4b33592a8b1d4b8430b02458d6ea3b Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 3 Mar 2020 17:40:37 +0300 Subject: [PATCH 023/177] code cleaned up + added SSM/SFM to the network --- .../encry/network/NodeViewSynchronizer.scala | 6 ----- .../scala/encry/nvg/IntermediaryNVH.scala | 2 ++ .../scala/encry/view/NodeViewHolder.scala | 25 ------------------- 3 files changed, 2 insertions(+), 31 deletions(-) diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index c3e37bf98d..50cb8e865c 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -161,12 +161,6 @@ class NodeViewSynchronizer(influxRef: Option[ActorRef], logger.debug(s"Get inv with tx: ${invData._2.map(Algos.encode).mkString(",")}, but " + s"chainSynced is $chainSynced and canProcessTransactions is $canProcessTransactions.") - case InvNetworkMessage(invData) => - logger.debug(s"Got inv message on NodeViewSynchronizer from ${remote.socketAddress} with modifiers of type:" + - s" $invData._1. Size of inv is: ${invData._2.size}. Sending CompareViews to NVH. " + - s"\nModifiers in inv message are: ${invData._2.map(Algos.encode).mkString(",")}") - nodeViewHolderRef ! CompareViews(remote, invData._1, invData._2) - case _ => logger.debug(s"NodeViewSyncronyzer got invalid type of DataFromPeer message!") } case msg@RequestPeersForFirstSyncInfo => diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 1f62613ee5..a1a2989e44 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -80,7 +80,9 @@ class IntermediaryNVH( case msg @ RollbackSucceed(_) => case msg @ RollbackFailed(_) => case msg @ SemanticallySuccessfulModifier(_) => + networkMessagesProcessor ! msg case msg @ SemanticallyFailedModification(_, _) => + networkMessagesProcessor ! msg } } diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index 9e3ead575b..41fd1d535f 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -4,14 +4,10 @@ import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props} import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.settings.EncryAppSettings import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetDataFromCurrentView} import encry.view.NodeViewHolder._ -import org.encryfoundation.common.modifiers.history._ -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.Seq @@ -31,31 +27,10 @@ class NodeViewHolder(memoryPoolRef: ActorRef, case resultFuture: Future[_] => resultFuture.pipeTo(sender()) case result => sender() ! result } - case GetNodeViewChanges(history, state, _) => if (history) sender() ! ChangedHistory(nodeView.history) if (state) sender() ! ChangedState(nodeView.state) - case CompareViews(peer, modifierTypeId, modifierIds) => - logger.info(s"Start processing CompareViews message on NVH.") - val startTime = System.currentTimeMillis() - val ids: Seq[ModifierId] = modifierTypeId match { - case _ => - modifierIds - .filterNot(mid => nodeView.history.isModifierDefined(mid) || ModifiersCache.contains(key(mid))) - } - if (modifierTypeId != Transaction.modifierTypeId) - logger.debug( - s"Got compare view message on NVH from ${peer.socketAddress}." + - s" Type of requesting modifiers is: $modifierTypeId. Requesting ids size are: ${ids.size}." + - s" Sending RequestFromLocal with ids to $sender." + - s"\n Requesting ids are: ${ids.map(Algos.encode).mkString(",")}." - ) - if (ids.nonEmpty && (modifierTypeId == Header.modifierTypeId || (nodeView.history.isHeadersChainSynced && modifierTypeId == Payload.modifierTypeId))) - sender() ! RequestFromLocal(peer, modifierTypeId, ids) - logger.debug( - s"Time processing of msg CompareViews from $sender with modTypeId $modifierTypeId: ${System.currentTimeMillis() - startTime}" - ) case SemanticallySuccessfulModifier(_) => case msg => logger.error(s"Got strange message on nvh: $msg") } From 79e1bd4e1424bd7976ea06173abf6b7e77f48a80 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 3 Mar 2020 18:04:40 +0300 Subject: [PATCH 024/177] add handler for modifierNetMsg --- src/main/scala/encry/network/DM.scala | 3 ++- src/main/scala/encry/network/NetworkRouter.scala | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index f92510536b..deaebb9cac 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -7,7 +7,7 @@ import com.typesafe.scalalogging.StrictLogging import encry.network.DM.{AwaitingRequest, RequestSent} import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkRouter.ModifierFromNetwork -import encry.network.NodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier +import encry.network.NodeViewSynchronizer.ReceivableMessages.{SemanticallyFailedModification, SemanticallySuccessfulModifier} import encry.settings.NetworkSettings import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} @@ -40,6 +40,7 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging context.parent ! ModifierFromNetwork(source, modTypeId, modId, modBytes) } else logger.info(s"Peer $source sent spam mod of type $modTypeId and id ${Algos.encode(modId)}") case SemanticallySuccessfulModifier(mod) => receivedModifier -= toKey(mod.id) + case SemanticallyFailedModification(mod, _) => receivedModifier -= toKey(mod.id) } def toKey(id: ModifierId): ModifierIdAsKey = new mutable.WrappedArray.ofByte(id) diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 1b6ab8e4cb..f815b1db88 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -10,6 +10,7 @@ import com.typesafe.scalalogging.StrictLogging import encry.network.BlackList.BanReason.InvalidNetworkMessage import encry.network.Messages.MessageToNetwork import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} +import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionVerified, NewConnection, OutgoingConnectionFailed} @@ -24,6 +25,7 @@ class NetworkRouter(settings: NetworkSettings, import context.system var messagesHandlers: Map[Seq[Byte], ActorRef] = Map.empty + var handlerForMods: ActorRef = ActorRef.noSender IO(Tcp) ! Bind(self, settings.bindAddress, options = KeepAlive(true) :: Nil, pullMode = false) @@ -37,6 +39,7 @@ class NetworkRouter(settings: NetworkSettings, val ids = types.map(_._1) messagesHandlers += (ids -> handler) case CommandFailed(cmd: Tcp.Command) => logger.info(s"Failed to execute: $cmd.") + case RegisterForModsHandling => handlerForMods = sender() case msg => logger.warn(s"NetworkController: got something strange $msg.") } @@ -55,6 +58,7 @@ class NetworkRouter(settings: NetworkSettings, case MessageFromNetwork(message, Some(remote)) => peersKeeper ! BanPeer(remote.socketAddress, InvalidNetworkMessage(message.messageName)) logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") + case msg: ModifierFromNetwork => handlerForMods ! msg case msg: MessageToNetwork => context.system.actorOf(MessageBuilder.props(msg, peersKeeper), "peersKeeper") } @@ -94,7 +98,7 @@ class NetworkRouter(settings: NetworkSettings, mH: Map[Seq[Byte], ActorRef]): Unit = mH.find(_._1.contains(messageId)).map(_._2) match { case Some(handler) => - handler ! DataFromPeer(message, remote) + handler ! DataFromPeer(message, remote.socketAddress) logger.debug(s"Send message DataFromPeer with ${message.messageName} to $handler.") case None => logger.info("No handlers found for message: " + message.messageName) } @@ -107,5 +111,7 @@ object NetworkRouter { modId: ModifierId, modBytes: Array[Byte]) + case object RegisterForModsHandling + def props(settings: NetworkSettings): Props = Props(new NetworkRouter(settings)) } From fef45611d601496bfc3e69840edef820aabc2442 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 12:06:47 +0300 Subject: [PATCH 025/177] network messages improved some additions --- .../encry/api/http/DataHolderForApi.scala | 4 +- .../cli/commands/CreateAccountFromSeed.scala | 1 - src/main/scala/encry/local/miner/Miner.scala | 4 +- .../scala/encry/network/DeliveryManager.scala | 8 +- .../DownloadedModifiersValidator.scala | 1 - .../encry/network/NodeViewSynchronizer.scala | 28 ---- .../scala/encry/nvg/IntermediaryNVH.scala | 28 +++- .../encry/nvg/NetworkMessagesProcessor.scala | 27 ++- src/main/scala/encry/nvg/NodeViewHolder.scala | 1 + .../scala/encry/view/NodeViewHolder.scala | 28 +--- .../encry/view/fast/sync/SnapshotHolder.scala | 154 +++++++++--------- .../encry/view/history/HistoryReader.scala | 4 + 12 files changed, 143 insertions(+), 145 deletions(-) diff --git a/src/main/scala/encry/api/http/DataHolderForApi.scala b/src/main/scala/encry/api/http/DataHolderForApi.scala index 72adcec939..7faaf529c0 100644 --- a/src/main/scala/encry/api/http/DataHolderForApi.scala +++ b/src/main/scala/encry/api/http/DataHolderForApi.scala @@ -22,7 +22,7 @@ import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeerFromAPI import encry.settings.EncryAppSettings import encry.utils.{NetworkTime, NetworkTimeProvider} -import encry.view.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, GetDataFromCurrentView} +import encry.view.NodeViewHolder.ReceivableMessages.{ GetDataFromCurrentView} import encry.view.history.History import encry.view.state.{UtxoState, UtxoStateReader} import encry.view.wallet.EncryWallet @@ -238,7 +238,7 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) sender() ! history.toList.flatMap(_.headerIdsAtHeight(i).map(Algos.encode)) case CreateAccountManagerFromSeedHelper(seed) => - (nvhRef ? CreateAccountManagerFromSeed(seed)).mapTo[Either[String, EncryWallet]].pipeTo(sender()) + //(nvhRef ? CreateAccountManagerFromSeed(seed)).mapTo[Either[String, EncryWallet]].pipeTo(sender()) case GetAllInfoHelper => diff --git a/src/main/scala/encry/cli/commands/CreateAccountFromSeed.scala b/src/main/scala/encry/cli/commands/CreateAccountFromSeed.scala index 110580e90a..4a566af27c 100644 --- a/src/main/scala/encry/cli/commands/CreateAccountFromSeed.scala +++ b/src/main/scala/encry/cli/commands/CreateAccountFromSeed.scala @@ -8,7 +8,6 @@ import encry.settings.EncryAppSettings import encry.EncryApp._ import encry.api.http.DataHolderForApi.CreateAccountManagerFromSeedHelper import encry.utils.NetworkTimeProvider -import encry.view.NodeViewHolder.ReceivableMessages.CreateAccountManagerFromSeed import encry.view.wallet.EncryWallet import scala.concurrent.Future diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index 68ca34bbe3..9bc8bde878 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -19,7 +19,7 @@ import encry.stats.StatsSender._ import encry.utils.NetworkTime.Time import encry.view.state.avlTree.utils.implicits.Instances._ import encry.view.NodeViewHolder.CurrentView -import encry.view.NodeViewHolder.ReceivableMessages.{GetDataFromCurrentView, LocallyGeneratedModifier} +import encry.view.NodeViewHolder.ReceivableMessages.{GetDataFromCurrentView} import encry.view.history.History import encry.view.mempool.MemoryPool.TransactionsForMiner import encry.view.state.UtxoState @@ -111,7 +111,7 @@ class Miner(dataHolder: ActorRef, s" from worker $workerIdx with nonce: ${block.header.nonce}.") logger.debug(s"Set previousSelfMinedBlockId: ${Algos.encode(block.id)}") killAllWorkers() - context.actorSelection("/user/nodeViewHolder") ! LocallyGeneratedModifier(block) + //context.actorSelection("/user/nodeViewHolder") ! LocallyGeneratedModifier(block) if (settings.influxDB.isDefined) { context.actorSelection("/user/statsSender") ! MiningEnd(block.header, workerIdx, context.children.size) context.actorSelection("/user/statsSender") ! MiningTime(System.currentTimeMillis() - startTime) diff --git a/src/main/scala/encry/network/DeliveryManager.scala b/src/main/scala/encry/network/DeliveryManager.scala index a1f70e1041..c5219ba652 100644 --- a/src/main/scala/encry/network/DeliveryManager.scala +++ b/src/main/scala/encry/network/DeliveryManager.scala @@ -1,6 +1,7 @@ package encry.network import java.net.InetSocketAddress + import akka.actor.{Actor, ActorRef, ActorSystem, Cancellable, PoisonPill, Props} import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus._ @@ -12,6 +13,7 @@ import encry.network.PeerConnectionHandler._ import encry.stats.StatsSender.{GetModifiers, SendDownloadRequest, SerializedModifierFromNetwork} import encry.view.history.History import encry.settings.EncryAppSettings + import scala.concurrent.duration._ import scala.collection.immutable.HashSet import scala.collection.{IndexedSeq, mutable} @@ -19,18 +21,20 @@ import scala.util.Random import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import com.typesafe.config.Config import encry.network.DownloadedModifiersValidator.{InvalidModifier, ModifiersForValidating} +import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.PeersKeeper._ import encry.network.PrioritiesCalculator.AccumulatedPeersStatistic import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus.BadNode -import encry.view.NodeViewHolder.DownloadRequest -import encry.view.mempool.MemoryPool.{ StartTransactionsValidation, StopTransactionsValidation } +import encry.nvg.NodeViewHolder.DownloadRequest +import encry.view.mempool.MemoryPool.{StartTransactionsValidation, StopTransactionsValidation} import org.encryfoundation.common.modifiers.history.{Block, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo._ import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} + import scala.concurrent.ExecutionContextExecutor class DeliveryManager(influxRef: Option[ActorRef], diff --git a/src/main/scala/encry/network/DownloadedModifiersValidator.scala b/src/main/scala/encry/network/DownloadedModifiersValidator.scala index bfeb7c691b..70999f12b6 100644 --- a/src/main/scala/encry/network/DownloadedModifiersValidator.scala +++ b/src/main/scala/encry/network/DownloadedModifiersValidator.scala @@ -13,7 +13,6 @@ import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeer import encry.settings.EncryAppSettings import encry.stats.StatsSender.ValidatedModifierFromNetwork -import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote import encry.view.history.History import encry.view.mempool.MemoryPool.NewTransaction import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionProtoSerializer} diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index 50cb8e865c..3f11d22ae2 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -32,7 +32,6 @@ import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import scala.concurrent.duration._ import encry.network.ModifiersToNetworkUtils._ -import encry.view.NodeViewHolder.DownloadRequest import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} import encry.view.fast.sync.SnapshotHolder import encry.view.fast.sync.SnapshotHolder.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks, UpdateSnapshot} @@ -126,32 +125,8 @@ class NodeViewSynchronizer(influxRef: Option[ActorRef], if (unrequestedModifiers.nonEmpty) typeId match { case Transaction.modifierTypeId => memoryPoolRef ! RequestModifiersForTransactions(remote, unrequestedModifiers) - case Payload.modifierTypeId => - getModsForRemote(unrequestedModifiers).foreach(_.foreach { - case (id, bytes) => - remote.handlerRef ! ModifiersNetworkMessage(typeId -> Map(id -> bytes)) - }) - case tId => getModsForRemote(unrequestedModifiers).foreach { modifiers => - modifiers.foreach(k => - logger.debug(s"Response to ${remote.socketAddress} header ${ - Try(HeaderProtoSerializer.fromProto(HeaderProtoMessage.parseFrom(k._2))) - }") - ) - remote.handlerRef ! ModifiersNetworkMessage(tId -> modifiers) - } } - def getModsForRemote(ids: Seq[ModifierId]): Option[Map[ModifierId, Array[Byte]]] = Option(history) - .map { historyStorage => - val modifiers: Map[ModifierId, Array[Byte]] = unrequestedModifiers - .view - .map(id => id -> historyStorage.modifierBytesById(id)) - .collect { case (id, mod) if mod.isDefined => id -> mod.get} - .toMap - logger.debug(s"Send response to $remote with ${modifiers.size} modifiers of type $typeId") - modifiers - } - case RequestModifiersNetworkMessage(requestedIds) => logger.info(s"Request from $remote for ${requestedIds._2.size} modifiers discarded cause to chain isn't synced") @@ -167,8 +142,6 @@ class NodeViewSynchronizer(influxRef: Option[ActorRef], logger.info(s"NodeViewSyncronizer got request from delivery manager to peers keeper for" + s" peers for first sync info message. Resending $msg to peers keeper.") peersKeeper ! msg - case msg@RequestFromLocal(_, _, _) => deliveryManager ! msg - case msg@DownloadRequest(_, _, _) => deliveryManager ! msg case msg@UpdatedPeersCollection(_) => deliveryManager ! msg case msg@PeersForSyncInfo(_) => logger.info(s"NodeViewSync got peers for sync info. Sending them to DM.") @@ -294,7 +267,6 @@ object NodeViewSynchronizer { class NodeViewSynchronizerPriorityQueue(settings: ActorSystem.Settings, config: Config) extends UnboundedStablePriorityMailbox( PriorityGenerator { - case RequestFromLocal(_, _, _) => 0 case DataFromPeer(msg, _) => msg match { case SyncInfoNetworkMessage(_) => 1 diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index a1a2989e44..2938226069 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -8,7 +8,7 @@ import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.DownloadedModifiersValidator.InvalidModifier import encry.network.Messages.MessageToNetwork.RequestFromLocal -import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } import encry.network.NetworkRouter.ModifierFromNetwork import encry.network.NodeViewSynchronizer.ReceivableMessages.{ OtherNodeSyncingStatus, @@ -31,6 +31,12 @@ import encry.view.fast.sync.SnapshotHolder.{ } import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions +import org.encryfoundation.common.network.BasicMessagesRepo.{ + InvNetworkMessage, + ModifiersNetworkMessage, + RequestModifiersNetworkMessage, + SyncInfoNetworkMessage +} import org.encryfoundation.common.utils.Algos class IntermediaryNVH( @@ -41,8 +47,19 @@ class IntermediaryNVH( ) extends Actor with StrictLogging { + intermediaryNetwork ! RegisterMessagesHandler( + Seq( + InvNetworkMessage.NetworkMessageTypeID -> "InvNetworkMessage", + RequestModifiersNetworkMessage.NetworkMessageTypeID -> "RequestModifiersNetworkMessage", + SyncInfoNetworkMessage.NetworkMessageTypeID -> "SyncInfoNetworkMessage" + ), + self + ) + + //todo add registration for ModifierFromNetwork msg + val networkMessagesProcessor: ActorRef = - context.actorOf(NetworkMessagesProcessor.props, name = "Network-messages-processor") + context.actorOf(NetworkMessagesProcessor.props(settings), name = "Network-messages-processor") val nodeViewHolder: ActorRef = context.actorOf(NodeViewHolder.props(settings, timeProvider, influxRef), name = "Node-view-holder") val modifiersValidatorRouter: ActorRef = @@ -68,6 +85,7 @@ class IntermediaryNVH( case msg @ DownloadRequest(_, _, _) => networkMessagesProcessor ! msg case msg @ OtherNodeSyncingStatus(_, _, _) => networkMessagesProcessor ! msg case msg @ RequestFromLocal(_, _, _) => networkMessagesProcessor ! msg + case msg @ ModifiersNetworkMessage(_, _) => networkMessagesProcessor ! msg case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync case msg @ TreeChunks(_, _) => //+ to fast sync case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg @@ -79,10 +97,8 @@ class IntermediaryNVH( case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) case msg @ RollbackSucceed(_) => case msg @ RollbackFailed(_) => - case msg @ SemanticallySuccessfulModifier(_) => - networkMessagesProcessor ! msg - case msg @ SemanticallyFailedModification(_, _) => - networkMessagesProcessor ! msg + case msg @ SemanticallySuccessfulModifier(_) => networkMessagesProcessor ! msg + case msg @ SemanticallyFailedModification(_, _) => networkMessagesProcessor ! msg } } diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index f5038bf53d..ca48059fb6 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -11,15 +11,15 @@ import encry.utils.Utils.idsToString import encry.view.ModifiersCache import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.history.{ Header, Payload } -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, + ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage } import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } +import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.Seq @@ -27,17 +27,20 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St var historyReader: HistoryReader = HistoryReader.empty + //todo last modifiers cache? + override def receive: Receive = { case DataFromPeer(message, remote) => message match { case SyncInfoNetworkMessage(syncInfo: SyncInfo) => - val ext: Seq[ModifierId] = historyReader.continuationIds(syncInfo, settings.network.syncPacketLength) + val ext: Seq[ModifierId] = + historyReader.continuationIds(syncInfo, settings.network.syncPacketLength) val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) logger.info( s"Comparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}. " + s"Comparison result is $comparison. Sending extension of length ${ext.length}." ) - if (!(ext.nonEmpty || comparison != Younger)) logger.warn("Extension is empty while comparison is younger") + if (!(ext.nonEmpty || comparison != Younger)) logger.info("Extension is empty while comparison is younger") context.parent ! OtherNodeSyncingStatus(remote, comparison, Some(ext.map(h => Header.modifierTypeId -> h))) case InvNetworkMessage(invData) if invData._1 == Payload.modifierTypeId && !historyReader.isFullChainSynced => @@ -62,11 +65,23 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St sender() ! RequestFromLocal(remote, invData._1, ids.toList) logger.info(s"Time of processing inv message is: ${(System.currentTimeMillis() - startTime) / 1000}s.") - case RequestModifiersNetworkMessage(data) => + case RequestModifiersNetworkMessage((typeId, requestedIds)) if typeId == Payload.modifierTypeId => + getModsForRemote(requestedIds.toList, historyReader).foreach { + case (id: ModifierId, bytes: Array[Byte]) => + context.parent ! ModifiersNetworkMessage(typeId -> Map(id -> bytes)) + } + case RequestModifiersNetworkMessage((typeId, requestedIds)) => + context.parent ! ModifiersNetworkMessage(typeId -> getModsForRemote(requestedIds.toList, historyReader)) } } + + def getModsForRemote(ids: List[ModifierId], reader: HistoryReader): Map[ModifierId, Array[Byte]] = + ids.view + .map(id => id -> reader.modifierBytesById(id)) + .collect { case (id, mod) if mod.isDefined => id -> mod.get } + .toMap } object NetworkMessagesProcessor { - def props: Props = Props(new NetworkMessagesProcessor) + def props(settings: EncryAppSettings): Props = Props(new NetworkMessagesProcessor(settings)) } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 1250d39c9e..e602315275 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -378,6 +378,7 @@ class NodeViewHolder( val rolledBackTxs: IndexedSeq[Transaction] = toRemove .flatMap(extractTransactions) .toIndexedSeq + //todo compare with toApply if (rolledBackTxs.nonEmpty) context.parent ! RolledBackTransactions(rolledBackTxs) } diff --git a/src/main/scala/encry/view/NodeViewHolder.scala b/src/main/scala/encry/view/NodeViewHolder.scala index 41fd1d535f..98fdfc965c 100644 --- a/src/main/scala/encry/view/NodeViewHolder.scala +++ b/src/main/scala/encry/view/NodeViewHolder.scala @@ -1,38 +1,25 @@ package encry.view -import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props} -import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} +import akka.actor.{ Actor, ActorRef, ActorSystem, PoisonPill, Props } +import akka.dispatch.{ PriorityGenerator, UnboundedStablePriorityMailbox } import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.network.NodeViewSynchronizer.ReceivableMessages._ +import encry.network.PeerConnectionHandler.ConnectedPeer import encry.settings.EncryAppSettings -import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetDataFromCurrentView} -import encry.view.NodeViewHolder._ -import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import encry.view.NodeViewHolder.ReceivableMessages.CompareViews +import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } import scala.collection.Seq -import scala.concurrent.Future class NodeViewHolder(memoryPoolRef: ActorRef, influxRef: Option[ActorRef], dataHolder: ActorRef, encrySettings: EncryAppSettings) extends Actor - with StrictLogging - with AutoCloseable { + with StrictLogging { override def receive: Receive = { - case GetDataFromCurrentView(f) => - f(CurrentView(nodeView.history, nodeView.state, nodeView.wallet)) match { - case resultFuture: Future[_] => resultFuture.pipeTo(sender()) - case result => sender() ! result - } - case GetNodeViewChanges(history, state, _) => - if (history) sender() ! ChangedHistory(nodeView.history) - if (state) sender() ! ChangedState(nodeView.state) - - case SemanticallySuccessfulModifier(_) => - case msg => logger.error(s"Got strange message on nvh: $msg") + case _ => } } @@ -41,7 +28,6 @@ object NodeViewHolder { case class CurrentView[HIS, MS, VL](history: HIS, state: MS, vault: VL) - object ReceivableMessages { case class GetNodeViewChanges(history: Boolean, state: Boolean, vault: Boolean) diff --git a/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala b/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala index f8b87a4df3..35e3ef98fc 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala @@ -88,52 +88,52 @@ class SnapshotHolder(settings: EncryAppSettings, responseTimeout: Option[Cancellable] ): Receive = { case DataFromPeer(message, remote) => - logger.debug(s"Snapshot holder got from ${remote.socketAddress} message ${message.NetworkMessageTypeID}.") + logger.debug(s"Snapshot holder got from ${remote} message ${message.NetworkMessageTypeID}.") message match { case ResponseManifestMessage(manifest) => logger.info( s"Got new manifest message ${Algos.encode(manifest.manifestId.toByteArray)} while processing chunks." ) - case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => - (for { - controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) - (controller, chunk) = controllerAndChunk - validChunk <- snapshotProcessor.validateChunkId(chunk) - processor = snapshotProcessor.updateCache(validChunk) - newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { - case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] - case t => t.asLeft[SnapshotProcessor] - } - } yield (newProcessor, controller)) match { - case Left(err: UnexpectedChunkMessage) => - logger.info(s"Error has occurred ${err.error} with peer $remote") - case Left(error) => - logger.info(s"Error has occurred: $error") - nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) - restartFastSync(history) - case Right((processor, controller)) - if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => - nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) - restartFastSync(history) - case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => - processor.assembleUTXOState() match { - case Right(state) => - logger.info(s"Tree is valid on Snapshot holder!") - processor.wallet.foreach { wallet: EncryWallet => - (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] - } - case _ => - nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) - restartFastSync(history).asLeft[Unit] - } - case Right((processor, controller)) => - snapshotDownloadController = controller - snapshotProcessor = processor - if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks - } +// case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => +// (for { +// controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) +// (controller, chunk) = controllerAndChunk +// validChunk <- snapshotProcessor.validateChunkId(chunk) +// processor = snapshotProcessor.updateCache(validChunk) +// newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { +// case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] +// case t => t.asLeft[SnapshotProcessor] +// } +// } yield (newProcessor, controller)) match { +// case Left(err: UnexpectedChunkMessage) => +// logger.info(s"Error has occurred ${err.error} with peer $remote") +// case Left(error) => +// logger.info(s"Error has occurred: $error") +// nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) +// restartFastSync(history) +// case Right((processor, controller)) +// if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => +// nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) +// restartFastSync(history) +// case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => +// processor.assembleUTXOState() match { +// case Right(state) => +// logger.info(s"Tree is valid on Snapshot holder!") +// processor.wallet.foreach { wallet: EncryWallet => +// (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] +// } +// case _ => +// nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) +// restartFastSync(history).asLeft[Unit] +// } +// case Right((processor, controller)) => +// snapshotDownloadController = controller +// snapshotProcessor = processor +// if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks +// } case ResponseChunkMessage(_) => - logger.info(s"Received chunk from unexpected peer ${remote.socketAddress}") + logger.info(s"Received chunk from unexpected peer ${remote}") case _ => } @@ -211,39 +211,39 @@ class SnapshotHolder(settings: EncryAppSettings, context.become(awaitManifestMod(newScheduler.some, history).orElse(commonMessages)) case DataFromPeer(message, remote) => - message match { - case ResponseManifestMessage(manifest) => - val isValidManifest: Boolean = - snapshotDownloadController.checkManifestValidity(manifest.manifestId.toByteArray, history) - val canBeProcessed: Boolean = snapshotDownloadController.canNewManifestBeProcessed - if (isValidManifest && canBeProcessed) { - (for { - controller <- snapshotDownloadController.processManifest(manifest, remote, history) - processor <- snapshotProcessor.initializeApplicableChunksCache( - history, - snapshotDownloadController.requiredManifestHeight - ) - } yield (controller, processor)) match { - case Left(error) => - nodeViewSynchronizer ! BanPeer(remote, InvalidResponseManifestMessage(error.error)) - case Right((controller, processor)) => - logger.debug(s"Request manifest message successfully processed.") - responseManifestTimeout.foreach(_.cancel()) - snapshotDownloadController = controller - snapshotProcessor = processor - self ! RequestNextChunks - logger.debug("Manifest processed successfully.") - context.become(fastSyncMod(history, none)) - } - } else if (!isValidManifest) { - logger.info(s"Got manifest with invalid id ${Algos.encode(manifest.manifestId.toByteArray)}") - nodeViewSynchronizer ! BanPeer( - remote, - InvalidResponseManifestMessage(s"Invalid manifest id ${Algos.encode(manifest.manifestId.toByteArray)}") - ) - } else logger.info(s"Doesn't need to process new manifest.") - case _ => - } +// message match { +// case ResponseManifestMessage(manifest) => +// val isValidManifest: Boolean = +// snapshotDownloadController.checkManifestValidity(manifest.manifestId.toByteArray, history) +// val canBeProcessed: Boolean = snapshotDownloadController.canNewManifestBeProcessed +// if (isValidManifest && canBeProcessed) { +// (for { +// controller <- snapshotDownloadController.processManifest(manifest, remote, history) +// processor <- snapshotProcessor.initializeApplicableChunksCache( +// history, +// snapshotDownloadController.requiredManifestHeight +// ) +// } yield (controller, processor)) match { +// case Left(error) => +// nodeViewSynchronizer ! BanPeer(remote, InvalidResponseManifestMessage(error.error)) +// case Right((controller, processor)) => +// logger.debug(s"Request manifest message successfully processed.") +// responseManifestTimeout.foreach(_.cancel()) +// snapshotDownloadController = controller +// snapshotProcessor = processor +// self ! RequestNextChunks +// logger.debug("Manifest processed successfully.") +// context.become(fastSyncMod(history, none)) +// } +// } else if (!isValidManifest) { +// logger.info(s"Got manifest with invalid id ${Algos.encode(manifest.manifestId.toByteArray)}") +// nodeViewSynchronizer ! BanPeer( +// remote, +// InvalidResponseManifestMessage(s"Invalid manifest id ${Algos.encode(manifest.manifestId.toByteArray)}") +// ) +// } else logger.info(s"Doesn't need to process new manifest.") +// case _ => +// } case msg @ RequiredManifestHeightAndId(_, _) => self ! msg @@ -278,19 +278,21 @@ class SnapshotHolder(settings: EncryAppSettings, if requestsProcessor.canBeProcessed(snapshotProcessor, requiredManifestId) => snapshotProcessor.actualManifest.foreach { m => logger.info(s"Sent to remote actual manifest with id ${Algos.encode(requiredManifestId)}") - remote.handlerRef ! ResponseManifestMessage(SnapshotManifestSerializer.toProto(m)) + //remote.handlerRef ! ResponseManifestMessage(SnapshotManifestSerializer.toProto(m)) } case RequestManifestMessage(manifest) => logger.debug(s"Got request for manifest with ${Algos.encode(manifest)}") - case RequestChunkMessage(chunkId) if requestsProcessor.canProcessRequest(remote) => + case RequestChunkMessage(chunkId) + //if requestsProcessor.canProcessRequest(remote) + => logger.debug(s"Got RequestChunkMessage. Current handledRequests ${requestsProcessor.handledRequests}.") val chunkFromDB: Option[SnapshotChunkMessage] = snapshotProcessor.getChunkById(chunkId) chunkFromDB.foreach { chunk => logger.debug(s"Sent to $remote chunk $chunk.") val networkMessage: NetworkMessage = ResponseChunkMessage(chunk) - remote.handlerRef ! networkMessage + //remote.handlerRef ! networkMessage } - requestsProcessor = requestsProcessor.processRequest(remote) + //requestsProcessor = requestsProcessor.processRequest(remote) case RequestChunkMessage(_) => case _ => } diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 78e3ff6fb9..54eefce65b 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -18,6 +18,8 @@ trait HistoryReader { def isModifierDefined(id: ModifierId): Boolean + def modifierBytesById(id: ModifierId): Option[Array[Byte]] + } object HistoryReader { @@ -27,6 +29,7 @@ object HistoryReader { def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty var isFullChainSynced: Boolean = true def compare(si: SyncInfo): HistoryComparisonResult = Older + def modifierBytesById(id: ModifierId): Option[Array[Byte]] = None } def apply(): HistoryReader = new HistoryReader { @@ -35,5 +38,6 @@ object HistoryReader { def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty def compare(si: SyncInfo): HistoryComparisonResult = Older var isFullChainSynced: Boolean = true + def modifierBytesById(id: ModifierId): Option[Array[Byte]] = None } } \ No newline at end of file From b52a1617642e54383aaf299f367b80066c327b8d Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 13:35:16 +0300 Subject: [PATCH 026/177] added RequestModifiersNetworkMessage processing --- .../scala/encry/nvg/IntermediaryNVH.scala | 43 +++++++++-------- .../encry/nvg/NetworkMessagesProcessor.scala | 48 +++++++++++++++---- 2 files changed, 61 insertions(+), 30 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 2938226069..10eaad95a9 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -17,7 +17,7 @@ import encry.network.NodeViewSynchronizer.ReceivableMessages.{ SemanticallyFailedModification, SemanticallySuccessfulModifier } -import encry.network.PeersKeeper.BanPeer +import encry.network.PeersKeeper.{ BanPeer, SendToNetwork } import encry.nvg.ModifiersValidator.ModifierForValidation import encry.nvg.NodeViewHolder.{ DownloadRequest, UpdateHistoryReader } import encry.settings.EncryAppSettings @@ -79,25 +79,28 @@ class IntermediaryNVH( case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! newReader - case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg - case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg - case msg @ FastSyncDone => networkMessagesProcessor ! msg - case msg @ DownloadRequest(_, _, _) => networkMessagesProcessor ! msg - case msg @ OtherNodeSyncingStatus(_, _, _) => networkMessagesProcessor ! msg - case msg @ RequestFromLocal(_, _, _) => networkMessagesProcessor ! msg - case msg @ ModifiersNetworkMessage(_, _) => networkMessagesProcessor ! msg - case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync - case msg @ TreeChunks(_, _) => //+ to fast sync - case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg - case msg @ FullBlockChainIsSynced => networkMessagesProcessor ! msg //+ to miner - case msg @ DisableMining => //+ to miner - case msg @ StartMining => //+ to miner - case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder - case msg @ RolledBackTransactions(_) => //+ to memory pool - case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) - case msg @ RollbackSucceed(_) => - case msg @ RollbackFailed(_) => - case msg @ SemanticallySuccessfulModifier(_) => networkMessagesProcessor ! msg + case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg + case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg + case msg @ FastSyncDone => networkMessagesProcessor ! msg + case msg @ DownloadRequest(_, _, _) => networkMessagesProcessor ! msg + case msg @ OtherNodeSyncingStatus(_, _, _) => networkMessagesProcessor ! msg + case msg @ RequestFromLocal(_, _, _) => networkMessagesProcessor ! msg + case msg @ ModifiersNetworkMessage(_, _) => networkMessagesProcessor ! msg + case msg @ SendToNetwork(_, _) => networkMessagesProcessor ! msg + case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync + case msg @ TreeChunks(_, _) => //+ to fast sync + case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg + case msg @ FullBlockChainIsSynced => networkMessagesProcessor ! msg //+ to miner + case msg @ DisableMining => //+ to miner + case msg @ StartMining => //+ to miner + case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder + case msg @ RolledBackTransactions(_) => //+ to memory pool + case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) + case msg @ RollbackSucceed(_) => + case msg @ RollbackFailed(_) => + case msg @ SemanticallySuccessfulModifier(_) => + networkMessagesProcessor ! msg + networkMessagesProcessor ! msg case msg @ SemanticallyFailedModification(_, _) => networkMessagesProcessor ! msg } } diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index ca48059fb6..8f351f5282 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -3,14 +3,18 @@ package encry.nvg import akka.actor.{ Actor, Props } import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } +import encry.network.Broadcast import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus +import encry.network.NodeViewSynchronizer.ReceivableMessages.{ OtherNodeSyncingStatus, SemanticallySuccessfulModifier } +import encry.network.PeersKeeper.SendToNetwork import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.ModifiersCache import encry.view.history.HistoryReader -import org.encryfoundation.common.modifiers.history.{ Header, Payload } +import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, ModifiersNetworkMessage, @@ -21,15 +25,25 @@ import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId -import scala.collection.Seq - class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { var historyReader: HistoryReader = HistoryReader.empty - //todo last modifiers cache? + var modifiersRequestCache: Map[String, Array[Byte]] = Map.empty override def receive: Receive = { + case SemanticallySuccessfulModifier(mod) => + mod match { + case block: Block if historyReader.isFullChainSynced => + List(block.header, block.payload).foreach { mod: PersistentModifier => + logger.info(s"Going to broadcast inv for modifier of type ${mod.modifierTypeId} with id: ${mod.encodedId}.") + context.parent ! SendToNetwork(InvNetworkMessage(mod.modifierTypeId -> Seq(mod.id)), Broadcast) + } + modifiersRequestCache = Map( + block.encodedId -> toProto(block.header), + block.payload.encodedId -> toProto(block.payload) + ) + } case DataFromPeer(message, remote) => message match { case SyncInfoNetworkMessage(syncInfo: SyncInfo) => @@ -66,12 +80,26 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St logger.info(s"Time of processing inv message is: ${(System.currentTimeMillis() - startTime) / 1000}s.") case RequestModifiersNetworkMessage((typeId, requestedIds)) if typeId == Payload.modifierTypeId => - getModsForRemote(requestedIds.toList, historyReader).foreach { - case (id: ModifierId, bytes: Array[Byte]) => - context.parent ! ModifiersNetworkMessage(typeId -> Map(id -> bytes)) + val modifiersFromCache: Map[ModifierId, Array[Byte]] = requestedIds + .flatMap( + (id: ModifierId) => + modifiersRequestCache + .get(Algos.encode(id)) + .map(id -> _) + ) + .toMap + if (modifiersFromCache.nonEmpty) context.parent ! ModifiersNetworkMessage(typeId -> modifiersFromCache) + val unrequestedModifiers: List[ModifierId] = requestedIds.filterNot(modifiersFromCache.contains).toList + + typeId match { + case h if h == Header.modifierTypeId => + context.parent ! ModifiersNetworkMessage(typeId -> getModsForRemote(unrequestedModifiers, historyReader)) + case _ => + getModsForRemote(unrequestedModifiers, historyReader).foreach { + case (id: ModifierId, bytes: Array[Byte]) => + context.parent ! ModifiersNetworkMessage(typeId -> Map(id -> bytes)) + } } - case RequestModifiersNetworkMessage((typeId, requestedIds)) => - context.parent ! ModifiersNetworkMessage(typeId -> getModsForRemote(requestedIds.toList, historyReader)) } } From 9e68e7b25ec938a839556ea091009dd0da86350f Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 13:52:27 +0300 Subject: [PATCH 027/177] added check payloads to download functionality --- .../scala/encry/nvg/IntermediaryNVH.scala | 2 ++ .../encry/nvg/NetworkMessagesProcessor.scala | 24 +++++++++++++++++-- .../encry/view/history/HistoryReader.scala | 6 +++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 10eaad95a9..2c32aa2284 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -19,6 +19,7 @@ import encry.network.NodeViewSynchronizer.ReceivableMessages.{ } import encry.network.PeersKeeper.{ BanPeer, SendToNetwork } import encry.nvg.ModifiersValidator.ModifierForValidation +import encry.nvg.NetworkMessagesProcessor.IdsForRequest import encry.nvg.NodeViewHolder.{ DownloadRequest, UpdateHistoryReader } import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage @@ -87,6 +88,7 @@ class IntermediaryNVH( case msg @ RequestFromLocal(_, _, _) => networkMessagesProcessor ! msg case msg @ ModifiersNetworkMessage(_, _) => networkMessagesProcessor ! msg case msg @ SendToNetwork(_, _) => networkMessagesProcessor ! msg + case msg @ IdsForRequest(_) => networkMessagesProcessor ! msg case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync case msg @ TreeChunks(_, _) => //+ to fast sync case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 8f351f5282..9cb4239a6b 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -1,14 +1,17 @@ package encry.nvg -import akka.actor.{ Actor, Props } +import akka.actor.{ Actor, Cancellable, Props } import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } import encry.network.Broadcast +import cats.syntax.option._ +import encry.network.DeliveryManager.CheckPayloadsToDownload import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.{ OtherNodeSyncingStatus, SemanticallySuccessfulModifier } import encry.network.PeersKeeper.SendToNetwork +import encry.nvg.NetworkMessagesProcessor.IdsForRequest import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.ModifiersCache @@ -27,11 +30,20 @@ import org.encryfoundation.common.utils.TaggedTypes.ModifierId class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { + import context.dispatcher + var historyReader: HistoryReader = HistoryReader.empty var modifiersRequestCache: Map[String, Array[Byte]] = Map.empty - override def receive: Receive = { + override def receive(): Receive = + workingCycle( + context.system.scheduler + .scheduleOnce(settings.network.modifierDeliverTimeCheck)(self ! CheckPayloadsToDownload) + .some + ) + + def workingCycle(modifiersRequester: Option[Cancellable]): Receive = { case SemanticallySuccessfulModifier(mod) => mod match { case block: Block if historyReader.isFullChainSynced => @@ -101,6 +113,13 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St } } } + case CheckPayloadsToDownload => + val newIds: Seq[ModifierId] = historyReader.payloadsIdsToDownload(settings.network.networkChunkSize) + logger.debug(s"newIds: ${newIds.map(elem => Algos.encode(elem)).mkString(",")}") + if (newIds.nonEmpty) context.parent ! IdsForRequest(newIds.toList) + val nextCheckModsScheduler: Cancellable = + context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)(self ! CheckPayloadsToDownload) + context.become(workingCycle(nextCheckModsScheduler.some)) } def getModsForRemote(ids: List[ModifierId], reader: HistoryReader): Map[ModifierId, Array[Byte]] = @@ -111,5 +130,6 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St } object NetworkMessagesProcessor { + final case class IdsForRequest(ids: List[ModifierId]) extends AnyVal def props(settings: EncryAppSettings): Props = Props(new NetworkMessagesProcessor(settings)) } diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 54eefce65b..3f0640ef69 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -4,6 +4,8 @@ import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Older} import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import scala.collection.immutable.HashSet + trait HistoryReader { def getBestHeaderHeight: Int @@ -20,6 +22,8 @@ trait HistoryReader { def modifierBytesById(id: ModifierId): Option[Array[Byte]] + def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] + } object HistoryReader { @@ -30,6 +34,7 @@ object HistoryReader { var isFullChainSynced: Boolean = true def compare(si: SyncInfo): HistoryComparisonResult = Older def modifierBytesById(id: ModifierId): Option[Array[Byte]] = None + def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty } def apply(): HistoryReader = new HistoryReader { @@ -39,5 +44,6 @@ object HistoryReader { def compare(si: SyncInfo): HistoryComparisonResult = Older var isFullChainSynced: Boolean = true def modifierBytesById(id: ModifierId): Option[Array[Byte]] = None + def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty } } \ No newline at end of file From 47d15113e82aa78f4bc4536b6f19962c8fa41b81 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 13:56:36 +0300 Subject: [PATCH 028/177] modifiers cache moved to the nvg package --- .../encry/{view => nvg}/ModifiersCache.scala | 6 +++--- .../encry/nvg/NetworkMessagesProcessor.scala | 17 ++++++----------- src/main/scala/encry/nvg/NodeViewHolder.scala | 19 +++++++++---------- 3 files changed, 18 insertions(+), 24 deletions(-) rename src/main/scala/encry/{view => nvg}/ModifiersCache.scala (99%) diff --git a/src/main/scala/encry/view/ModifiersCache.scala b/src/main/scala/encry/nvg/ModifiersCache.scala similarity index 99% rename from src/main/scala/encry/view/ModifiersCache.scala rename to src/main/scala/encry/nvg/ModifiersCache.scala index 67ff14c4a7..97e4d5b21c 100644 --- a/src/main/scala/encry/view/ModifiersCache.scala +++ b/src/main/scala/encry/nvg/ModifiersCache.scala @@ -1,4 +1,4 @@ -package encry.view +package encry.nvg import com.typesafe.scalalogging.StrictLogging import encry.view.history.History @@ -7,11 +7,11 @@ import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId + import scala.annotation.tailrec -import scala.collection.immutable.SortedMap import scala.collection.concurrent.TrieMap +import scala.collection.immutable.SortedMap import scala.collection.mutable -import encry.EncryApp.settings object ModifiersCache extends StrictLogging { diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 9cb4239a6b..7ed1c56f3e 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -1,29 +1,23 @@ package encry.nvg -import akka.actor.{ Actor, Cancellable, Props } +import akka.actor.{Actor, Cancellable, Props} import com.typesafe.scalalogging.StrictLogging -import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } +import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Younger} import encry.network.Broadcast import cats.syntax.option._ import encry.network.DeliveryManager.CheckPayloadsToDownload import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.NodeViewSynchronizer.ReceivableMessages.{ OtherNodeSyncingStatus, SemanticallySuccessfulModifier } +import encry.network.NodeViewSynchronizer.ReceivableMessages.{OtherNodeSyncingStatus, SemanticallySuccessfulModifier} import encry.network.PeersKeeper.SendToNetwork import encry.nvg.NetworkMessagesProcessor.IdsForRequest import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString -import encry.view.ModifiersCache import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } -import org.encryfoundation.common.network.BasicMessagesRepo.{ - InvNetworkMessage, - ModifiersNetworkMessage, - RequestModifiersNetworkMessage, - SyncInfoNetworkMessage -} +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -45,6 +39,7 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St def workingCycle(modifiersRequester: Option[Cancellable]): Receive = { case SemanticallySuccessfulModifier(mod) => + //todo possible way to call CheckPayloadsToDownload mod match { case block: Block if historyReader.isFullChainSynced => List(block.header, block.payload).foreach { mod: PersistentModifier => diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index e602315275..a8f8aae222 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -2,29 +2,28 @@ package encry.nvg import java.io.File -import akka.actor.{ Actor, ActorRef, Props } +import akka.actor.{Actor, ActorRef, Props} import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.local.miner.Miner.{DisableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.nvg.NodeViewHolder.{ DownloadRequest, NodeView, UpdateInformation } +import encry.nvg.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} +import encry.nvg.NodeViewHolder.{DownloadRequest, NodeView, UpdateInformation} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider -import encry.view.ModifiersCache import encry.view.NodeViewErrors.ModifierApplyError import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.storage.HistoryStorage -import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } +import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} import encry.view.mempool.MemoryPool.RolledBackTransactions import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree @@ -32,14 +31,14 @@ import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } +import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} -import scala.collection.{ mutable, IndexedSeq, Seq } +import scala.collection.{IndexedSeq, Seq, mutable} import scala.concurrent.duration._ -import scala.util.{ Failure, Success, Try } +import scala.util.{Failure, Success, Try} class NodeViewHolder( settings: EncryAppSettings, From 55bdbcf83d1261de5cf524998e1c681d182f9962 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 14:02:42 +0300 Subject: [PATCH 029/177] added registration for ModifierFromNetwork msg --- src/main/scala/encry/nvg/IntermediaryNVH.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 2c32aa2284..c39df85e62 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -9,7 +9,7 @@ import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.DownloadedModifiersValidator.InvalidModifier import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } -import encry.network.NetworkRouter.ModifierFromNetwork +import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } import encry.network.NodeViewSynchronizer.ReceivableMessages.{ OtherNodeSyncingStatus, RollbackFailed, @@ -57,7 +57,7 @@ class IntermediaryNVH( self ) - //todo add registration for ModifierFromNetwork msg + intermediaryNetwork ! RegisterForModsHandling val networkMessagesProcessor: ActorRef = context.actorOf(NetworkMessagesProcessor.props(settings), name = "Network-messages-processor") From d3df356c68c0dccc6ef5f431e75b6e8137317087 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 14:04:08 +0300 Subject: [PATCH 030/177] minor fix --- .../scala/encry/nvg/IntermediaryNVH.scala | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index c39df85e62..d338e100fb 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -80,19 +80,19 @@ class IntermediaryNVH( case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! newReader - case msg @ BanPeer(_, _) => networkMessagesProcessor ! msg - case msg @ InvalidModifier(_) => networkMessagesProcessor ! msg - case msg @ FastSyncDone => networkMessagesProcessor ! msg - case msg @ DownloadRequest(_, _, _) => networkMessagesProcessor ! msg - case msg @ OtherNodeSyncingStatus(_, _, _) => networkMessagesProcessor ! msg - case msg @ RequestFromLocal(_, _, _) => networkMessagesProcessor ! msg - case msg @ ModifiersNetworkMessage(_, _) => networkMessagesProcessor ! msg - case msg @ SendToNetwork(_, _) => networkMessagesProcessor ! msg - case msg @ IdsForRequest(_) => networkMessagesProcessor ! msg + case msg @ BanPeer(_, _) => intermediaryNetwork ! msg + case msg @ InvalidModifier(_) => intermediaryNetwork ! msg + case msg @ FastSyncDone => intermediaryNetwork ! msg + case msg @ DownloadRequest(_, _, _) => intermediaryNetwork ! msg + case msg @ OtherNodeSyncingStatus(_, _, _) => intermediaryNetwork ! msg + case msg @ RequestFromLocal(_, _, _) => intermediaryNetwork ! msg + case msg @ ModifiersNetworkMessage(_, _) => intermediaryNetwork ! msg + case msg @ SendToNetwork(_, _) => intermediaryNetwork ! msg + case msg @ IdsForRequest(_) => intermediaryNetwork ! msg case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync case msg @ TreeChunks(_, _) => //+ to fast sync - case msg @ HeaderChainIsSynced => networkMessagesProcessor ! msg - case msg @ FullBlockChainIsSynced => networkMessagesProcessor ! msg //+ to miner + case msg @ HeaderChainIsSynced => intermediaryNetwork ! msg + case msg @ FullBlockChainIsSynced => intermediaryNetwork ! msg //+ to miner case msg @ DisableMining => //+ to miner case msg @ StartMining => //+ to miner case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder @@ -101,9 +101,9 @@ class IntermediaryNVH( case msg @ RollbackSucceed(_) => case msg @ RollbackFailed(_) => case msg @ SemanticallySuccessfulModifier(_) => + intermediaryNetwork ! msg networkMessagesProcessor ! msg - networkMessagesProcessor ! msg - case msg @ SemanticallyFailedModification(_, _) => networkMessagesProcessor ! msg + case msg @ SemanticallyFailedModification(_, _) => intermediaryNetwork ! msg } } From 2f28e798353e2e9682dd55ea9282d940a984f61c Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 5 Mar 2020 15:12:23 +0300 Subject: [PATCH 031/177] comment old actors --- src/main/scala/encry/Starter.scala | 25 +- src/main/scala/encry/network/DM.scala | 9 +- .../scala/encry/network/DeliveryManager.scala | 967 +++++++++--------- .../DownloadedModifiersValidator.scala | 110 +- .../scala/encry/network/MessageBuilder.scala | 22 +- .../encry/network/NetworkController.scala | 224 ++-- .../scala/encry/network/NetworkRouter.scala | 16 +- .../encry/network/NodeViewSynchronizer.scala | 374 +++---- src/main/scala/encry/network/PK.scala | 29 +- .../encry/network/PeerConnectionHandler.scala | 2 +- .../scala/encry/network/PeersKeeper.scala | 570 +++++------ .../scala/encry/network/BlackListTests.scala | 194 ++-- .../network/ConnectWithNewPeerTests.scala | 408 ++++---- .../DeliveryManagerTests/DMUtils.scala | 76 +- .../DeliveryManagerPriorityTests.scala | 438 ++++---- ...DeliveryManagerReRequestModifiesSpec.scala | 374 +++---- .../DeliveryManagerRequestModifiesSpec.scala | 634 ++++++------ .../DownloadedModifiersValidatorTests.scala | 406 ++++---- .../HistoryComparisionResultTest.scala | 240 ++--- .../history/ModifiersValidationTest.scala | 90 +- 20 files changed, 2634 insertions(+), 2574 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index a7b8249851..25b5fd71e7 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -1,10 +1,11 @@ package encry import java.net.InetSocketAddress -import akka.actor.{ Actor, ActorRef } + +import akka.actor.{Actor, ActorRef} import akka.http.scaladsl.Http import cats.Functor -import cats.data.{ NonEmptyChain, Validated } +import cats.data.{NonEmptyChain, Validated} import cats.instances.future._ import cats.instances.option._ import cats.syntax.apply._ @@ -15,19 +16,20 @@ import encry.Starter.InitNodeResult import encry.api.http.DataHolderForApi import encry.api.http.DataHolderForApi.PassForStorage import encry.cli.ConsoleListener -import encry.cli.ConsoleListener.{ prompt, StartListening } +import encry.cli.ConsoleListener.{StartListening, prompt} import encry.local.miner.Miner import encry.local.miner.Miner.StartMining -import encry.network.NodeViewSynchronizer +import encry.network.{NetworkRouter, NodeViewSynchronizer} import encry.settings._ import encry.stats.StatsSender -import encry.utils.{ Mnemonic, NetworkTimeProvider } +import encry.utils.{Mnemonic, NetworkTimeProvider} import encry.view.NodeViewHolder import encry.view.mempool.MemoryPool import encry.view.wallet.AccountManager + import scala.concurrent.Future import scala.io.StdIn -import scala.util.{ Failure, Success, Try } +import scala.util.{Failure, Success, Try} class Starter(settings: EncryAppSettings, timeProvider: NetworkTimeProvider, @@ -422,9 +424,16 @@ class Starter(settings: EncryAppSettings, if (nodePass.nonEmpty) dataHolderForApi ! PassForStorage(nodePass) +// context.system.actorOf( +// NodeViewSynchronizer +// .props(influxRef, nodeViewHolder, newSettings, memoryPool, dataHolderForApi) +// .withDispatcher("nvsh-dispatcher"), +// "nodeViewSynchronizer" +// ) + context.system.actorOf( - NodeViewSynchronizer - .props(influxRef, nodeViewHolder, newSettings, memoryPool, dataHolderForApi) + NetworkRouter + .props(networkSettings, settings.blackList) .withDispatcher("nvsh-dispatcher"), "nodeViewSynchronizer" ) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index deaebb9cac..60e775f32e 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -6,9 +6,11 @@ import akka.actor.{Actor, Props} import com.typesafe.scalalogging.StrictLogging import encry.network.DM.{AwaitingRequest, RequestSent} import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.NetworkController.ReceivableMessages.RegisterMessagesHandler import encry.network.NetworkRouter.ModifierFromNetwork import encry.network.NodeViewSynchronizer.ReceivableMessages.{SemanticallyFailedModification, SemanticallySuccessfulModifier} import encry.settings.NetworkSettings +import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} @@ -23,6 +25,9 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging var expectedModifiers: Set[ModifierIdAsKey] = Set.empty var receivedModifier: Set[ModifierIdAsKey] = Set.empty + override def preStart(): Unit = + context.parent ! RegisterMessagesHandler(Seq(ModifiersNetworkMessage.NetworkMessageTypeID -> "ModifiersNetworkMessage"), self) + override def receive: Receive = { case RequestSent(peer, modTypeId, modId) => expectedModifiers += toKey(modId) @@ -50,7 +55,7 @@ object DM { case class AwaitingRequest(peer: InetSocketAddress, modTypeId: ModifierTypeId, modId: ModifierId, attempts: Int) case class RequestSent(peer: InetSocketAddress, modTypeId: ModifierTypeId, modId: ModifierId) + case class IsRequested(modifierId: ModifierId) - - def props(): Props = ??? + def props(networkSettings: NetworkSettings): Props = Props(new DM(networkSettings)) } diff --git a/src/main/scala/encry/network/DeliveryManager.scala b/src/main/scala/encry/network/DeliveryManager.scala index c5219ba652..3fcd742e84 100644 --- a/src/main/scala/encry/network/DeliveryManager.scala +++ b/src/main/scala/encry/network/DeliveryManager.scala @@ -22,6 +22,7 @@ import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import com.typesafe.config.Config import encry.network.DownloadedModifiersValidator.{InvalidModifier, ModifiersForValidating} import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.PeersKeeper.ConnectionStatusMessages.ConnectionStopped import encry.network.PeersKeeper._ import encry.network.PrioritiesCalculator.AccumulatedPeersStatistic import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus @@ -36,475 +37,475 @@ import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import scala.concurrent.ExecutionContextExecutor - -class DeliveryManager(influxRef: Option[ActorRef], - nodeViewHolderRef: ActorRef, - networkControllerRef: ActorRef, - memoryPoolRef: ActorRef, - nodeViewSync: ActorRef, - downloadedModifiersValidator: ActorRef, - settings: EncryAppSettings) extends Actor with StrictLogging { - - type ModifierIdAsKey = scala.collection.mutable.WrappedArray.ofByte - - implicit val exCon: ExecutionContextExecutor = context.dispatcher - - /** - * Collection with spam modifiers. - * Modifier considered spam if we receive it but it doesn't contain in expected modifiers collection. - */ - var receivedSpamModifiers: Map[ModifierIdAsKey, ConnectedPeer] = Map.empty - /** - * Collection of received modifiers ids. - * Modifier considered received if we sent request for it and received it in special period. - */ - var receivedModifiers: HashSet[ModifierIdAsKey] = HashSet.empty[ModifierIdAsKey] - /** - * Collection of expected modifiers ids. - * Modifier considered expected if we sent request for it. - */ - var expectedModifiers: Map[InetSocketAddress, Map[ModifierIdAsKey, (Cancellable, Int)]] = Map.empty - - var expectedTransactions: HashSet[ModifierIdAsKey] = HashSet.empty[ModifierIdAsKey] - - var peersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryComparisonResult, PeersPriorityStatus)] = Map.empty - - var priorityCalculator: PrioritiesCalculator = PrioritiesCalculator(settings.network) - - var canProcessTransactions: Boolean = true - - override def preStart(): Unit = { - networkControllerRef ! RegisterMessagesHandler( - Seq(ModifiersNetworkMessage.NetworkMessageTypeID -> "ModifiersNetworkMessage"), self) - context.system.eventStream.subscribe(self, classOf[ModificationOutcome]) - } - - override def receive: Receive = { - case UpdatedHistory(historyReader) => - logger.debug(s"Got message with history. Starting normal actor's work.") - context.system.scheduler.schedule(0.second, priorityCalculator.updatingStatisticTime) { - val (accumulatedStatistic: Map[InetSocketAddress, PeersPriorityStatus], newStat: PrioritiesCalculator) = - priorityCalculator.accumulatePeersStatistic - priorityCalculator = newStat - context.parent ! AccumulatedPeersStatistic(accumulatedStatistic) - } - val checkModsSch = context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)( - self ! CheckPayloadsToDownload - ) - nodeViewSync ! SendLocalSyncInfo - context.become(basicMessageHandler(historyReader, isBlockChainSynced = false, isMining = settings.node.mining, checkModsSch)) - case message => logger.debug(s"Got new message $message while awaiting history.") - } - - def basicMessageHandler(history: History, - isBlockChainSynced: Boolean, - isMining: Boolean, - checkModScheduler: Cancellable): Receive = { - case InvalidModifier(id) => receivedModifiers -= toKey(id) - - case CheckDelivery(peer: ConnectedPeer, modifierTypeId: ModifierTypeId, modifierId: ModifierId) => - checkDelivery(peer, modifierTypeId, modifierId) - - case UpdatedPeersCollection(newPeers) => - logger.info(s"Delivery manager got updated peers collection.") - peersCollection = newPeers - - case ConnectionStopped(peer) => - peersCollection -= peer - logger.info(s"Removed peer: $peer from peers collection on Delivery Manager." + - s" Current peers are: ${peersCollection.mkString(",")}") - - case OtherNodeSyncingStatus(remote, status, extOpt) => - status match { - case Unknown => logger.info("Peer status is still unknown.") - case Younger | Fork if isBlockChainSynced => sendInvData(remote, status, extOpt) - case _ => - } - - case CheckPayloadsToDownload => - val currentQueue: HashSet[ModifierIdAsKey] = - expectedModifiers.flatMap { case (_, modIds) => modIds.keys }.to[HashSet] - logger.debug(s"Current queue: ${currentQueue.map(elem => Algos.encode(elem.toArray)).mkString(",")}") - logger.debug(s"receivedModifiers: ${receivedModifiers.map(id => Algos.encode(id.toArray)).mkString(",")}") - logger.debug(s"Qty to req: ${settings.network.networkChunkSize - currentQueue.size - receivedModifiers.size}") - logger.debug(s"currentQueue.size: ${currentQueue.size}") - logger.debug(s"receivedModifiers.size: ${receivedModifiers.size}") - val newIds: Seq[ModifierId] = - history.payloadsIdsToDownload( - settings.network.networkChunkSize - currentQueue.size - receivedModifiers.size, - currentQueue.map(elem => ModifierId @@ elem.toArray) - ).filterNot(modId => currentQueue.contains(toKey(modId)) || receivedModifiers.contains(toKey(modId))) - logger.debug(s"newIds: ${newIds.map(elem => Algos.encode(elem)).mkString(",")}") - if (newIds.nonEmpty) requestDownload(Payload.modifierTypeId, newIds, history, isBlockChainSynced, isMining) - val nextCheckModsScheduler = - context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)(self ! CheckPayloadsToDownload) - context.become(basicMessageHandler(history, isBlockChainSynced, settings.node.mining, nextCheckModsScheduler)) - - case SemanticallySuccessfulModifier(mod) => - logger.info(s"Got SemanticallySuccessfulModifier with id: ${Algos.encode(mod.id)} of type ${mod.modifierTypeId} on dm") - mod match { - case block: Block => receivedModifiers -= toKey(block.payload.id) - case _ => receivedModifiers -= toKey(mod.id) - } - if (!isBlockChainSynced && expectedModifiers.isEmpty && receivedModifiers.isEmpty) { - checkModScheduler.cancel() - logger.debug(s"SemanticallySuccessfulModifier case, if condition true. Resend CheckPayloadsToDownload to DM") - self ! CheckPayloadsToDownload - } - case SemanticallyFailedModification(mod, _) => receivedModifiers -= toKey(mod.id) - - case SyntacticallyFailedModification(mod, _) => receivedModifiers -= toKey(mod.id) - - case SuccessfulTransaction(_) => //do nothing - - case RequestFromLocal(peer, modifierTypeId, modifierIds) => - if (modifierTypeId != Transaction.modifierTypeId) logger.debug(s"Got RequestFromLocal on NVSH from $sender with " + - s"ids of type: $modifierTypeId. Number of ids is: ${modifierIds.size}. Ids: ${modifierIds.map(Algos.encode).mkString(",")}. Sending request from local to DeliveryManager.") - if (modifierIds.nonEmpty) requestModifies(history, peer, modifierTypeId, modifierIds, isBlockChainSynced, isMining) - - case DataFromPeer(message, remote) => message match { - case ModifiersNetworkMessage((typeId, modifiers)) => - logger.debug(s"Received modifiers are: ${modifiers.map(x => Algos.encode(x._1)).mkString(",")}") - influxRef.foreach(_ ! GetModifiers(typeId, modifiers.keys.toSeq)) - for ((id, _) <- modifiers) receive(typeId, id, remote, isBlockChainSynced) - val (spam: Map[ModifierId, Array[Byte]], fm: Map[ModifierId, Array[Byte]]) = modifiers.partition(p => isSpam(p._1)) - if (spam.nonEmpty) { - if (typeId != Transaction.modifierTypeId) - logger.info(s"Spam attempt: peer $remote has sent a non-requested modifiers of type $typeId with ids" + - s": ${spam.keys.map(Algos.encode)}.") - receivedSpamModifiers = Map.empty - } - val filteredModifiers: Map[ModifierId, Array[Byte]] = fm.filterKeys(k => !history.isModifierDefined(k)) - if (typeId != Transaction.modifierTypeId) influxRef - .foreach(ref => (0 to filteredModifiers.size).foreach(_ => ref ! SerializedModifierFromNetwork(typeId))) - //todo check this logic - logger.debug(s"Type of mod: ${typeId}. canProcessTransactions: ${canProcessTransactions}") - if ((typeId == Transaction.modifierTypeId && canProcessTransactions) || (typeId != Transaction.modifierTypeId)) - downloadedModifiersValidator ! ModifiersForValidating(remote, typeId, filteredModifiers) - - case _ => logger.debug(s"DeliveryManager got invalid type of DataFromPeer message!") - } - - case DownloadRequest(modifierTypeId, modifiersId, previousModifier) => //todo check this condition - if (modifierTypeId != Transaction.modifierTypeId) - logger.info(s"DownloadRequest for mod ${Algos.encode(modifiersId)} of type: $modifierTypeId prev mod: " + - s"${previousModifier.map(Algos.encode)}") - requestDownload(modifierTypeId, Seq(modifiersId), history, isBlockChainSynced, isMining) - - case PeersForSyncInfo(peers) => sendSync(history.syncInfo, peers) - - case FullBlockChainIsSynced => context.become(basicMessageHandler(history, isBlockChainSynced = true, isMining, checkModScheduler)) - - case StartMining => context.become(basicMessageHandler(history, isBlockChainSynced, isMining = true, checkModScheduler)) - - case DisableMining => context.become(basicMessageHandler(history, isBlockChainSynced, isMining = false, checkModScheduler)) - - case UpdatedHistory(historyReader) => context.become(basicMessageHandler(historyReader, isBlockChainSynced, isMining, checkModScheduler)) - - case StopTransactionsValidation => canProcessTransactions = false - - case StartTransactionsValidation => canProcessTransactions = true - - case message => logger.debug(s"Got strange message $message(${message.getClass}) on DeliveryManager from $sender") - } - - /** - * This function check if modifier has received or not. - * If modifier has transaction type id, it won't be re-asked. - * If we still have no this modifier and number of attempts have no expired, we will re-ask it. - * If we still have no this modifier and number of attempts have expired, we will remove it from expected modifiers collection. - * Otherwise - do nothing. - * - * @param peer - peer, from whom we are expecting modifier - * @param modifierTypeId - type of expected modifier - * @param modifierId - expected modifier id - */ - def checkDelivery(peer: ConnectedPeer, modifierTypeId: ModifierTypeId, modifierId: ModifierId): Unit = { - val expectedModifiersByPeer: Map[ModifierIdAsKey, (Cancellable, Int)] = - expectedModifiers.getOrElse(peer.socketAddress, Map.empty) - if (modifierTypeId == Transaction.modifierTypeId) - expectedModifiers = clearExpectedModifiersCollection(expectedModifiersByPeer, toKey(modifierId), peer.socketAddress) - else expectedModifiersByPeer.find { case (id, (_, _)) => id == toKey(modifierId) } match { - case Some((_, (_, attempts))) if attempts <= settings.network.maxDeliveryChecks => - logger.debug(s"Modifier ${Algos.encode(modifierId)} needed to be requested from $peer!") - reRequestModifier(peer, modifierTypeId, modifierId, expectedModifiersByPeer) - case Some((modId, (_, _))) => - logger.debug(s"Maximum number of attempts has expired. Remove modifier ${Algos.encode(modifierId)} from $peer.") - expectedModifiers = clearExpectedModifiersCollection(expectedModifiersByPeer, modId, peer.socketAddress) - case _ => - logger.debug(s"This modifiers ${Algos.encode(modifierId)} is not contained in expectedModifiers collection from $peer.") - } - } - - /** - * If node is not synced, send sync info to random peer, otherwise to all known peers. - * - * @param syncInfo - sync info - */ - def sendSync(syncInfo: SyncInfo, peers: Seq[ConnectedPeer]): Unit = peers.foreach { peer => - logger.info(s"Sending to $peer sync info message.") - peer.handlerRef ! SyncInfoNetworkMessage(syncInfo) - } - - /** - * Send request to 'peer' with modifiers ids of type 'modifierTypeId'. - * We can do this activity only if 'peer' status != Younger. - * If current chain isn't synced and mining is off, we can't request transactions, otherwise can. - * - * We should filter our requesting modifiers to avoid request repeated modifiers. - * - * @param history - current history reader - * @param peer - peer, whom message will be send - * @param mTypeId - modifier type id - * @param modifierIds - modifiers ids - * @param isBlockChainSynced - current block chain status - * @param isMining - current mining status - */ - - def requestModifies(history: History, - peer: ConnectedPeer, - mTypeId: ModifierTypeId, - modifierIds: Seq[ModifierId], - isBlockChainSynced: Boolean, - isMining: Boolean): Unit = { - val firstCondition: Boolean = mTypeId == Transaction.modifierTypeId && isBlockChainSynced && isMining - val secondCondition: Boolean = mTypeId != Transaction.modifierTypeId - val thirdCondition: Boolean = - if (!isBlockChainSynced) peersCollection.get(peer.socketAddress).exists(p => p._2 != Younger) - else peersCollection.contains(peer.socketAddress) - if (mTypeId != Transaction.modifierTypeId) - logger.debug(s"Got requestModifier for modifiers of type: $mTypeId to $peer with modifiers ${modifierIds.size}." + - s" Try to check conditions: $firstCondition -> $secondCondition -> $thirdCondition.") - if ((firstCondition || secondCondition) && thirdCondition) { - val requestedModifiersFromPeer: Map[ModifierIdAsKey, (Cancellable, Int)] = expectedModifiers - .getOrElse(peer.socketAddress, Map.empty) - - val notYetRequested: Seq[ModifierId] = modifierIds - .filter(id => - !history.isModifierDefined(id) && - !requestedModifiersFromPeer.contains(toKey(id)) && - !receivedModifiers.contains(toKey(id)) - ) - - if (notYetRequested.nonEmpty) { - if (mTypeId != Transaction.modifierTypeId) - logger.debug(s"Send request to ${peer.socketAddress} for ${notYetRequested.size} modifiers of type $mTypeId ") - peer.handlerRef ! RequestModifiersNetworkMessage(mTypeId -> notYetRequested) - priorityCalculator = priorityCalculator.incrementRequestForNModifiers(peer.socketAddress, notYetRequested.size) - if (mTypeId != Transaction.modifierTypeId) { - val requestedModIds: Map[ModifierIdAsKey, (Cancellable, Int)] = - notYetRequested.foldLeft(requestedModifiersFromPeer) { case (rYet, id) => - rYet.updated(toKey(id), - context.system - .scheduler.scheduleOnce(settings.network.deliveryTimeout)(self ! CheckDelivery(peer, mTypeId, id)) -> 1) - } - expectedModifiers = expectedModifiers.updated(peer.socketAddress, requestedModIds) - } else expectedTransactions = expectedTransactions ++ modifierIds.map(toKey) - } - } - } - - /** - * Re-ask 'modifierId' from 'peer' if needed. We will do this only if we are expecting these modifier from 'peer' - * and if number of attempts doesn't expired yet. - * This activity will update timer on re-asked modifier. - * - * @param peer - peer, whom message will be send - * @param mTypeId - modifier type id - * @param modId - re-asked modifier id - */ - def reRequestModifier(peer: ConnectedPeer, - mTypeId: ModifierTypeId, - modId: ModifierId, - peerRequests: Map[ModifierIdAsKey, (Cancellable, Int)]): Unit = - peerRequests.get(toKey(modId)) match { - case Some((_, attempts)) => peersCollection.find { case (innerAddr, (_, cResult, _)) => - innerAddr == peer.socketAddress && cResult != Younger - } match { - case Some((_, (cP, _, _))) => - cP.handlerRef ! RequestModifiersNetworkMessage(mTypeId -> Seq(modId)) - logger.debug(s"Re-asked ${peer.socketAddress} and handler: ${peer.handlerRef} for modifier of type: " + - s"$mTypeId with id: ${Algos.encode(modId)}. Attempts: $attempts") - priorityCalculator = priorityCalculator.incrementRequest(peer.socketAddress) - expectedModifiers = expectedModifiers.updated(peer.socketAddress, peerRequests.updated( - toKey(modId), - context.system.scheduler - .scheduleOnce(settings.network.deliveryTimeout)(self ! CheckDelivery(peer, mTypeId, modId)) -> (attempts + 1) - )) - case None => - expectedModifiers = clearExpectedModifiersCollection(peerRequests, toKey(modId), peer.socketAddress) - logger.debug(s"Tried to re-ask modifier ${Algos.encode(modId)}, but this id not needed from this peer") - } - case _ => logger.debug(s"There is no such modifier ${Algos.encode(modId)} in expected collection.") - } - - /** - * Check 'expectedModifiers' for awaiting modifier with id 'mId' from 'peer' - * - * @param mId - id of checkable modifier - * @param peer - peer from which we possibly expecting modifier - * @return 'true' if we are expecting this modifier from this peer otherwise 'false' - */ - def isExpecting(mId: ModifierId, modifierTypeId: ModifierTypeId, peer: ConnectedPeer): Boolean = { - if (modifierTypeId != Transaction.modifierTypeId) { - val result: Boolean = expectedModifiers.getOrElse(peer.socketAddress, Map.empty).contains(toKey(mId)) - logger.debug(s"isExpecting -->> modId ${Algos.encode(mId)} --> $result") - result - } else expectedTransactions.contains(toKey(mId)) - } - - /** - * Clear the 'receivedSpamModifiers' collection - * - * @param mIds - sequence of modifiers ids which will be deleted from spam collection - */ - def deleteSpam(mIds: Seq[ModifierId]): Unit = for (id <- mIds) receivedSpamModifiers -= toKey(id) - - /** - * Check receivedSpamModifiers for contains received modifier - * - * @param mId - checkable modifier - * @return 'true' if received modifier is in spam collection otherwise 'false' - */ - def isSpam(mId: ModifierId): Boolean = receivedSpamModifiers.contains(toKey(mId)) - - /** - * Send inv data to the 'peer'. - * - * @param peer - peer whom will send a message - * @param status - current peer's status - * @param dataForInvMessage - data for inv message - */ - def sendInvData(peer: ConnectedPeer, - status: HistoryComparisonResult, - dataForInvMessage: Option[Seq[(ModifierTypeId, ModifierId)]]): Unit = dataForInvMessage match { - case Some(data) => - data.groupBy(_._1).mapValues(_.map(_._2)).foreach { - case (mTid, mods) if mods.size <= settings.network.maxInvObjects => - logger.debug(s"Send to peer $peer inv msg with mods: ${mods.map(Algos.encode).mkString(",")}") - peer.handlerRef ! InvNetworkMessage(mTid -> mods) - case (mTid, mods) => - val modifiers: Seq[ModifierId] = mods.take(settings.network.maxInvObjects) - logger.debug(s"Send to peer $peer dropped inv msg with mods: ${modifiers.map(Algos.encode).mkString(",")}") - peer.handlerRef ! InvNetworkMessage(mTid -> modifiers) - } - case None => logger.info(s"dataForInvMessage is empty for: $peer. Peer's status is: $status.") - } - - /** - * If node is not synced, `requestDownload` sends request for the one peer which will be find by 2 criteria: - * 1) HistoryComparisonResult != Younger. - * 2) Choose random peer with non bad priority. - * Otherwise this function sends requests for all known peers selected by 1-st criterion as above. - * - * If there are no any peers, request won't be sent. - * - * @param modifierTypeId - modifier type id - * @param modifierIds - modifier id - * @param history - current history state - * @param isBlockChainSynced - current block chain status - * @param isMining - current mining status - */ - def requestDownload(modifierTypeId: ModifierTypeId, - modifierIds: Seq[ModifierId], - history: History, - isBlockChainSynced: Boolean, - isMining: Boolean): Unit = - if (!isBlockChainSynced) { - logger.debug(s"requestDownload -> !isBlockChainSynced = true") - val (withBadNodesMap, withoutBadNodesMap) = peersCollection.filter(p => p._2._2 != Younger).partition { - case (_, (_, _, priority)) => priority == BadNode - } - logger.debug(s"withBadNodesMap -> ${withBadNodesMap.keys.mkString(",")}") - logger.debug(s"withoutBadNodesMap -> ${withoutBadNodesMap.keys.mkString(",")}") - val withBadNodes: IndexedSeq[(ConnectedPeer, HistoryComparisonResult)] = - withBadNodesMap.map(x => x._2._1 -> x._2._2).toIndexedSeq - val withoutBadNodes: IndexedSeq[(ConnectedPeer, HistoryComparisonResult)] = - withoutBadNodesMap.map(x => x._2._1 -> x._2._2).toIndexedSeq - val resultedPeerCollection = - if (withBadNodes.nonEmpty) withoutBadNodes :+ Random.shuffle(withBadNodes).head - else withoutBadNodes - logger.debug(s"resultedPeerCollection -> $resultedPeerCollection") - logger.debug(s"Block chain is not synced. acceptedPeers: $resultedPeerCollection") - if (resultedPeerCollection.nonEmpty) { - val shuffle: IndexedSeq[(ConnectedPeer, HistoryComparisonResult)] = Random.shuffle(resultedPeerCollection) - val cP = shuffle.last._1 - influxRef.foreach(_ ! SendDownloadRequest(modifierTypeId, modifierIds)) - if (modifierTypeId != Transaction.modifierTypeId) - logger.debug(s"requestModifies for peer ${cP.socketAddress} for mods: ${modifierIds.map(Algos.encode).mkString(",")}") - requestModifies(history, cP, modifierTypeId, modifierIds, isBlockChainSynced, isMining) - } else logger.info(s"BlockChain is not synced. There is no nodes, which we can connect with.") - } - else peersCollection.filter(p => p._2._2 != Younger) match { - case coll: Map[_, _] if coll.nonEmpty => - influxRef.foreach(_ ! SendDownloadRequest(modifierTypeId, modifierIds)) - coll.foreach { case (_, (cp, _, _)) => - if (modifierTypeId != Transaction.modifierTypeId) - logger.info(s"Sent download request to the ${cp.socketAddress} to modifiers of type: $modifierTypeId.") - requestModifies(history, cp, modifierTypeId, modifierIds, isBlockChainSynced, isMining) - } - case _ => logger.info(s"BlockChain is synced. There is no nodes, which we can connect with.") - } - - /** - * Handle received modifier. We will process received modifier only if we are expecting this on. - * - * @param mTid - modifier type id - * @param mId - modifier id - * @param peer - peer who sent modifier - * @param isBlockChainSynced - current chain status - */ - def receive(mTid: ModifierTypeId, - mId: ModifierId, - peer: ConnectedPeer, - isBlockChainSynced: Boolean): Unit = - if (isExpecting(mId, mTid, peer)) { - if (mTid != Transaction.modifierTypeId) { - logger.debug(s"Got new modifier with type $mTid from: ${peer.socketAddress}. with id ${Algos.encode(mId)}") - } - priorityCalculator = priorityCalculator.incrementReceive(peer.socketAddress) - val peerExpectedModifiers: Map[ModifierIdAsKey, (Cancellable, Int)] = expectedModifiers - .getOrElse(peer.socketAddress, Map.empty) - peerExpectedModifiers.get(toKey(mId)).foreach(_._1.cancel()) - if (mTid != Transaction.modifierTypeId) receivedModifiers += toKey(mId) - if (mTid != Transaction.modifierTypeId) expectedModifiers = clearExpectedModifiersCollection(peerExpectedModifiers, toKey(mId), peer.socketAddress) - else expectedTransactions = expectedTransactions - toKey(mId) - } else { - receivedSpamModifiers = receivedSpamModifiers - toKey(mId) + (toKey(mId) -> peer) - priorityCalculator = priorityCalculator.decrementRequest(peer.socketAddress) - } - - /** - * Transform modifier id to WrappedArray.ofBytes - * - * @param id - modifier id which will be transform to WrappedArray of bytes. - * @return transformed modifier id - */ - def toKey(id: ModifierId): ModifierIdAsKey = new mutable.WrappedArray.ofByte(id) - - /** - * This function gets collection of current expected modifiers from 'peer' and modifier, which - * will be removed from received collection as a parameters. - * If expected modifiers collection will contain other modifiers even after removing, - * this function will return collection of expectedModifiers with updated 'peer' expected collection - * otherwise it will return expectedModifiers collection without 'peer'. - * - * @param expectedModifiersFromPeer - collection of expected modifiers from 'peer' - * @param modifierId - modifier id, which will be removed from 'expectedModifiersFromPeer' - * @param peer - 'peer' from which expected modifiers collection we remove received modifier - * @return - expectedModifiers collection without 'peer' or expectedModifiers with updated 'peer' expected collection - */ - def clearExpectedModifiersCollection(expectedModifiersFromPeer: Map[ModifierIdAsKey, (Cancellable, Int)], - modifierId: ModifierIdAsKey, - peer: InetSocketAddress): Map[InetSocketAddress, Map[ModifierIdAsKey, (Cancellable, Int)]] = { - val collectionWithoutModId: Map[ModifierIdAsKey, (Cancellable, Int)] = expectedModifiersFromPeer - modifierId - collectionWithoutModId match { - case coll: Map[_, _] if coll.nonEmpty => expectedModifiers.updated(peer, coll) - case _ => expectedModifiers - peer - } - } -} - +// +//class DeliveryManager(influxRef: Option[ActorRef], +// nodeViewHolderRef: ActorRef, +// networkControllerRef: ActorRef, +// memoryPoolRef: ActorRef, +// nodeViewSync: ActorRef, +// downloadedModifiersValidator: ActorRef, +// settings: EncryAppSettings) extends Actor with StrictLogging { +// +// type ModifierIdAsKey = scala.collection.mutable.WrappedArray.ofByte +// +// implicit val exCon: ExecutionContextExecutor = context.dispatcher +// +// /** +// * Collection with spam modifiers. +// * Modifier considered spam if we receive it but it doesn't contain in expected modifiers collection. +// */ +// var receivedSpamModifiers: Map[ModifierIdAsKey, ConnectedPeer] = Map.empty +// /** +// * Collection of received modifiers ids. +// * Modifier considered received if we sent request for it and received it in special period. +// */ +// var receivedModifiers: HashSet[ModifierIdAsKey] = HashSet.empty[ModifierIdAsKey] +// /** +// * Collection of expected modifiers ids. +// * Modifier considered expected if we sent request for it. +// */ +// var expectedModifiers: Map[InetSocketAddress, Map[ModifierIdAsKey, (Cancellable, Int)]] = Map.empty +// +// var expectedTransactions: HashSet[ModifierIdAsKey] = HashSet.empty[ModifierIdAsKey] +// +// var peersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryComparisonResult, PeersPriorityStatus)] = Map.empty +// +// var priorityCalculator: PrioritiesCalculator = PrioritiesCalculator(settings.network) +// +// var canProcessTransactions: Boolean = true +// +// override def preStart(): Unit = { +// networkControllerRef ! RegisterMessagesHandler( +// Seq(ModifiersNetworkMessage.NetworkMessageTypeID -> "ModifiersNetworkMessage"), self) +// context.system.eventStream.subscribe(self, classOf[ModificationOutcome]) +// } +// +// override def receive: Receive = { +// case UpdatedHistory(historyReader) => +// logger.debug(s"Got message with history. Starting normal actor's work.") +// context.system.scheduler.schedule(0.second, priorityCalculator.updatingStatisticTime) { +// val (accumulatedStatistic: Map[InetSocketAddress, PeersPriorityStatus], newStat: PrioritiesCalculator) = +// priorityCalculator.accumulatePeersStatistic +// priorityCalculator = newStat +// context.parent ! AccumulatedPeersStatistic(accumulatedStatistic) +// } +// val checkModsSch = context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)( +// self ! CheckPayloadsToDownload +// ) +// nodeViewSync ! SendLocalSyncInfo +// context.become(basicMessageHandler(historyReader, isBlockChainSynced = false, isMining = settings.node.mining, checkModsSch)) +// case message => logger.debug(s"Got new message $message while awaiting history.") +// } +// +// def basicMessageHandler(history: History, +// isBlockChainSynced: Boolean, +// isMining: Boolean, +// checkModScheduler: Cancellable): Receive = { +// case InvalidModifier(id) => receivedModifiers -= toKey(id) +// +// case CheckDelivery(peer: ConnectedPeer, modifierTypeId: ModifierTypeId, modifierId: ModifierId) => +// checkDelivery(peer, modifierTypeId, modifierId) +// +// case UpdatedPeersCollection(newPeers) => +// logger.info(s"Delivery manager got updated peers collection.") +// peersCollection = newPeers +// +// case ConnectionStopped(peer) => +// peersCollection -= peer +// logger.info(s"Removed peer: $peer from peers collection on Delivery Manager." + +// s" Current peers are: ${peersCollection.mkString(",")}") +// +// case OtherNodeSyncingStatus(remote, status, extOpt) => +// status match { +// case Unknown => logger.info("Peer status is still unknown.") +// case Younger | Fork if isBlockChainSynced => sendInvData(remote, status, extOpt) +// case _ => +// } +// +// case CheckPayloadsToDownload => +// val currentQueue: HashSet[ModifierIdAsKey] = +// expectedModifiers.flatMap { case (_, modIds) => modIds.keys }.to[HashSet] +// logger.debug(s"Current queue: ${currentQueue.map(elem => Algos.encode(elem.toArray)).mkString(",")}") +// logger.debug(s"receivedModifiers: ${receivedModifiers.map(id => Algos.encode(id.toArray)).mkString(",")}") +// logger.debug(s"Qty to req: ${settings.network.networkChunkSize - currentQueue.size - receivedModifiers.size}") +// logger.debug(s"currentQueue.size: ${currentQueue.size}") +// logger.debug(s"receivedModifiers.size: ${receivedModifiers.size}") +// val newIds: Seq[ModifierId] = +// history.payloadsIdsToDownload( +// settings.network.networkChunkSize - currentQueue.size - receivedModifiers.size, +// currentQueue.map(elem => ModifierId @@ elem.toArray) +// ).filterNot(modId => currentQueue.contains(toKey(modId)) || receivedModifiers.contains(toKey(modId))) +// logger.debug(s"newIds: ${newIds.map(elem => Algos.encode(elem)).mkString(",")}") +// if (newIds.nonEmpty) requestDownload(Payload.modifierTypeId, newIds, history, isBlockChainSynced, isMining) +// val nextCheckModsScheduler = +// context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)(self ! CheckPayloadsToDownload) +// context.become(basicMessageHandler(history, isBlockChainSynced, settings.node.mining, nextCheckModsScheduler)) +// +// case SemanticallySuccessfulModifier(mod) => +// logger.info(s"Got SemanticallySuccessfulModifier with id: ${Algos.encode(mod.id)} of type ${mod.modifierTypeId} on dm") +// mod match { +// case block: Block => receivedModifiers -= toKey(block.payload.id) +// case _ => receivedModifiers -= toKey(mod.id) +// } +// if (!isBlockChainSynced && expectedModifiers.isEmpty && receivedModifiers.isEmpty) { +// checkModScheduler.cancel() +// logger.debug(s"SemanticallySuccessfulModifier case, if condition true. Resend CheckPayloadsToDownload to DM") +// self ! CheckPayloadsToDownload +// } +// case SemanticallyFailedModification(mod, _) => receivedModifiers -= toKey(mod.id) +// +// case SyntacticallyFailedModification(mod, _) => receivedModifiers -= toKey(mod.id) +// +// case SuccessfulTransaction(_) => //do nothing +// +// case RequestFromLocal(peer, modifierTypeId, modifierIds) => +// if (modifierTypeId != Transaction.modifierTypeId) logger.debug(s"Got RequestFromLocal on NVSH from $sender with " + +// s"ids of type: $modifierTypeId. Number of ids is: ${modifierIds.size}. Ids: ${modifierIds.map(Algos.encode).mkString(",")}. Sending request from local to DeliveryManager.") +// if (modifierIds.nonEmpty) requestModifies(history, peer, modifierTypeId, modifierIds, isBlockChainSynced, isMining) +// +// case DataFromPeer(message, remote) => message match { +// case ModifiersNetworkMessage((typeId, modifiers)) => +// logger.debug(s"Received modifiers are: ${modifiers.map(x => Algos.encode(x._1)).mkString(",")}") +// influxRef.foreach(_ ! GetModifiers(typeId, modifiers.keys.toSeq)) +// for ((id, _) <- modifiers) receive(typeId, id, remote, isBlockChainSynced) +// val (spam: Map[ModifierId, Array[Byte]], fm: Map[ModifierId, Array[Byte]]) = modifiers.partition(p => isSpam(p._1)) +// if (spam.nonEmpty) { +// if (typeId != Transaction.modifierTypeId) +// logger.info(s"Spam attempt: peer $remote has sent a non-requested modifiers of type $typeId with ids" + +// s": ${spam.keys.map(Algos.encode)}.") +// receivedSpamModifiers = Map.empty +// } +// val filteredModifiers: Map[ModifierId, Array[Byte]] = fm.filterKeys(k => !history.isModifierDefined(k)) +// if (typeId != Transaction.modifierTypeId) influxRef +// .foreach(ref => (0 to filteredModifiers.size).foreach(_ => ref ! SerializedModifierFromNetwork(typeId))) +// //todo check this logic +// logger.debug(s"Type of mod: ${typeId}. canProcessTransactions: ${canProcessTransactions}") +// if ((typeId == Transaction.modifierTypeId && canProcessTransactions) || (typeId != Transaction.modifierTypeId)) +// downloadedModifiersValidator ! ModifiersForValidating(remote, typeId, filteredModifiers) +// +// case _ => logger.debug(s"DeliveryManager got invalid type of DataFromPeer message!") +// } +// +// case DownloadRequest(modifierTypeId, modifiersId, previousModifier) => //todo check this condition +// if (modifierTypeId != Transaction.modifierTypeId) +// logger.info(s"DownloadRequest for mod ${Algos.encode(modifiersId)} of type: $modifierTypeId prev mod: " + +// s"${previousModifier.map(Algos.encode)}") +// requestDownload(modifierTypeId, Seq(modifiersId), history, isBlockChainSynced, isMining) +// +// case PeersForSyncInfo(peers) => sendSync(history.syncInfo, peers) +// +// case FullBlockChainIsSynced => context.become(basicMessageHandler(history, isBlockChainSynced = true, isMining, checkModScheduler)) +// +// case StartMining => context.become(basicMessageHandler(history, isBlockChainSynced, isMining = true, checkModScheduler)) +// +// case DisableMining => context.become(basicMessageHandler(history, isBlockChainSynced, isMining = false, checkModScheduler)) +// +// case UpdatedHistory(historyReader) => context.become(basicMessageHandler(historyReader, isBlockChainSynced, isMining, checkModScheduler)) +// +// case StopTransactionsValidation => canProcessTransactions = false +// +// case StartTransactionsValidation => canProcessTransactions = true +// +// case message => logger.debug(s"Got strange message $message(${message.getClass}) on DeliveryManager from $sender") +// } +// +// /** +// * This function check if modifier has received or not. +// * If modifier has transaction type id, it won't be re-asked. +// * If we still have no this modifier and number of attempts have no expired, we will re-ask it. +// * If we still have no this modifier and number of attempts have expired, we will remove it from expected modifiers collection. +// * Otherwise - do nothing. +// * +// * @param peer - peer, from whom we are expecting modifier +// * @param modifierTypeId - type of expected modifier +// * @param modifierId - expected modifier id +// */ +// def checkDelivery(peer: ConnectedPeer, modifierTypeId: ModifierTypeId, modifierId: ModifierId): Unit = { +// val expectedModifiersByPeer: Map[ModifierIdAsKey, (Cancellable, Int)] = +// expectedModifiers.getOrElse(peer.socketAddress, Map.empty) +// if (modifierTypeId == Transaction.modifierTypeId) +// expectedModifiers = clearExpectedModifiersCollection(expectedModifiersByPeer, toKey(modifierId), peer.socketAddress) +// else expectedModifiersByPeer.find { case (id, (_, _)) => id == toKey(modifierId) } match { +// case Some((_, (_, attempts))) if attempts <= settings.network.maxDeliveryChecks => +// logger.debug(s"Modifier ${Algos.encode(modifierId)} needed to be requested from $peer!") +// reRequestModifier(peer, modifierTypeId, modifierId, expectedModifiersByPeer) +// case Some((modId, (_, _))) => +// logger.debug(s"Maximum number of attempts has expired. Remove modifier ${Algos.encode(modifierId)} from $peer.") +// expectedModifiers = clearExpectedModifiersCollection(expectedModifiersByPeer, modId, peer.socketAddress) +// case _ => +// logger.debug(s"This modifiers ${Algos.encode(modifierId)} is not contained in expectedModifiers collection from $peer.") +// } +// } +// +// /** +// * If node is not synced, send sync info to random peer, otherwise to all known peers. +// * +// * @param syncInfo - sync info +// */ +// def sendSync(syncInfo: SyncInfo, peers: Seq[ConnectedPeer]): Unit = peers.foreach { peer => +// logger.info(s"Sending to $peer sync info message.") +// peer.handlerRef ! SyncInfoNetworkMessage(syncInfo) +// } +// +// /** +// * Send request to 'peer' with modifiers ids of type 'modifierTypeId'. +// * We can do this activity only if 'peer' status != Younger. +// * If current chain isn't synced and mining is off, we can't request transactions, otherwise can. +// * +// * We should filter our requesting modifiers to avoid request repeated modifiers. +// * +// * @param history - current history reader +// * @param peer - peer, whom message will be send +// * @param mTypeId - modifier type id +// * @param modifierIds - modifiers ids +// * @param isBlockChainSynced - current block chain status +// * @param isMining - current mining status +// */ +// +// def requestModifies(history: History, +// peer: ConnectedPeer, +// mTypeId: ModifierTypeId, +// modifierIds: Seq[ModifierId], +// isBlockChainSynced: Boolean, +// isMining: Boolean): Unit = { +// val firstCondition: Boolean = mTypeId == Transaction.modifierTypeId && isBlockChainSynced && isMining +// val secondCondition: Boolean = mTypeId != Transaction.modifierTypeId +// val thirdCondition: Boolean = +// if (!isBlockChainSynced) peersCollection.get(peer.socketAddress).exists(p => p._2 != Younger) +// else peersCollection.contains(peer.socketAddress) +// if (mTypeId != Transaction.modifierTypeId) +// logger.debug(s"Got requestModifier for modifiers of type: $mTypeId to $peer with modifiers ${modifierIds.size}." + +// s" Try to check conditions: $firstCondition -> $secondCondition -> $thirdCondition.") +// if ((firstCondition || secondCondition) && thirdCondition) { +// val requestedModifiersFromPeer: Map[ModifierIdAsKey, (Cancellable, Int)] = expectedModifiers +// .getOrElse(peer.socketAddress, Map.empty) +// +// val notYetRequested: Seq[ModifierId] = modifierIds +// .filter(id => +// !history.isModifierDefined(id) && +// !requestedModifiersFromPeer.contains(toKey(id)) && +// !receivedModifiers.contains(toKey(id)) +// ) +// +// if (notYetRequested.nonEmpty) { +// if (mTypeId != Transaction.modifierTypeId) +// logger.debug(s"Send request to ${peer.socketAddress} for ${notYetRequested.size} modifiers of type $mTypeId ") +// peer.handlerRef ! RequestModifiersNetworkMessage(mTypeId -> notYetRequested) +// priorityCalculator = priorityCalculator.incrementRequestForNModifiers(peer.socketAddress, notYetRequested.size) +// if (mTypeId != Transaction.modifierTypeId) { +// val requestedModIds: Map[ModifierIdAsKey, (Cancellable, Int)] = +// notYetRequested.foldLeft(requestedModifiersFromPeer) { case (rYet, id) => +// rYet.updated(toKey(id), +// context.system +// .scheduler.scheduleOnce(settings.network.deliveryTimeout)(self ! CheckDelivery(peer, mTypeId, id)) -> 1) +// } +// expectedModifiers = expectedModifiers.updated(peer.socketAddress, requestedModIds) +// } else expectedTransactions = expectedTransactions ++ modifierIds.map(toKey) +// } +// } +// } +// +// /** +// * Re-ask 'modifierId' from 'peer' if needed. We will do this only if we are expecting these modifier from 'peer' +// * and if number of attempts doesn't expired yet. +// * This activity will update timer on re-asked modifier. +// * +// * @param peer - peer, whom message will be send +// * @param mTypeId - modifier type id +// * @param modId - re-asked modifier id +// */ +// def reRequestModifier(peer: ConnectedPeer, +// mTypeId: ModifierTypeId, +// modId: ModifierId, +// peerRequests: Map[ModifierIdAsKey, (Cancellable, Int)]): Unit = +// peerRequests.get(toKey(modId)) match { +// case Some((_, attempts)) => peersCollection.find { case (innerAddr, (_, cResult, _)) => +// innerAddr == peer.socketAddress && cResult != Younger +// } match { +// case Some((_, (cP, _, _))) => +// cP.handlerRef ! RequestModifiersNetworkMessage(mTypeId -> Seq(modId)) +// logger.debug(s"Re-asked ${peer.socketAddress} and handler: ${peer.handlerRef} for modifier of type: " + +// s"$mTypeId with id: ${Algos.encode(modId)}. Attempts: $attempts") +// priorityCalculator = priorityCalculator.incrementRequest(peer.socketAddress) +// expectedModifiers = expectedModifiers.updated(peer.socketAddress, peerRequests.updated( +// toKey(modId), +// context.system.scheduler +// .scheduleOnce(settings.network.deliveryTimeout)(self ! CheckDelivery(peer, mTypeId, modId)) -> (attempts + 1) +// )) +// case None => +// expectedModifiers = clearExpectedModifiersCollection(peerRequests, toKey(modId), peer.socketAddress) +// logger.debug(s"Tried to re-ask modifier ${Algos.encode(modId)}, but this id not needed from this peer") +// } +// case _ => logger.debug(s"There is no such modifier ${Algos.encode(modId)} in expected collection.") +// } +// +// /** +// * Check 'expectedModifiers' for awaiting modifier with id 'mId' from 'peer' +// * +// * @param mId - id of checkable modifier +// * @param peer - peer from which we possibly expecting modifier +// * @return 'true' if we are expecting this modifier from this peer otherwise 'false' +// */ +// def isExpecting(mId: ModifierId, modifierTypeId: ModifierTypeId, peer: ConnectedPeer): Boolean = { +// if (modifierTypeId != Transaction.modifierTypeId) { +// val result: Boolean = expectedModifiers.getOrElse(peer.socketAddress, Map.empty).contains(toKey(mId)) +// logger.debug(s"isExpecting -->> modId ${Algos.encode(mId)} --> $result") +// result +// } else expectedTransactions.contains(toKey(mId)) +// } +// +// /** +// * Clear the 'receivedSpamModifiers' collection +// * +// * @param mIds - sequence of modifiers ids which will be deleted from spam collection +// */ +// def deleteSpam(mIds: Seq[ModifierId]): Unit = for (id <- mIds) receivedSpamModifiers -= toKey(id) +// +// /** +// * Check receivedSpamModifiers for contains received modifier +// * +// * @param mId - checkable modifier +// * @return 'true' if received modifier is in spam collection otherwise 'false' +// */ +// def isSpam(mId: ModifierId): Boolean = receivedSpamModifiers.contains(toKey(mId)) +// +// /** +// * Send inv data to the 'peer'. +// * +// * @param peer - peer whom will send a message +// * @param status - current peer's status +// * @param dataForInvMessage - data for inv message +// */ +// def sendInvData(peer: ConnectedPeer, +// status: HistoryComparisonResult, +// dataForInvMessage: Option[Seq[(ModifierTypeId, ModifierId)]]): Unit = dataForInvMessage match { +// case Some(data) => +// data.groupBy(_._1).mapValues(_.map(_._2)).foreach { +// case (mTid, mods) if mods.size <= settings.network.maxInvObjects => +// logger.debug(s"Send to peer $peer inv msg with mods: ${mods.map(Algos.encode).mkString(",")}") +// peer.handlerRef ! InvNetworkMessage(mTid -> mods) +// case (mTid, mods) => +// val modifiers: Seq[ModifierId] = mods.take(settings.network.maxInvObjects) +// logger.debug(s"Send to peer $peer dropped inv msg with mods: ${modifiers.map(Algos.encode).mkString(",")}") +// peer.handlerRef ! InvNetworkMessage(mTid -> modifiers) +// } +// case None => logger.info(s"dataForInvMessage is empty for: $peer. Peer's status is: $status.") +// } +// +// /** +// * If node is not synced, `requestDownload` sends request for the one peer which will be find by 2 criteria: +// * 1) HistoryComparisonResult != Younger. +// * 2) Choose random peer with non bad priority. +// * Otherwise this function sends requests for all known peers selected by 1-st criterion as above. +// * +// * If there are no any peers, request won't be sent. +// * +// * @param modifierTypeId - modifier type id +// * @param modifierIds - modifier id +// * @param history - current history state +// * @param isBlockChainSynced - current block chain status +// * @param isMining - current mining status +// */ +// def requestDownload(modifierTypeId: ModifierTypeId, +// modifierIds: Seq[ModifierId], +// history: History, +// isBlockChainSynced: Boolean, +// isMining: Boolean): Unit = +// if (!isBlockChainSynced) { +// logger.debug(s"requestDownload -> !isBlockChainSynced = true") +// val (withBadNodesMap, withoutBadNodesMap) = peersCollection.filter(p => p._2._2 != Younger).partition { +// case (_, (_, _, priority)) => priority == BadNode +// } +// logger.debug(s"withBadNodesMap -> ${withBadNodesMap.keys.mkString(",")}") +// logger.debug(s"withoutBadNodesMap -> ${withoutBadNodesMap.keys.mkString(",")}") +// val withBadNodes: IndexedSeq[(ConnectedPeer, HistoryComparisonResult)] = +// withBadNodesMap.map(x => x._2._1 -> x._2._2).toIndexedSeq +// val withoutBadNodes: IndexedSeq[(ConnectedPeer, HistoryComparisonResult)] = +// withoutBadNodesMap.map(x => x._2._1 -> x._2._2).toIndexedSeq +// val resultedPeerCollection = +// if (withBadNodes.nonEmpty) withoutBadNodes :+ Random.shuffle(withBadNodes).head +// else withoutBadNodes +// logger.debug(s"resultedPeerCollection -> $resultedPeerCollection") +// logger.debug(s"Block chain is not synced. acceptedPeers: $resultedPeerCollection") +// if (resultedPeerCollection.nonEmpty) { +// val shuffle: IndexedSeq[(ConnectedPeer, HistoryComparisonResult)] = Random.shuffle(resultedPeerCollection) +// val cP = shuffle.last._1 +// influxRef.foreach(_ ! SendDownloadRequest(modifierTypeId, modifierIds)) +// if (modifierTypeId != Transaction.modifierTypeId) +// logger.debug(s"requestModifies for peer ${cP.socketAddress} for mods: ${modifierIds.map(Algos.encode).mkString(",")}") +// requestModifies(history, cP, modifierTypeId, modifierIds, isBlockChainSynced, isMining) +// } else logger.info(s"BlockChain is not synced. There is no nodes, which we can connect with.") +// } +// else peersCollection.filter(p => p._2._2 != Younger) match { +// case coll: Map[_, _] if coll.nonEmpty => +// influxRef.foreach(_ ! SendDownloadRequest(modifierTypeId, modifierIds)) +// coll.foreach { case (_, (cp, _, _)) => +// if (modifierTypeId != Transaction.modifierTypeId) +// logger.info(s"Sent download request to the ${cp.socketAddress} to modifiers of type: $modifierTypeId.") +// requestModifies(history, cp, modifierTypeId, modifierIds, isBlockChainSynced, isMining) +// } +// case _ => logger.info(s"BlockChain is synced. There is no nodes, which we can connect with.") +// } +// +// /** +// * Handle received modifier. We will process received modifier only if we are expecting this on. +// * +// * @param mTid - modifier type id +// * @param mId - modifier id +// * @param peer - peer who sent modifier +// * @param isBlockChainSynced - current chain status +// */ +// def receive(mTid: ModifierTypeId, +// mId: ModifierId, +// peer: ConnectedPeer, +// isBlockChainSynced: Boolean): Unit = +// if (isExpecting(mId, mTid, peer)) { +// if (mTid != Transaction.modifierTypeId) { +// logger.debug(s"Got new modifier with type $mTid from: ${peer.socketAddress}. with id ${Algos.encode(mId)}") +// } +// priorityCalculator = priorityCalculator.incrementReceive(peer.socketAddress) +// val peerExpectedModifiers: Map[ModifierIdAsKey, (Cancellable, Int)] = expectedModifiers +// .getOrElse(peer.socketAddress, Map.empty) +// peerExpectedModifiers.get(toKey(mId)).foreach(_._1.cancel()) +// if (mTid != Transaction.modifierTypeId) receivedModifiers += toKey(mId) +// if (mTid != Transaction.modifierTypeId) expectedModifiers = clearExpectedModifiersCollection(peerExpectedModifiers, toKey(mId), peer.socketAddress) +// else expectedTransactions = expectedTransactions - toKey(mId) +// } else { +// receivedSpamModifiers = receivedSpamModifiers - toKey(mId) + (toKey(mId) -> peer) +// priorityCalculator = priorityCalculator.decrementRequest(peer.socketAddress) +// } +// +// /** +// * Transform modifier id to WrappedArray.ofBytes +// * +// * @param id - modifier id which will be transform to WrappedArray of bytes. +// * @return transformed modifier id +// */ +// def toKey(id: ModifierId): ModifierIdAsKey = new mutable.WrappedArray.ofByte(id) +// +// /** +// * This function gets collection of current expected modifiers from 'peer' and modifier, which +// * will be removed from received collection as a parameters. +// * If expected modifiers collection will contain other modifiers even after removing, +// * this function will return collection of expectedModifiers with updated 'peer' expected collection +// * otherwise it will return expectedModifiers collection without 'peer'. +// * +// * @param expectedModifiersFromPeer - collection of expected modifiers from 'peer' +// * @param modifierId - modifier id, which will be removed from 'expectedModifiersFromPeer' +// * @param peer - 'peer' from which expected modifiers collection we remove received modifier +// * @return - expectedModifiers collection without 'peer' or expectedModifiers with updated 'peer' expected collection +// */ +// def clearExpectedModifiersCollection(expectedModifiersFromPeer: Map[ModifierIdAsKey, (Cancellable, Int)], +// modifierId: ModifierIdAsKey, +// peer: InetSocketAddress): Map[InetSocketAddress, Map[ModifierIdAsKey, (Cancellable, Int)]] = { +// val collectionWithoutModId: Map[ModifierIdAsKey, (Cancellable, Int)] = expectedModifiersFromPeer - modifierId +// collectionWithoutModId match { +// case coll: Map[_, _] if coll.nonEmpty => expectedModifiers.updated(peer, coll) +// case _ => expectedModifiers - peer +// } +// } +//} +// object DeliveryManager { final case class CheckDelivery(peer: ConnectedPeer, modifierTypeId: ModifierTypeId, modifierId: ModifierId) @@ -515,15 +516,15 @@ object DeliveryManager { final case class CheckModifiersWithQueueSize(size: Int) extends AnyVal - def props(influxRef: Option[ActorRef], - nodeViewHolderRef: ActorRef, - networkControllerRef: ActorRef, - memoryPoolRef: ActorRef, - nodeViewSync: ActorRef, - downloadedModifiersValidator: ActorRef, - settings: EncryAppSettings): Props = - Props(new DeliveryManager(influxRef, nodeViewHolderRef, networkControllerRef, memoryPoolRef, nodeViewSync, - downloadedModifiersValidator, settings)) +// def props(influxRef: Option[ActorRef], +// nodeViewHolderRef: ActorRef, +// networkControllerRef: ActorRef, +// memoryPoolRef: ActorRef, +// nodeViewSync: ActorRef, +// downloadedModifiersValidator: ActorRef, +// settings: EncryAppSettings): Props = +// Props(new DeliveryManager(influxRef, nodeViewHolderRef, networkControllerRef, memoryPoolRef, nodeViewSync, +// downloadedModifiersValidator, settings)) class DeliveryManagerPriorityQueue(settings: ActorSystem.Settings, config: Config) extends UnboundedStablePriorityMailbox( @@ -542,11 +543,11 @@ object DeliveryManager { case InvalidModifier(_) => 2 - case DataFromPeer(msg: ModifiersNetworkMessage, _) => - msg match { - case ModifiersNetworkMessage((typeId, _)) if typeId != Transaction.modifierTypeId => 1 - case _ => 3 - } +// case DataFromPeer(msg: ModifiersNetworkMessage, _) => +// msg match { +// case ModifiersNetworkMessage((typeId, _)) if typeId != Transaction.modifierTypeId => 1 +// case _ => 3 +// } case PoisonPill => 4 diff --git a/src/main/scala/encry/network/DownloadedModifiersValidator.scala b/src/main/scala/encry/network/DownloadedModifiersValidator.scala index 70999f12b6..3c4651e727 100644 --- a/src/main/scala/encry/network/DownloadedModifiersValidator.scala +++ b/src/main/scala/encry/network/DownloadedModifiersValidator.scala @@ -20,45 +20,45 @@ import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import scala.util.{Failure, Success, Try} -class DownloadedModifiersValidator(modifierIdSize: Int, - nodeViewHolder: ActorRef, - peersKeeper: ActorRef, - nodeViewSync: ActorRef, - memoryPoolRef: ActorRef, - influxRef: Option[ActorRef], - settings: EncryAppSettings) - extends Actor - with StrictLogging { - - override def receive: Receive = { - case UpdatedHistory(historyReader) => context.become(workingCycle(historyReader)) - case msg => logger.info(s"Got $msg on DownloadedModifiersValidator") - } - - def workingCycle(history: History): Receive = { - case ModifiersForValidating(remote, typeId, filteredModifiers) => - typeId match { - case Transaction.modifierTypeId => - filteredModifiers.foreach { - case (id, bytes) => - Try(TransactionProtoSerializer.fromProto(TransactionProtoMessage.parseFrom(bytes))).flatten match { - case Success(tx) if tx.semanticValidity.isSuccess => memoryPoolRef ! NewTransaction(tx) - case Success(tx) => - logger.info(s"Transaction with id: ${tx.encodedId} invalid cause of: ${tx.semanticValidity}.") - context.parent ! BanPeer(remote, SyntacticallyInvalidTransaction) - nodeViewSync ! InvalidModifier(id) - case Failure(ex) => - context.parent ! BanPeer(remote, CorruptedSerializedBytes) - nodeViewSync ! InvalidModifier(id) - logger.info(s"Received modifier from $remote can't be parsed cause of: ${ex.getMessage}.") - } - } - } - case UpdatedHistory(historyReader) => context.become(workingCycle(historyReader)) - case msg => logger.info(s"Got $msg on DownloadedModifiersValidator") - } - -} +//class DownloadedModifiersValidator(modifierIdSize: Int, +// nodeViewHolder: ActorRef, +// peersKeeper: ActorRef, +// nodeViewSync: ActorRef, +// memoryPoolRef: ActorRef, +// influxRef: Option[ActorRef], +// settings: EncryAppSettings) +// extends Actor +// with StrictLogging { +// +// override def receive: Receive = { +// case UpdatedHistory(historyReader) => context.become(workingCycle(historyReader)) +// case msg => logger.info(s"Got $msg on DownloadedModifiersValidator") +// } +// +// def workingCycle(history: History): Receive = { +// case ModifiersForValidating(remote, typeId, filteredModifiers) => +// typeId match { +// case Transaction.modifierTypeId => +// filteredModifiers.foreach { +// case (id, bytes) => +// Try(TransactionProtoSerializer.fromProto(TransactionProtoMessage.parseFrom(bytes))).flatten match { +// case Success(tx) if tx.semanticValidity.isSuccess => memoryPoolRef ! NewTransaction(tx) +// case Success(tx) => +// logger.info(s"Transaction with id: ${tx.encodedId} invalid cause of: ${tx.semanticValidity}.") +// context.parent ! BanPeer(remote, SyntacticallyInvalidTransaction) +// nodeViewSync ! InvalidModifier(id) +// case Failure(ex) => +// context.parent ! BanPeer(remote, CorruptedSerializedBytes) +// nodeViewSync ! InvalidModifier(id) +// logger.info(s"Received modifier from $remote can't be parsed cause of: ${ex.getMessage}.") +// } +// } +// } +// case UpdatedHistory(historyReader) => context.become(workingCycle(historyReader)) +// case msg => logger.info(s"Got $msg on DownloadedModifiersValidator") +// } +// +//} object DownloadedModifiersValidator { @@ -68,22 +68,22 @@ object DownloadedModifiersValidator { final case class InvalidModifier(ids: ModifierId) extends AnyVal - def props(modifierIdSize: Int, - nodeViewHolder: ActorRef, - peersKeeper: ActorRef, - nodeViewSync: ActorRef, - memoryPoolRef: ActorRef, - influxRef: Option[ActorRef], - settings: EncryAppSettings): Props = - Props( - new DownloadedModifiersValidator(modifierIdSize, - nodeViewHolder, - peersKeeper, - nodeViewSync, - memoryPoolRef, - influxRef, - settings) - ) +// def props(modifierIdSize: Int, +// nodeViewHolder: ActorRef, +// peersKeeper: ActorRef, +// nodeViewSync: ActorRef, +// memoryPoolRef: ActorRef, +// influxRef: Option[ActorRef], +// settings: EncryAppSettings): Props = +// Props( +// new DownloadedModifiersValidator(modifierIdSize, +// nodeViewHolder, +// peersKeeper, +// nodeViewSync, +// memoryPoolRef, +// influxRef, +// settings) +// ) class DownloadedModifiersValidatorPriorityQueue(settings: ActorSystem.Settings, config: Config) extends UnboundedStablePriorityMailbox(PriorityGenerator { diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 4df0ef55a9..952e95fcb9 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -5,16 +5,21 @@ import java.net.InetSocketAddress import akka.actor.{Actor, ActorRef, Props} import akka.pattern._ import akka.util.Timeout +import com.typesafe.scalalogging.StrictLogging +import encry.network.DM.{IsRequested, RequestSent} import encry.network.MessageBuilder.{GetPeerInfo, GetPeers} import encry.network.Messages.MessageToNetwork import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} import encry.network.PeerConnectionHandler.ConnectedPeer import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.utils.Algos import scala.concurrent.duration._ import scala.util.Try -case class MessageBuilder(msg: MessageToNetwork, peersKeeper: ActorRef) extends Actor { +case class MessageBuilder(msg: MessageToNetwork, + peersKeeper: ActorRef, + deliveryManager: ActorRef) extends Actor with StrictLogging { import context.dispatcher @@ -23,8 +28,15 @@ case class MessageBuilder(msg: MessageToNetwork, peersKeeper: ActorRef) extends override def receive: Receive = { case RequestFromLocal(peer, modTypeId, modsIds) => Try { - (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].map { peer => - peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) + (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].foreach { peer => + modsIds.foreach { modId => + for { + isRequested <- (deliveryManager ? IsRequested(modId)).mapTo[Boolean] + } yield if (!isRequested) { + peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) + deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId) + } else logger.debug(s"Duplicate request for modifier of type ${modTypeId} and id: ${Algos.encode(modId)}") + } } } case SendSyncInfo(syncInfo) => @@ -49,5 +61,7 @@ object MessageBuilder { case object GetPeers case class GetPeerInfo(peerIp: InetSocketAddress) - def props(msg: MessageToNetwork, peersKeeper: ActorRef): Props = Props(new MessageBuilder(msg, peersKeeper)) + def props(msg: MessageToNetwork, + peersKeeper: ActorRef, + deliveryManager: ActorRef): Props = Props(new MessageBuilder(msg, peersKeeper, deliveryManager)) } \ No newline at end of file diff --git a/src/main/scala/encry/network/NetworkController.scala b/src/main/scala/encry/network/NetworkController.scala index 0213394830..36d63e9798 100755 --- a/src/main/scala/encry/network/NetworkController.scala +++ b/src/main/scala/encry/network/NetworkController.scala @@ -20,121 +20,121 @@ import scala.concurrent.duration._ import scala.language.{existentials, postfixOps} import scala.util.Try -class NetworkController(networkSettings: NetworkSettings, - peersKeeper: ActorRef, - nodeViewSync: ActorRef) extends Actor with StrictLogging { - - import context.dispatcher - import context.system - - override def preStart(): Unit = logger.info(s"Network controller started") - - var messagesHandlers: Map[Seq[Byte], ActorRef] = Map.empty - val externalSocketAddress: Option[InetSocketAddress] = networkSettings.declaredAddress - logger.info(s"Declared address is: $externalSocketAddress.") - - if (!networkSettings.localOnly.getOrElse(false)) networkSettings.declaredAddress.foreach(myAddress => - Try(NetworkInterface.getNetworkInterfaces.asScala.exists(interface => - interface.getInterfaceAddresses.asScala.exists(interfaceAddress => - InetAddress.getAllByName(new URI("http://" + myAddress).getHost).contains(interfaceAddress.getAddress) - ))).recover { case t: Throwable => logger.error(s"Declared address validation failed: $t") } - ) - - IO(Tcp) ! Bind(self, networkSettings.bindAddress, options = KeepAlive(true) :: Nil, pullMode = false) - - override def supervisorStrategy: SupervisorStrategy = OneForOneStrategy( - maxNrOfRetries = 5, - withinTimeRange = 60 seconds) { - case _ => Restart - } - - override def receive: Receive = bindingLogic - .orElse(businessLogic) - .orElse(peersLogic) - .orElse { - case RegisterMessagesHandler(types, handler) => - logger.info(s"Registering handlers for ${types.mkString(",")}.") - val ids = types.map(_._1) - messagesHandlers += (ids -> handler) - case CommandFailed(cmd: Tcp.Command) => logger.info(s"Failed to execute: $cmd.") - case msg => logger.warn(s"NetworkController: got something strange $msg.") - } - - def bindingLogic: Receive = { - case Bound(address) => - logger.info(s"Successfully bound to the port ${address.getPort}.") - context.system.scheduler.schedule(2.seconds, 5.second)(peersKeeper ! RequestPeerForConnection) - case CommandFailed(add: Bind) => - logger.info(s"Node can't be bind to the address: ${add.localAddress}.") - context.stop(self) - } - - def businessLogic: Receive = { - case MessageFromNetwork(message, Some(remote)) if message.isValid(networkSettings.syncPacketLength) => - logger.debug(s"Got ${message.messageName} on the NetworkController.") - findHandler(message, message.NetworkMessageTypeID, remote, messagesHandlers) - case MessageFromNetwork(message, Some(remote)) => - peersKeeper ! BanPeer(remote, InvalidNetworkMessage(message.messageName)) - logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") - } - - def peersLogic: Receive = { - case PeerForConnection(peer) => - logger.info(s"Network controller got new peer for connection: $peer. Trying to set connection with remote...") - IO(Tcp) ! Connect( - peer, - None, - KeepAlive(true) :: Nil, - Some(networkSettings.connectionTimeout), - pullMode = true - ) - - case Connected(remote, localAddress) => - logger.info(s"Network controller got 'Connected' message from: $remote. " + - s"Trying to set stable connection with remote... " + - s"Local TCP endpoint is: $localAddress.") - peersKeeper ! NewConnection(remote, sender()) - - case ConnectionVerified(remote, remoteConnection, connectionType) => - logger.info(s"Network controller got approvement for stable connection with: $remote. Starting interaction process...") - val peerConnectionHandler: ActorRef = context.actorOf( - PeerConnectionHandler.props(remoteConnection, connectionType, externalSocketAddress, remote, networkSettings) - .withDispatcher("network-dispatcher") - ) - peerConnectionHandler ! StartInteraction - - case HandshakedDone(remote) => - logger.info(s"Network controller got approvement from peer handler about successful handshake. " + - s"Sending to peerKeeper connected peer.") - peersKeeper ! HandshakedDone(remote) - - case ConnectionStopped(peer) => - logger.info(s"Network controller got signal about breaking connection with: $peer. " + - s"Sending to peerKeeper actual information.") - peersKeeper ! ConnectionStopped(peer) - nodeViewSync ! ConnectionStopped(peer) - - case CommandFailed(connect: Connect) => - logger.info(s"Failed to connect to: ${connect.remoteAddress}.") - peersKeeper ! OutgoingConnectionFailed(connect.remoteAddress) - } - - private def findHandler(message: NetworkMessage, - messageId: Byte, - remote: ConnectedPeer, - mH: Map[Seq[Byte], ActorRef]): Unit = - mH.find(_._1.contains(messageId)).map(_._2) match { - case Some(handler) => - handler ! DataFromPeer(message, remote) - logger.debug(s"Send message DataFromPeer with ${message.messageName} to $handler.") - case None => logger.info("No handlers found for message: " + message.messageName) - } -} +//class NetworkController(networkSettings: NetworkSettings, +// peersKeeper: ActorRef, +// nodeViewSync: ActorRef) extends Actor with StrictLogging { +// +// import context.dispatcher +// import context.system +// +// override def preStart(): Unit = logger.info(s"Network controller started") +// +// var messagesHandlers: Map[Seq[Byte], ActorRef] = Map.empty +// val externalSocketAddress: Option[InetSocketAddress] = networkSettings.declaredAddress +// logger.info(s"Declared address is: $externalSocketAddress.") +// +// if (!networkSettings.localOnly.getOrElse(false)) networkSettings.declaredAddress.foreach(myAddress => +// Try(NetworkInterface.getNetworkInterfaces.asScala.exists(interface => +// interface.getInterfaceAddresses.asScala.exists(interfaceAddress => +// InetAddress.getAllByName(new URI("http://" + myAddress).getHost).contains(interfaceAddress.getAddress) +// ))).recover { case t: Throwable => logger.error(s"Declared address validation failed: $t") } +// ) +// +// IO(Tcp) ! Bind(self, networkSettings.bindAddress, options = KeepAlive(true) :: Nil, pullMode = false) +// +// override def supervisorStrategy: SupervisorStrategy = OneForOneStrategy( +// maxNrOfRetries = 5, +// withinTimeRange = 60 seconds) { +// case _ => Restart +// } +// +// override def receive: Receive = bindingLogic +// .orElse(businessLogic) +// .orElse(peersLogic) +// .orElse { +// case RegisterMessagesHandler(types, handler) => +// logger.info(s"Registering handlers for ${types.mkString(",")}.") +// val ids = types.map(_._1) +// messagesHandlers += (ids -> handler) +// case CommandFailed(cmd: Tcp.Command) => logger.info(s"Failed to execute: $cmd.") +// case msg => logger.warn(s"NetworkController: got something strange $msg.") +// } +// +// def bindingLogic: Receive = { +// case Bound(address) => +// logger.info(s"Successfully bound to the port ${address.getPort}.") +// context.system.scheduler.schedule(2.seconds, 5.second)(peersKeeper ! RequestPeerForConnection) +// case CommandFailed(add: Bind) => +// logger.info(s"Node can't be bind to the address: ${add.localAddress}.") +// context.stop(self) +// } +// +// def businessLogic: Receive = { +// case MessageFromNetwork(message, Some(remote)) if message.isValid(networkSettings.syncPacketLength) => +// logger.debug(s"Got ${message.messageName} on the NetworkController.") +// findHandler(message, message.NetworkMessageTypeID, remote, messagesHandlers) +// case MessageFromNetwork(message, Some(remote)) => +// peersKeeper ! BanPeer(remote, InvalidNetworkMessage(message.messageName)) +// logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") +// } +// +// def peersLogic: Receive = { +// case PeerForConnection(peer) => +// logger.info(s"Network controller got new peer for connection: $peer. Trying to set connection with remote...") +// IO(Tcp) ! Connect( +// peer, +// None, +// KeepAlive(true) :: Nil, +// Some(networkSettings.connectionTimeout), +// pullMode = true +// ) +// +// case Connected(remote, localAddress) => +// logger.info(s"Network controller got 'Connected' message from: $remote. " + +// s"Trying to set stable connection with remote... " + +// s"Local TCP endpoint is: $localAddress.") +// peersKeeper ! NewConnection(remote, sender()) +// +// case ConnectionVerified(remote, remoteConnection, connectionType) => +// logger.info(s"Network controller got approvement for stable connection with: $remote. Starting interaction process...") +// val peerConnectionHandler: ActorRef = context.actorOf( +// PeerConnectionHandler.props(remoteConnection, connectionType, externalSocketAddress, remote, networkSettings) +// .withDispatcher("network-dispatcher") +// ) +// peerConnectionHandler ! StartInteraction +// +// case HandshakedDone(remote) => +// logger.info(s"Network controller got approvement from peer handler about successful handshake. " + +// s"Sending to peerKeeper connected peer.") +// peersKeeper ! HandshakedDone(remote) +// +// case ConnectionStopped(peer) => +// logger.info(s"Network controller got signal about breaking connection with: $peer. " + +// s"Sending to peerKeeper actual information.") +// peersKeeper ! ConnectionStopped(peer) +// nodeViewSync ! ConnectionStopped(peer) +// +// case CommandFailed(connect: Connect) => +// logger.info(s"Failed to connect to: ${connect.remoteAddress}.") +// peersKeeper ! OutgoingConnectionFailed(connect.remoteAddress) +// } +// +// private def findHandler(message: NetworkMessage, +// messageId: Byte, +// remote: ConnectedPeer, +// mH: Map[Seq[Byte], ActorRef]): Unit = +// mH.find(_._1.contains(messageId)).map(_._2) match { +// case Some(handler) => +// handler ! DataFromPeer(message, remote) +// logger.debug(s"Send message DataFromPeer with ${message.messageName} to $handler.") +// case None => logger.info("No handlers found for message: " + message.messageName) +// } +//} object NetworkController { - def props(networkSettings: NetworkSettings, peersKeeper: ActorRef, nodeViewSync: ActorRef): Props = - Props(new NetworkController(networkSettings, peersKeeper, nodeViewSync)) +// def props(networkSettings: NetworkSettings, peersKeeper: ActorRef, nodeViewSync: ActorRef): Props = +// Props(new NetworkController(networkSettings, peersKeeper, nodeViewSync)) object ReceivableMessages { diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index f815b1db88..2cd0ed37cd 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -11,18 +11,21 @@ import encry.network.BlackList.BanReason.InvalidNetworkMessage import encry.network.Messages.MessageToNetwork import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} +import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionVerified, NewConnection, OutgoingConnectionFailed} -import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConnection} +import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConnection, RequestPeerForConnection} import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} +import scala.concurrent.duration._ class NetworkRouter(settings: NetworkSettings, blackListSettings: BlackListSettings) extends Actor with StrictLogging { import context.system + import context.dispatcher var messagesHandlers: Map[Seq[Byte], ActorRef] = Map.empty var handlerForMods: ActorRef = ActorRef.noSender @@ -30,10 +33,10 @@ class NetworkRouter(settings: NetworkSettings, IO(Tcp) ! Bind(self, settings.bindAddress, options = KeepAlive(true) :: Nil, pullMode = false) val peersKeeper = context.system.actorOf(PK.props(settings, blackListSettings), "peersKeeper") - val deliveryManager = context.system.actorOf(DM.props(), "deliveryManager") + val deliveryManager = context.system.actorOf(DM.props(settings), "deliveryManager") val externalSocketAddress: Option[InetSocketAddress] = settings.declaredAddress - override def receive: Receive = bindingLogic orElse businessLogic orElse { + override def receive: Receive = bindingLogic orElse businessLogic orElse peersLogic orElse { case RegisterMessagesHandler(types, handler) => logger.info(s"Registering handlers for ${types.mkString(",")}.") val ids = types.map(_._1) @@ -46,6 +49,7 @@ class NetworkRouter(settings: NetworkSettings, def bindingLogic: Receive = { case Bound(address) => logger.info(s"Successfully bound to the port ${address.getPort}.") + context.system.scheduler.schedule(2.seconds, 5.second)(peersKeeper ! RequestPeerForConnection) case CommandFailed(add: Bind) => logger.info(s"Node can't be bind to the address: ${add.localAddress}.") context.stop(self) @@ -59,7 +63,8 @@ class NetworkRouter(settings: NetworkSettings, peersKeeper ! BanPeer(remote.socketAddress, InvalidNetworkMessage(message.messageName)) logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") case msg: ModifierFromNetwork => handlerForMods ! msg - case msg: MessageToNetwork => context.system.actorOf(MessageBuilder.props(msg, peersKeeper), "peersKeeper") + case msg: OtherNodeSyncingStatus => peersKeeper ! msg + case msg: MessageToNetwork => context.system.actorOf(MessageBuilder.props(msg, peersKeeper, deliveryManager), "peersKeeper") } def peersLogic: Receive = { @@ -113,5 +118,6 @@ object NetworkRouter { case object RegisterForModsHandling - def props(settings: NetworkSettings): Props = Props(new NetworkRouter(settings)) + def props(settings: NetworkSettings, + blackListSettings: BlackListSettings): Props = Props(new NetworkRouter(settings, blackListSettings)) } diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index 3f11d22ae2..8dc068636f 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -37,181 +37,181 @@ import encry.view.fast.sync.SnapshotHolder import encry.view.fast.sync.SnapshotHolder.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks, UpdateSnapshot} import scala.util.Try -class NodeViewSynchronizer(influxRef: Option[ActorRef], - nodeViewHolderRef: ActorRef, - settings: EncryAppSettings, - memoryPoolRef: ActorRef, - dataHolder: ActorRef) extends Actor with StrictLogging { - - val peersKeeper: ActorRef = context.system.actorOf(PeersKeeper.props(settings, self, dataHolder) - .withDispatcher("peers-keeper-dispatcher"), "PeersKeeper") - - - val networkController: ActorRef = context.system.actorOf(NetworkController.props(settings.network, peersKeeper, self) - .withDispatcher("network-dispatcher"), "NetworkController") - - val snapshotHolder: ActorRef = context.system.actorOf(SnapshotHolder.props(settings, networkController, nodeViewHolderRef, self) - .withDispatcher("snapshot-holder-dispatcher"), "snapshotHolder") - - networkController ! RegisterMessagesHandler(Seq( - InvNetworkMessage.NetworkMessageTypeID -> "InvNetworkMessage", - RequestModifiersNetworkMessage.NetworkMessageTypeID -> "RequestModifiersNetworkMessage", - SyncInfoNetworkMessage.NetworkMessageTypeID -> "SyncInfoNetworkMessage" - ), self) - - implicit val timeout: Timeout = Timeout(5.seconds) - - var historyReaderOpt: Option[History] = None - var modifiersRequestCache: Map[String, Array[Byte]] = Map.empty - var chainSynced: Boolean = false - - var canProcessTransactions: Boolean = true - - val downloadedModifiersValidator: ActorRef = context.system - .actorOf(DownloadedModifiersValidator.props(settings.constants.ModifierIdSize, nodeViewHolderRef, - peersKeeper, self, memoryPoolRef, influxRef, settings) - .withDispatcher("Downloaded-Modifiers-Validator-dispatcher"), "DownloadedModifiersValidator") - - val deliveryManager: ActorRef = context.actorOf( - DeliveryManager.props(influxRef, nodeViewHolderRef, networkController, memoryPoolRef, self, - downloadedModifiersValidator, settings) - .withDispatcher("delivery-manager-dispatcher"), "DeliveryManager") - - override def preStart(): Unit = { - context.system.eventStream.subscribe(self, classOf[ModificationOutcome]) - context.system.eventStream.subscribe(self, classOf[ClIMiner]) - context.system.eventStream.subscribe(self, classOf[CLIPeer]) - nodeViewHolderRef ! GetNodeViewChanges(history = true, state = false, vault = false) - } - - override def receive: Receive = awaitingHistoryCycle - - def awaitingHistoryCycle: Receive = { - case msg@ChangedHistory(reader: History) => - logger.info(s"get history: $reader from $sender") - deliveryManager ! UpdatedHistory(reader) - snapshotHolder ! msg - downloadedModifiersValidator ! UpdatedHistory(reader) - context.become(workingCycle(reader)) - case msg@RegisterMessagesHandler(_, _) => networkController ! msg - case msg => logger.info(s"Nvsh got strange message: $msg during history awaiting.") - } - - def workingCycle(history: History): Receive = { - case msg@InvalidModifier(_) => deliveryManager ! msg - case msg@RegisterMessagesHandler(_, _) => networkController ! msg - case SemanticallySuccessfulModifier(mod) => mod match { - case block: Block if chainSynced => - broadcastModifierInv(block.header) - broadcastModifierInv(block.payload) - modifiersRequestCache = Map( - Algos.encode(block.id) -> toProto(block.header), - Algos.encode(block.payload.id) -> toProto(block.payload) - ) - case tx: Transaction => broadcastModifierInv(tx) - case _ => //Do nothing - } - case DataFromPeer(message, remote) => message match { - - case RequestModifiersNetworkMessage((typeId, requestedIds)) if chainSynced || settings.node.offlineGeneration => - val modifiersFromCache: Map[ModifierId, Array[Byte]] = requestedIds - .flatMap(id => modifiersRequestCache - .get(Algos.encode(id)) - .map(id -> _)) - .toMap - if (modifiersFromCache.nonEmpty) remote.handlerRef ! ModifiersNetworkMessage(typeId -> modifiersFromCache) - val unrequestedModifiers: Seq[ModifierId] = requestedIds.filterNot(modifiersFromCache.contains) - - if (unrequestedModifiers.nonEmpty) typeId match { - case Transaction.modifierTypeId => - memoryPoolRef ! RequestModifiersForTransactions(remote, unrequestedModifiers) - } - - case RequestModifiersNetworkMessage(requestedIds) => - logger.info(s"Request from $remote for ${requestedIds._2.size} modifiers discarded cause to chain isn't synced") - - case InvNetworkMessage(invData) if invData._1 == Transaction.modifierTypeId && chainSynced && canProcessTransactions => - memoryPoolRef ! CompareViews(remote, invData._1, invData._2) - case InvNetworkMessage(invData) if invData._1 == Transaction.modifierTypeId => - logger.debug(s"Get inv with tx: ${invData._2.map(Algos.encode).mkString(",")}, but " + - s"chainSynced is $chainSynced and canProcessTransactions is $canProcessTransactions.") - - case _ => logger.debug(s"NodeViewSyncronyzer got invalid type of DataFromPeer message!") - } - case msg@RequestPeersForFirstSyncInfo => - logger.info(s"NodeViewSyncronizer got request from delivery manager to peers keeper for" + - s" peers for first sync info message. Resending $msg to peers keeper.") - peersKeeper ! msg - case msg@UpdatedPeersCollection(_) => deliveryManager ! msg - case msg@PeersForSyncInfo(_) => - logger.info(s"NodeViewSync got peers for sync info. Sending them to DM.") - deliveryManager ! msg - case msg@TreeChunks(l, b) => snapshotHolder ! msg - case msg@ConnectionStopped(_) => deliveryManager ! msg - case msg@StartMining => deliveryManager ! msg - case msg@DisableMining => deliveryManager ! msg - case msg@BanPeer(_, _) => peersKeeper ! msg - case msg@AccumulatedPeersStatistic(_) => peersKeeper ! msg - case msg@SendLocalSyncInfo => peersKeeper ! msg - case msg@RemovePeerFromBlackList(_) => peersKeeper ! msg - case msg@RequiredManifestHeightAndId(_, _) => snapshotHolder ! msg - case msg@SendToNetwork(_, _) => - logger.info(s"NVSH got SendToNetwork") - peersKeeper ! msg - case msg@HeaderChainIsSynced => - snapshotHolder ! msg - case msg@UpdateSnapshot(_, _) => snapshotHolder ! msg - case msg@FastSyncDone => snapshotHolder ! FastSyncDone - case ChangedHistory(reader: History@unchecked) if reader.isInstanceOf[History] => - deliveryManager ! UpdatedHistory(reader) - downloadedModifiersValidator ! UpdatedHistory(reader) - context.become(workingCycle(reader)) - case RequestedModifiersForRemote(remote, txs) => sendResponse( - remote, Transaction.modifierTypeId, txs.map(tx => tx.id -> TransactionProtoSerializer.toProto(tx).toByteArray) - ) - case SuccessfulTransaction(tx) => broadcastModifierInv(tx) - case SemanticallyFailedModification(_, _) => - case SyntacticallyFailedModification(_, _) => - case msg@PeerFromCli(peer) => peersKeeper ! msg - case FullBlockChainIsSynced => - chainSynced = true - deliveryManager ! FullBlockChainIsSynced - peersKeeper ! FullBlockChainIsSynced - if (!settings.snapshotSettings.enableFastSynchronization) snapshotHolder ! FullBlockChainIsSynced - case StopTransactionsValidation => - deliveryManager ! StopTransactionsValidation - canProcessTransactions = false - case StartTransactionsValidation => - deliveryManager ! StartTransactionsValidation - canProcessTransactions = true - case a: Any => logger.error(s"Strange input(sender: ${sender()}): ${a.getClass}\n" + a) - } - - def sendResponse(peer: ConnectedPeer, typeId: ModifierTypeId, modifiersBytes: Seq[(ModifierId, Array[Byte])]): Unit = - if (modifiersBytes.nonEmpty) { - if (typeId != Transaction.modifierTypeId) - logger.debug(s"Sent modifiers to $peer size is: ${modifiersBytes.length}") - typeId match { - case Header.modifierTypeId => - logger.debug(s"Sent to peer handler for $peer ModfiersNetworkMessage for HEADERS with ${modifiersBytes.size} headers." + - s" \n Headers are: ${modifiersBytes.map(x => Algos.encode(x._1)).mkString(",")}.") - peer.handlerRef ! ModifiersNetworkMessage(typeId -> modifiersBytes.toMap) - case Payload.modifierTypeId => - logger.debug(s"Sent to peer handler for $peer ModfiersNetworkMessage for PAYLOADS with ${modifiersBytes.size} payloads." + - s" Mods length: ${modifiersBytes.map(_._2.length).mkString(",")}" + - s" \n Payloads are: ${modifiersBytes.map(x => Algos.encode(x._1)).mkString(",")}.") - peer.handlerRef ! ModifiersNetworkMessage(typeId -> modifiersBytes.toMap) - case Transaction.modifierTypeId => - peer.handlerRef ! ModifiersNetworkMessage(typeId -> modifiersBytes.toMap) - } - } - - def broadcastModifierInv(m: NodeViewModifier): Unit = - if (chainSynced) { - logger.debug(s"NVSH is synced. Going to broadcast inv for: ${m.encodedId}") - peersKeeper ! SendToNetwork(InvNetworkMessage(m.modifierTypeId -> Seq(m.id)), Broadcast) - } -} +//class NodeViewSynchronizer(influxRef: Option[ActorRef], +// nodeViewHolderRef: ActorRef, +// settings: EncryAppSettings, +// memoryPoolRef: ActorRef, +// dataHolder: ActorRef) extends Actor with StrictLogging { +// +// val peersKeeper: ActorRef = context.system.actorOf(PeersKeeper.props(settings, self, dataHolder) +// .withDispatcher("peers-keeper-dispatcher"), "PeersKeeper") +// +// +// val networkController: ActorRef = context.system.actorOf(NetworkController.props(settings.network, peersKeeper, self) +// .withDispatcher("network-dispatcher"), "NetworkController") +// +// val snapshotHolder: ActorRef = context.system.actorOf(SnapshotHolder.props(settings, networkController, nodeViewHolderRef, self) +// .withDispatcher("snapshot-holder-dispatcher"), "snapshotHolder") +// +// networkController ! RegisterMessagesHandler(Seq( +// InvNetworkMessage.NetworkMessageTypeID -> "InvNetworkMessage", +// RequestModifiersNetworkMessage.NetworkMessageTypeID -> "RequestModifiersNetworkMessage", +// SyncInfoNetworkMessage.NetworkMessageTypeID -> "SyncInfoNetworkMessage" +// ), self) +// +// implicit val timeout: Timeout = Timeout(5.seconds) +// +// var historyReaderOpt: Option[History] = None +// var modifiersRequestCache: Map[String, Array[Byte]] = Map.empty +// var chainSynced: Boolean = false +// +// var canProcessTransactions: Boolean = true +// +// val downloadedModifiersValidator: ActorRef = context.system +// .actorOf(DownloadedModifiersValidator.props(settings.constants.ModifierIdSize, nodeViewHolderRef, +// peersKeeper, self, memoryPoolRef, influxRef, settings) +// .withDispatcher("Downloaded-Modifiers-Validator-dispatcher"), "DownloadedModifiersValidator") +// +// val deliveryManager: ActorRef = context.actorOf( +// DeliveryManager.props(influxRef, nodeViewHolderRef, networkController, memoryPoolRef, self, +// downloadedModifiersValidator, settings) +// .withDispatcher("delivery-manager-dispatcher"), "DeliveryManager") +// +// override def preStart(): Unit = { +// context.system.eventStream.subscribe(self, classOf[ModificationOutcome]) +// context.system.eventStream.subscribe(self, classOf[ClIMiner]) +// context.system.eventStream.subscribe(self, classOf[CLIPeer]) +// nodeViewHolderRef ! GetNodeViewChanges(history = true, state = false, vault = false) +// } +// +// override def receive: Receive = awaitingHistoryCycle +// +// def awaitingHistoryCycle: Receive = { +// case msg@ChangedHistory(reader: History) => +// logger.info(s"get history: $reader from $sender") +// deliveryManager ! UpdatedHistory(reader) +// snapshotHolder ! msg +// downloadedModifiersValidator ! UpdatedHistory(reader) +// context.become(workingCycle(reader)) +// case msg@RegisterMessagesHandler(_, _) => networkController ! msg +// case msg => logger.info(s"Nvsh got strange message: $msg during history awaiting.") +// } +// +// def workingCycle(history: History): Receive = { +// case msg@InvalidModifier(_) => deliveryManager ! msg +// case msg@RegisterMessagesHandler(_, _) => networkController ! msg +// case SemanticallySuccessfulModifier(mod) => mod match { +// case block: Block if chainSynced => +// broadcastModifierInv(block.header) +// broadcastModifierInv(block.payload) +// modifiersRequestCache = Map( +// Algos.encode(block.id) -> toProto(block.header), +// Algos.encode(block.payload.id) -> toProto(block.payload) +// ) +// case tx: Transaction => broadcastModifierInv(tx) +// case _ => //Do nothing +// } +// case DataFromPeer(message, remote) => message match { +// +// case RequestModifiersNetworkMessage((typeId, requestedIds)) if chainSynced || settings.node.offlineGeneration => +// val modifiersFromCache: Map[ModifierId, Array[Byte]] = requestedIds +// .flatMap(id => modifiersRequestCache +// .get(Algos.encode(id)) +// .map(id -> _)) +// .toMap +// if (modifiersFromCache.nonEmpty) remote.handlerRef ! ModifiersNetworkMessage(typeId -> modifiersFromCache) +// val unrequestedModifiers: Seq[ModifierId] = requestedIds.filterNot(modifiersFromCache.contains) +// +// if (unrequestedModifiers.nonEmpty) typeId match { +// case Transaction.modifierTypeId => +// memoryPoolRef ! RequestModifiersForTransactions(remote, unrequestedModifiers) +// } +// +// case RequestModifiersNetworkMessage(requestedIds) => +// logger.info(s"Request from $remote for ${requestedIds._2.size} modifiers discarded cause to chain isn't synced") +// +// case InvNetworkMessage(invData) if invData._1 == Transaction.modifierTypeId && chainSynced && canProcessTransactions => +// memoryPoolRef ! CompareViews(remote, invData._1, invData._2) +// case InvNetworkMessage(invData) if invData._1 == Transaction.modifierTypeId => +// logger.debug(s"Get inv with tx: ${invData._2.map(Algos.encode).mkString(",")}, but " + +// s"chainSynced is $chainSynced and canProcessTransactions is $canProcessTransactions.") +// +// case _ => logger.debug(s"NodeViewSyncronyzer got invalid type of DataFromPeer message!") +// } +// case msg@RequestPeersForFirstSyncInfo => +// logger.info(s"NodeViewSyncronizer got request from delivery manager to peers keeper for" + +// s" peers for first sync info message. Resending $msg to peers keeper.") +// peersKeeper ! msg +// case msg@UpdatedPeersCollection(_) => deliveryManager ! msg +// case msg@PeersForSyncInfo(_) => +// logger.info(s"NodeViewSync got peers for sync info. Sending them to DM.") +// deliveryManager ! msg +// case msg@TreeChunks(l, b) => snapshotHolder ! msg +// case msg@ConnectionStopped(_) => deliveryManager ! msg +// case msg@StartMining => deliveryManager ! msg +// case msg@DisableMining => deliveryManager ! msg +// case msg@BanPeer(_, _) => peersKeeper ! msg +// case msg@AccumulatedPeersStatistic(_) => peersKeeper ! msg +// case msg@SendLocalSyncInfo => peersKeeper ! msg +// case msg@RemovePeerFromBlackList(_) => peersKeeper ! msg +// case msg@RequiredManifestHeightAndId(_, _) => snapshotHolder ! msg +// case msg@SendToNetwork(_, _) => +// logger.info(s"NVSH got SendToNetwork") +// peersKeeper ! msg +// case msg@HeaderChainIsSynced => +// snapshotHolder ! msg +// case msg@UpdateSnapshot(_, _) => snapshotHolder ! msg +// case msg@FastSyncDone => snapshotHolder ! FastSyncDone +// case ChangedHistory(reader: History@unchecked) if reader.isInstanceOf[History] => +// deliveryManager ! UpdatedHistory(reader) +// downloadedModifiersValidator ! UpdatedHistory(reader) +// context.become(workingCycle(reader)) +// case RequestedModifiersForRemote(remote, txs) => sendResponse( +// remote, Transaction.modifierTypeId, txs.map(tx => tx.id -> TransactionProtoSerializer.toProto(tx).toByteArray) +// ) +// case SuccessfulTransaction(tx) => broadcastModifierInv(tx) +// case SemanticallyFailedModification(_, _) => +// case SyntacticallyFailedModification(_, _) => +// case msg@PeerFromCli(peer) => peersKeeper ! msg +// case FullBlockChainIsSynced => +// chainSynced = true +// deliveryManager ! FullBlockChainIsSynced +// peersKeeper ! FullBlockChainIsSynced +// if (!settings.snapshotSettings.enableFastSynchronization) snapshotHolder ! FullBlockChainIsSynced +// case StopTransactionsValidation => +// deliveryManager ! StopTransactionsValidation +// canProcessTransactions = false +// case StartTransactionsValidation => +// deliveryManager ! StartTransactionsValidation +// canProcessTransactions = true +// case a: Any => logger.error(s"Strange input(sender: ${sender()}): ${a.getClass}\n" + a) +// } +// +// def sendResponse(peer: ConnectedPeer, typeId: ModifierTypeId, modifiersBytes: Seq[(ModifierId, Array[Byte])]): Unit = +// if (modifiersBytes.nonEmpty) { +// if (typeId != Transaction.modifierTypeId) +// logger.debug(s"Sent modifiers to $peer size is: ${modifiersBytes.length}") +// typeId match { +// case Header.modifierTypeId => +// logger.debug(s"Sent to peer handler for $peer ModfiersNetworkMessage for HEADERS with ${modifiersBytes.size} headers." + +// s" \n Headers are: ${modifiersBytes.map(x => Algos.encode(x._1)).mkString(",")}.") +// peer.handlerRef ! ModifiersNetworkMessage(typeId -> modifiersBytes.toMap) +// case Payload.modifierTypeId => +// logger.debug(s"Sent to peer handler for $peer ModfiersNetworkMessage for PAYLOADS with ${modifiersBytes.size} payloads." + +// s" Mods length: ${modifiersBytes.map(_._2.length).mkString(",")}" + +// s" \n Payloads are: ${modifiersBytes.map(x => Algos.encode(x._1)).mkString(",")}.") +// peer.handlerRef ! ModifiersNetworkMessage(typeId -> modifiersBytes.toMap) +// case Transaction.modifierTypeId => +// peer.handlerRef ! ModifiersNetworkMessage(typeId -> modifiersBytes.toMap) +// } +// } +// +// def broadcastModifierInv(m: NodeViewModifier): Unit = +// if (chainSynced) { +// logger.debug(s"NVSH is synced. Going to broadcast inv for: ${m.encodedId}") +// peersKeeper ! SendToNetwork(InvNetworkMessage(m.modifierTypeId -> Seq(m.id)), Broadcast) +// } +//} object NodeViewSynchronizer { @@ -257,23 +257,23 @@ object NodeViewSynchronizer { } - def props(influxRef: Option[ActorRef], - nodeViewHolderRef: ActorRef, - settings: EncryAppSettings, - memoryPoolRef: ActorRef, - dataHolder: ActorRef): Props = - Props(new NodeViewSynchronizer(influxRef, nodeViewHolderRef, settings, memoryPoolRef, dataHolder)) +// def props(influxRef: Option[ActorRef], +// nodeViewHolderRef: ActorRef, +// settings: EncryAppSettings, +// memoryPoolRef: ActorRef, +// dataHolder: ActorRef): Props = +// Props(new NodeViewSynchronizer(influxRef, nodeViewHolderRef, settings, memoryPoolRef, dataHolder)) class NodeViewSynchronizerPriorityQueue(settings: ActorSystem.Settings, config: Config) extends UnboundedStablePriorityMailbox( PriorityGenerator { - case DataFromPeer(msg, _) => msg match { - case SyncInfoNetworkMessage(_) => 1 - case InvNetworkMessage(data) if data._1 != Transaction.modifierTypeId => 1 - case RequestModifiersNetworkMessage(data) if data._1 != Transaction.modifierTypeId => 2 - case _ => 4 - } +// case DataFromPeer(msg, _) => msg match { +// case SyncInfoNetworkMessage(_) => 1 +// case InvNetworkMessage(data) if data._1 != Transaction.modifierTypeId => 1 +// case RequestModifiersNetworkMessage(data) if data._1 != Transaction.modifierTypeId => 2 +// case _ => 4 +// } case SemanticallySuccessfulModifier(mod) => mod match { case _: Transaction => 4 diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index fd67067aed..c8608160d0 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -4,14 +4,15 @@ import java.net.{InetAddress, InetSocketAddress} import akka.actor.{Actor, Props} import com.typesafe.scalalogging.StrictLogging +import encry.network.BlackList.{BanReason, BanTime, BanType} import encry.network.MessageBuilder.{GetPeerInfo, GetPeers} import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection import encry.network.PeerConnectionHandler.{Incoming, Outgoing} -import encry.network.PeersKeeper.{BanPeer, BanPeerFromAPI} +import encry.network.PeersKeeper.{BanPeer, BanPeerFromAPI, PeerForConnection, RequestPeerForConnection} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} import encry.settings.{BlackListSettings, NetworkSettings} -import scala.util.Try +import scala.util.{Random, Try} class PK(networkSettings: NetworkSettings, blacklistSettings: BlackListSettings) extends Actor with StrictLogging { @@ -33,6 +34,30 @@ class PK(networkSettings: NetworkSettings, .collect { case peer: InetSocketAddress if !isSelf(peer) => peer -> 0 }.toMap override def receive: Receive = banPeersLogic orElse { + case RequestPeerForConnection if connectedPeers.size < networkSettings.maxConnections => + def mapReason(address: InetAddress, r: BanReason, t: BanTime, bt: BanType): (InetAddress, BanReason) = address -> r + logger.info(s"Got request for new connection. Current number of connections is: ${connectedPeers.size}, " + + s"so peer keeper allows to add one more connection. Current available peers are: " + + s"${peersForConnection.mkString(",")}. Current black list is: ${ + blackList.collect((_, _, _, _) => true, mapReason).mkString(",") + }. Current known peers: ${knownPeers.mkString(",")}.") + logger.info(s"awaitingHandshakeConnections ${awaitingHandshakeConnections.mkString(",")}") + logger.info(s"connectedPeers.getAll ${connectedPeers.getAll.mkString(",")}") + val peers = peersForConnection + .filterNot(p => awaitingHandshakeConnections.contains(p._1) || connectedPeers.contains(p._1)) + logger.info(s"peers size: ${peers.size}") + Random.shuffle(peers.toSeq) + .headOption + .foreach { case (peer, _) => + outgoingConnections += peer + logger.info(s"Selected peer: $peer. Sending 'PeerForConnection' message to network controller. " + + s"Adding new outgoing connection to outgoingConnections collection. Current collection is: " + + s"${outgoingConnections.mkString(",")}.") + sender() ! PeerForConnection(peer) + awaitingHandshakeConnections += peer + logger.info(s"Adding new peer: $peer to awaitingHandshakeConnections." + + s" Current is: ${awaitingHandshakeConnections.mkString(",")}") + } case NewConnection(remote, remoteConnection) if connectedPeers.size < networkSettings.maxConnections && !isSelf(remote) => logger.info(s"Peers keeper got request for verifying the connection with remote: $remote. " + s"Remote InetSocketAddress is: $remote. Remote InetAddress is ${remote.getAddress}. " + diff --git a/src/main/scala/encry/network/PeerConnectionHandler.scala b/src/main/scala/encry/network/PeerConnectionHandler.scala index 55b9e0c065..b11a23b428 100755 --- a/src/main/scala/encry/network/PeerConnectionHandler.scala +++ b/src/main/scala/encry/network/PeerConnectionHandler.scala @@ -12,7 +12,7 @@ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp.timeProvider import encry.network.PeerConnectionHandler.{AwaitingHandshake, CommunicationState, _} import encry.network.PeerConnectionHandler.ReceivableMessages._ -import encry.network.PeersKeeper.{ConnectionStopped, HandshakedDone} +import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, HandshakedDone} import encry.settings.NetworkSettings import org.encryfoundation.common.network.BasicMessagesRepo.{GeneralizedNetworkMessage, Handshake, NetworkMessage} import org.encryfoundation.common.utils.Algos diff --git a/src/main/scala/encry/network/PeersKeeper.scala b/src/main/scala/encry/network/PeersKeeper.scala index a6e380fc9e..dc92e76c20 100644 --- a/src/main/scala/encry/network/PeersKeeper.scala +++ b/src/main/scala/encry/network/PeersKeeper.scala @@ -11,11 +11,11 @@ import encry.consensus.HistoryConsensus.HistoryComparisonResult import encry.network.BlackList.BanReason.SentPeersMessageWithoutRequest import encry.network.BlackList.{BanReason, BanTime, BanType} import encry.network.ConnectedPeersCollection.{LastUptime, PeerInfo} -import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler._ import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection +import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, HandshakedDone, NewConnection, OutgoingConnectionFailed} import encry.network.PeersKeeper._ import encry.network.PrioritiesCalculator.AccumulatedPeersStatistic import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus @@ -25,287 +25,287 @@ import org.encryfoundation.common.network.BasicMessagesRepo._ import scala.concurrent.duration._ import scala.util.{Random, Try} - -class PeersKeeper(settings: EncryAppSettings, - nodeViewSync: ActorRef, - dataHolder: ActorRef) extends Actor with StrictLogging { - - import context.dispatcher - - val connectWithOnlyKnownPeers: Boolean = settings.network.connectOnlyWithKnownPeers.getOrElse(true) - - var connectedPeers: ConnectedPeersCollection = ConnectedPeersCollection() - - var blackList: BlackList = BlackList(settings) - - var knownPeers: Set[InetAddress] = settings.network.knownPeers - .collect { case peer: InetSocketAddress if !isSelf(peer) => peer.getAddress }.toSet - - //todo behaviour is incorrect while outgoing connection with connectWithOnlyKnownPeers param - var peersForConnection: Map[InetSocketAddress, Int] = settings.network.knownPeers - .collect { case peer: InetSocketAddress if !isSelf(peer) => peer -> 0 }.toMap - - var awaitingHandshakeConnections: Set[InetSocketAddress] = Set.empty - - var outgoingConnections: Set[InetSocketAddress] = Set.empty - - override def preStart(): Unit = { - nodeViewSync ! RegisterMessagesHandler(Seq( - PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", - GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" - ), self) - if (!connectWithOnlyKnownPeers) context.system.scheduler.schedule(2.seconds, settings.network.syncInterval)( - self ! SendToNetwork(GetPeersNetworkMessage, SendToRandom) - ) - context.system.scheduler.schedule(600.millis, settings.blackList.cleanupTime){blackList = blackList.cleanupBlackList} - context.system.scheduler.schedule(10.seconds, 5.seconds) (dataHolder ! ConnectedPeersConnectionHelper(connectedPeers)) - context.system.scheduler.schedule(10.seconds, 5.seconds)( - nodeViewSync ! UpdatedPeersCollection(connectedPeers.collect(getAllPeers, getPeersForDM).toMap) - ) - context.system.eventStream.subscribe(self, classOf[PeerCommandHelper]) - context.system.scheduler.schedule(5.seconds, 5.seconds){ - dataHolder ! UpdatingPeersInfo( - peersForConnection.keys.toSeq, - connectedPeers.collect(getAllPeers, getConnectedPeers), - blackList.getAll - ) - } - } - - override def receive: Receive = workingBehaviour(isBlockChainSynced = false) - - def workingBehaviour(isBlockChainSynced: Boolean): Receive = setupConnectionsLogic - .orElse(networkMessagesProcessingLogic) - .orElse(banPeersLogic) - .orElse(additionalMessages(isBlockChainSynced)) - - def setupConnectionsLogic: Receive = { - case RequestPeerForConnection if connectedPeers.size < settings.network.maxConnections => - def mapReason(address: InetAddress, r: BanReason, t: BanTime, bt: BanType): (InetAddress, BanReason) = address -> r - logger.info(s"Got request for new connection. Current number of connections is: ${connectedPeers.size}, " + - s"so peer keeper allows to add one more connection. Current available peers are: " + - s"${peersForConnection.mkString(",")}. Current black list is: ${ - blackList.collect((_, _, _, _) => true, mapReason).mkString(",") - }. Current known peers: ${knownPeers.mkString(",")}.") - logger.info(s"awaitingHandshakeConnections ${awaitingHandshakeConnections.mkString(",")}") - logger.info(s"connectedPeers.getAll ${connectedPeers.getAll.mkString(",")}") - val peers = peersForConnection - .filterNot(p => awaitingHandshakeConnections.contains(p._1) || connectedPeers.contains(p._1)) - logger.info(s"peers size: ${peers.size}") - Random.shuffle(peers.toSeq) - .headOption - .foreach { case (peer, _) => - outgoingConnections += peer - logger.info(s"Selected peer: $peer. Sending 'PeerForConnection' message to network controller. " + - s"Adding new outgoing connection to outgoingConnections collection. Current collection is: " + - s"${outgoingConnections.mkString(",")}.") - sender() ! PeerForConnection(peer) - awaitingHandshakeConnections += peer - logger.info(s"Adding new peer: $peer to awaitingHandshakeConnections." + - s" Current is: ${awaitingHandshakeConnections.mkString(",")}") - } - - case RequestPeerForConnection => - logger.info(s"Got request for a new connection but current number of connection is max: ${connectedPeers.size}.") - - case NewConnection(remote, remoteConnection) if connectedPeers.size < settings.network.maxConnections && !isSelf(remote) => - logger.info(s"Peers keeper got request for verifying the connection with remote: $remote. " + - s"Remote InetSocketAddress is: $remote. Remote InetAddress is ${remote.getAddress}. " + - s"Current known peers: ${knownPeers.mkString(",")}") - val notConnectedYet: Boolean = !connectedPeers.contains(remote) - val notBannedPeer: Boolean = !blackList.contains(remote.getAddress) - if (notConnectedYet && notBannedPeer) { - logger.info(s"Peer: $remote is available to setup connect with.") - if (outgoingConnections.contains(remote)) { - logger.info(s"Got outgoing connection.") - outgoingConnections -= remote - sender() ! ConnectionVerified(remote, remoteConnection, Outgoing) - } - else if (connectWithOnlyKnownPeers && knownPeers.contains(remote.getAddress)) { - logger.info(s"connectWithOnlyKnownPeers - true, but connected peer is contained in known peers collection.") - sender() ! ConnectionVerified(remote, remoteConnection, Incoming) - } - else if (connectWithOnlyKnownPeers) - logger.info(s"Got incoming connection but we can connect only with known peers.") - else { - logger.info(s"Got new incoming connection. Sending to network controller approvement for connect.") - sender() ! ConnectionVerified(remote, remoteConnection, Incoming) - } - } else logger.info(s"Connection for requested peer: $remote is unavailable cause of:" + - s" Didn't banned: $notBannedPeer, Didn't connected: $notConnectedYet.") - - case NewConnection(remote, remoteConnection) => - logger.info(s"Peers keeper got request for verifying the connection but current number of max connection is " + - s"bigger than possible or isSelf: ${isSelf(remote)}.") - - case HandshakedDone(connectedPeer) => - logger.info(s"Peers keeper got approvement about finishing a handshake." + - s" Initializing new peer: ${connectedPeer.socketAddress}") - connectedPeers = connectedPeers.initializePeer(connectedPeer) - logger.info(s"Remove ${connectedPeer.socketAddress} from awaitingHandshakeConnections collection. Current is: " + - s"${awaitingHandshakeConnections.mkString(",")}.") - awaitingHandshakeConnections -= connectedPeer.socketAddress - peersForConnection = peersForConnection.updated(connectedPeer.socketAddress, 0) - logger.info(s"Adding new peer: ${connectedPeer.socketAddress} to available collection." + - s" Current collection is: ${peersForConnection.keys.mkString(",")}.") - - case ConnectionStopped(peer) => - logger.info(s"Connection stopped for: $peer.") - awaitingHandshakeConnections -= peer - connectedPeers = connectedPeers.removePeer(peer) - if (blackList.contains(peer.getAddress)) { - peersForConnection -= peer - logger.info(s"Peer: $peer removed from availablePeers cause of it has been banned. " + - s"Current is: ${peersForConnection.mkString(",")}.") - } - - case OutgoingConnectionFailed(peer) => - logger.info(s"Connection failed for: $peer.") - outgoingConnections -= peer - awaitingHandshakeConnections -= peer - val connectionAttempts: Int = peersForConnection.getOrElse(peer, 0) + 1 - if (connectionAttempts >= settings.network.maxNumberOfReConnections) { - logger.info(s"Removing peer: $peer from available peers for ExpiredNumberOfConnections.") - //todo think about penalty for the less time than general ban - //blackList.banPeer(ExpiredNumberOfConnections, peer.getAddress) - peersForConnection -= peer - } else peersForConnection = peersForConnection.updated(peer, connectionAttempts) - } - - def networkMessagesProcessingLogic: Receive = { - case DataFromPeer(message, remote) => message match { - case PeersNetworkMessage(peers) if !connectWithOnlyKnownPeers => - logger.info(s"Got peers message from $remote with peers ${peers.mkString(",")}") - peers - .filterNot { p => - blackList.contains(p.getAddress) || connectedPeers.contains(p) || isSelf(p) || peersForConnection.contains(p) - }.foreach { p => - logger.info(s"Found new peer: $p. Adding it to the available peers collection.") - peersForConnection = peersForConnection.updated(p, 0) - } - logger.info(s"New available peers collection after processing peers from $remote is: ${peersForConnection.keys.mkString(",")}.") - - case PeersNetworkMessage(_) => - logger.info(s"Got PeersNetworkMessage from $remote, but connectWithOnlyKnownPeers: $connectWithOnlyKnownPeers, " + - s"so ignore this message and ban this peer.") - self ! BanPeer(remote, SentPeersMessageWithoutRequest) - - case GetPeersNetworkMessage => - def findPeersForRemote(add: InetSocketAddress, info: PeerInfo): Boolean = - Try { - if (remote.socketAddress.getAddress.isSiteLocalAddress) true - else add.getAddress.isSiteLocalAddress && add != remote.socketAddress - }.getOrElse(false) - - val peers: Seq[InetSocketAddress] = connectedPeers.collect(findPeersForRemote, getPeersForRemote) - logger.info(s"Got request for local known peers. Sending to: $remote peers: ${peers.mkString(",")}.") - logger.info(s"Remote is side local: ${remote.socketAddress} : ${Try(remote.socketAddress.getAddress.isSiteLocalAddress)}") - remote.handlerRef ! PeersNetworkMessage(peers) - } - } - - def additionalMessages(isBlockChainSynced: Boolean): Receive = { - case OtherNodeSyncingStatus(remote, comparison, _) => - connectedPeers = connectedPeers.updateHistoryComparisonResult(Map(remote.socketAddress -> comparison)) - - case AccumulatedPeersStatistic(statistic) => - connectedPeers = connectedPeers.updatePriorityStatus(statistic) - - case SendToNetwork(message, strategy) => - val peers: Seq[ConnectedPeer] = connectedPeers.collect(getAllPeers, getConnectedPeers) - strategy.choose(peers).foreach { peer => - logger.debug(s"Sending message: ${message.messageName} to: ${peer.socketAddress}.") - peer.handlerRef ! message - } - - case SendLocalSyncInfo => - logger.debug(s"Received SendLocalSyncInfo from $sender on PK") - val peersWithHP: Seq[ConnectedPeer] = connectedPeers.collect(filterByPriority(HighPriority), getConnectedPeers) - val peersWithIP: Seq[ConnectedPeer] = connectedPeers.collect(filterByPriority(InitialPriority), getConnectedPeers) - - val accumulatedHPPeers = accumulatePeersForSync(peersWithHP, isBlockChainSynced) - val accumulatedIPPeers = accumulatePeersForSync(peersWithIP, isBlockChainSynced) - val accumulatedPeers = accumulatedHPPeers ++: accumulatedIPPeers - - accumulatedPeers.foreach { p => - logger.debug(s"Update uptime from $p") - connectedPeers = connectedPeers.updateLastUptime(Map(p.socketAddress -> LastUptime(System.currentTimeMillis()))) - } - nodeViewSync ! PeersForSyncInfo(accumulatedPeers) - - context.system.scheduler.scheduleOnce(settings.network.syncInterval) { - logger.debug("Scheduler once for SendLocalSyncInfo triggered") - self ! SendLocalSyncInfo - } - - case PeerFromCli(peer) => - if (!blackList.contains(peer.getAddress) && !peersForConnection.contains(peer) && !connectedPeers.contains(peer) && !isSelf(peer)) { - peersForConnection += (peer -> 0) - knownPeers += peer.getAddress - logger.info(s"Added peer: $peer to known peers. Current newPeers are: ${peersForConnection.mkString(",")}." + - s" Current known peers are: ${knownPeers.mkString(",")}.") - } - - case RemovePeerFromBlackList(peer) => blackList = blackList.remove(peer.getAddress) - - case FullBlockChainIsSynced => - logger.info(s"Peers keeper got message: FullBlockChainIsSynced") - context.become(workingBehaviour(isBlockChainSynced = true)) - - case msg => logger.info(s"Peers keeper got unhandled message: $msg.") - } - - def banPeersLogic: Receive = { - case BanPeer(peer, reason) => - logger.info(s"Banning peer: ${peer.socketAddress} for $reason.") - blackList = blackList.banPeer(reason, peer.socketAddress.getAddress) - peer.handlerRef ! CloseConnection - - case BanPeerFromAPI(peer, reason) => - logger.info(s"Got msg from API... Removing peer: $peer, reason: $reason") - blackList = blackList.banPeer(reason, peer.getAddress) - } - - //todo NPE in InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) - def isSelf(address: InetSocketAddress): Boolean = Try(address == settings.network.bindAddress || - settings.network.declaredAddress.contains(address) || - InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) || - InetAddress.getLoopbackAddress.getAddress.sameElements(address.getAddress.getAddress)).getOrElse(true) - - def filterByPriority(priority: PeersPriorityStatus)(address: InetSocketAddress, info: PeerInfo): Boolean = { - val isTimeRangeConserved: Boolean = (System.currentTimeMillis() - info.lastUptime.time) > settings.network.syncInterval.toMillis - val isNecessaryPriority: Boolean = info.peerPriorityStatus == priority - logger.debug(s"findByPriorityForSync: peer: $address, isTimeRangeConserved: $isTimeRangeConserved," + - s" isNecessaryPriority: $isNecessaryPriority") - isTimeRangeConserved && isNecessaryPriority - } - - def getConnectedPeers(add: InetSocketAddress, info: PeerInfo): ConnectedPeer = info.connectedPeer - - def getPeersForRemote(add: InetSocketAddress, info: PeerInfo): InetSocketAddress = add - - def getPeersForDM(address: InetSocketAddress, info: PeerInfo): (InetSocketAddress, (ConnectedPeer, HistoryComparisonResult, PeersPriorityStatus)) = - address -> (info.connectedPeer, info.historyComparisonResult, info.peerPriorityStatus) - - def getAllPeers: (InetSocketAddress, PeerInfo) => Boolean = (_, _) => true - - def accumulatePeersForSync(peers: Seq[ConnectedPeer], isChainSynced: Boolean): Seq[ConnectedPeer] = peers match { - case coll: Seq[_] if coll.nonEmpty && isChainSynced => - logger.info(s"Peers collection for sync info non empty and block chain is synced. Sending to DM" + - s" peers collection: ${coll.mkString(",")}.") - coll - case coll: Seq[_] if coll.nonEmpty => scala.util.Random.shuffle(coll).headOption.toSeq.map { p => - logger.info(s"Peers collection for sync info non empty but block chain is not synced. Sending to DM" + - s" peer for sync: $p.") - p - } - case _ => - logger.info(s"Peers collection for sync info message is empty.") - Seq.empty[ConnectedPeer] - - } -} - +// +//class PeersKeeper(settings: EncryAppSettings, +// nodeViewSync: ActorRef, +// dataHolder: ActorRef) extends Actor with StrictLogging { +// +// import context.dispatcher +// +// val connectWithOnlyKnownPeers: Boolean = settings.network.connectOnlyWithKnownPeers.getOrElse(true) +// +// var connectedPeers: ConnectedPeersCollection = ConnectedPeersCollection() +// +// var blackList: BlackList = BlackList(settings) +// +// var knownPeers: Set[InetAddress] = settings.network.knownPeers +// .collect { case peer: InetSocketAddress if !isSelf(peer) => peer.getAddress }.toSet +// +// //todo behaviour is incorrect while outgoing connection with connectWithOnlyKnownPeers param +// var peersForConnection: Map[InetSocketAddress, Int] = settings.network.knownPeers +// .collect { case peer: InetSocketAddress if !isSelf(peer) => peer -> 0 }.toMap +// +// var awaitingHandshakeConnections: Set[InetSocketAddress] = Set.empty +// +// var outgoingConnections: Set[InetSocketAddress] = Set.empty +// +// override def preStart(): Unit = { +// nodeViewSync ! RegisterMessagesHandler(Seq( +// PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", +// GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" +// ), self) +// if (!connectWithOnlyKnownPeers) context.system.scheduler.schedule(2.seconds, settings.network.syncInterval)( +// self ! SendToNetwork(GetPeersNetworkMessage, SendToRandom) +// ) +// context.system.scheduler.schedule(600.millis, settings.blackList.cleanupTime){blackList = blackList.cleanupBlackList} +// context.system.scheduler.schedule(10.seconds, 5.seconds) (dataHolder ! ConnectedPeersConnectionHelper(connectedPeers)) +// context.system.scheduler.schedule(10.seconds, 5.seconds)( +// nodeViewSync ! UpdatedPeersCollection(connectedPeers.collect(getAllPeers, getPeersForDM).toMap) +// ) +// context.system.eventStream.subscribe(self, classOf[PeerCommandHelper]) +// context.system.scheduler.schedule(5.seconds, 5.seconds){ +// dataHolder ! UpdatingPeersInfo( +// peersForConnection.keys.toSeq, +// connectedPeers.collect(getAllPeers, getConnectedPeers), +// blackList.getAll +// ) +// } +// } +// +// override def receive: Receive = workingBehaviour(isBlockChainSynced = false) +// +// def workingBehaviour(isBlockChainSynced: Boolean): Receive = setupConnectionsLogic +// .orElse(networkMessagesProcessingLogic) +// .orElse(banPeersLogic) +// .orElse(additionalMessages(isBlockChainSynced)) +// +// def setupConnectionsLogic: Receive = { +// case RequestPeerForConnection if connectedPeers.size < settings.network.maxConnections => +// def mapReason(address: InetAddress, r: BanReason, t: BanTime, bt: BanType): (InetAddress, BanReason) = address -> r +// logger.info(s"Got request for new connection. Current number of connections is: ${connectedPeers.size}, " + +// s"so peer keeper allows to add one more connection. Current available peers are: " + +// s"${peersForConnection.mkString(",")}. Current black list is: ${ +// blackList.collect((_, _, _, _) => true, mapReason).mkString(",") +// }. Current known peers: ${knownPeers.mkString(",")}.") +// logger.info(s"awaitingHandshakeConnections ${awaitingHandshakeConnections.mkString(",")}") +// logger.info(s"connectedPeers.getAll ${connectedPeers.getAll.mkString(",")}") +// val peers = peersForConnection +// .filterNot(p => awaitingHandshakeConnections.contains(p._1) || connectedPeers.contains(p._1)) +// logger.info(s"peers size: ${peers.size}") +// Random.shuffle(peers.toSeq) +// .headOption +// .foreach { case (peer, _) => +// outgoingConnections += peer +// logger.info(s"Selected peer: $peer. Sending 'PeerForConnection' message to network controller. " + +// s"Adding new outgoing connection to outgoingConnections collection. Current collection is: " + +// s"${outgoingConnections.mkString(",")}.") +// sender() ! PeerForConnection(peer) +// awaitingHandshakeConnections += peer +// logger.info(s"Adding new peer: $peer to awaitingHandshakeConnections." + +// s" Current is: ${awaitingHandshakeConnections.mkString(",")}") +// } +// +// case RequestPeerForConnection => +// logger.info(s"Got request for a new connection but current number of connection is max: ${connectedPeers.size}.") +// +// case NewConnection(remote, remoteConnection) if connectedPeers.size < settings.network.maxConnections && !isSelf(remote) => +// logger.info(s"Peers keeper got request for verifying the connection with remote: $remote. " + +// s"Remote InetSocketAddress is: $remote. Remote InetAddress is ${remote.getAddress}. " + +// s"Current known peers: ${knownPeers.mkString(",")}") +// val notConnectedYet: Boolean = !connectedPeers.contains(remote) +// val notBannedPeer: Boolean = !blackList.contains(remote.getAddress) +// if (notConnectedYet && notBannedPeer) { +// logger.info(s"Peer: $remote is available to setup connect with.") +// if (outgoingConnections.contains(remote)) { +// logger.info(s"Got outgoing connection.") +// outgoingConnections -= remote +// sender() ! ConnectionVerified(remote, remoteConnection, Outgoing) +// } +// else if (connectWithOnlyKnownPeers && knownPeers.contains(remote.getAddress)) { +// logger.info(s"connectWithOnlyKnownPeers - true, but connected peer is contained in known peers collection.") +// sender() ! ConnectionVerified(remote, remoteConnection, Incoming) +// } +// else if (connectWithOnlyKnownPeers) +// logger.info(s"Got incoming connection but we can connect only with known peers.") +// else { +// logger.info(s"Got new incoming connection. Sending to network controller approvement for connect.") +// sender() ! ConnectionVerified(remote, remoteConnection, Incoming) +// } +// } else logger.info(s"Connection for requested peer: $remote is unavailable cause of:" + +// s" Didn't banned: $notBannedPeer, Didn't connected: $notConnectedYet.") +// +// case NewConnection(remote, remoteConnection) => +// logger.info(s"Peers keeper got request for verifying the connection but current number of max connection is " + +// s"bigger than possible or isSelf: ${isSelf(remote)}.") +// +// case HandshakedDone(connectedPeer) => +// logger.info(s"Peers keeper got approvement about finishing a handshake." + +// s" Initializing new peer: ${connectedPeer.socketAddress}") +// connectedPeers = connectedPeers.initializePeer(connectedPeer) +// logger.info(s"Remove ${connectedPeer.socketAddress} from awaitingHandshakeConnections collection. Current is: " + +// s"${awaitingHandshakeConnections.mkString(",")}.") +// awaitingHandshakeConnections -= connectedPeer.socketAddress +// peersForConnection = peersForConnection.updated(connectedPeer.socketAddress, 0) +// logger.info(s"Adding new peer: ${connectedPeer.socketAddress} to available collection." + +// s" Current collection is: ${peersForConnection.keys.mkString(",")}.") +// +// case ConnectionStopped(peer) => +// logger.info(s"Connection stopped for: $peer.") +// awaitingHandshakeConnections -= peer +// connectedPeers = connectedPeers.removePeer(peer) +// if (blackList.contains(peer.getAddress)) { +// peersForConnection -= peer +// logger.info(s"Peer: $peer removed from availablePeers cause of it has been banned. " + +// s"Current is: ${peersForConnection.mkString(",")}.") +// } +// +// case OutgoingConnectionFailed(peer) => +// logger.info(s"Connection failed for: $peer.") +// outgoingConnections -= peer +// awaitingHandshakeConnections -= peer +// val connectionAttempts: Int = peersForConnection.getOrElse(peer, 0) + 1 +// if (connectionAttempts >= settings.network.maxNumberOfReConnections) { +// logger.info(s"Removing peer: $peer from available peers for ExpiredNumberOfConnections.") +// //todo think about penalty for the less time than general ban +// //blackList.banPeer(ExpiredNumberOfConnections, peer.getAddress) +// peersForConnection -= peer +// } else peersForConnection = peersForConnection.updated(peer, connectionAttempts) +// } +// +// def networkMessagesProcessingLogic: Receive = { +// case DataFromPeer(message, remote) => message match { +// case PeersNetworkMessage(peers) if !connectWithOnlyKnownPeers => +// logger.info(s"Got peers message from $remote with peers ${peers.mkString(",")}") +// peers +// .filterNot { p => +// blackList.contains(p.getAddress) || connectedPeers.contains(p) || isSelf(p) || peersForConnection.contains(p) +// }.foreach { p => +// logger.info(s"Found new peer: $p. Adding it to the available peers collection.") +// peersForConnection = peersForConnection.updated(p, 0) +// } +// logger.info(s"New available peers collection after processing peers from $remote is: ${peersForConnection.keys.mkString(",")}.") +// +// case PeersNetworkMessage(_) => +// logger.info(s"Got PeersNetworkMessage from $remote, but connectWithOnlyKnownPeers: $connectWithOnlyKnownPeers, " + +// s"so ignore this message and ban this peer.") +// self ! BanPeer(remote, SentPeersMessageWithoutRequest) +// +// case GetPeersNetworkMessage => +// def findPeersForRemote(add: InetSocketAddress, info: PeerInfo): Boolean = +// Try { +// if (remote.socketAddress.getAddress.isSiteLocalAddress) true +// else add.getAddress.isSiteLocalAddress && add != remote.socketAddress +// }.getOrElse(false) +// +// val peers: Seq[InetSocketAddress] = connectedPeers.collect(findPeersForRemote, getPeersForRemote) +// logger.info(s"Got request for local known peers. Sending to: $remote peers: ${peers.mkString(",")}.") +// logger.info(s"Remote is side local: ${remote.socketAddress} : ${Try(remote.socketAddress.getAddress.isSiteLocalAddress)}") +// remote.handlerRef ! PeersNetworkMessage(peers) +// } +// } +// +// def additionalMessages(isBlockChainSynced: Boolean): Receive = { +// case OtherNodeSyncingStatus(remote, comparison, _) => +// connectedPeers = connectedPeers.updateHistoryComparisonResult(Map(remote.socketAddress -> comparison)) +// +// case AccumulatedPeersStatistic(statistic) => +// connectedPeers = connectedPeers.updatePriorityStatus(statistic) +// +// case SendToNetwork(message, strategy) => +// val peers: Seq[ConnectedPeer] = connectedPeers.collect(getAllPeers, getConnectedPeers) +// strategy.choose(peers).foreach { peer => +// logger.debug(s"Sending message: ${message.messageName} to: ${peer.socketAddress}.") +// peer.handlerRef ! message +// } +// +// case SendLocalSyncInfo => +// logger.debug(s"Received SendLocalSyncInfo from $sender on PK") +// val peersWithHP: Seq[ConnectedPeer] = connectedPeers.collect(filterByPriority(HighPriority), getConnectedPeers) +// val peersWithIP: Seq[ConnectedPeer] = connectedPeers.collect(filterByPriority(InitialPriority), getConnectedPeers) +// +// val accumulatedHPPeers = accumulatePeersForSync(peersWithHP, isBlockChainSynced) +// val accumulatedIPPeers = accumulatePeersForSync(peersWithIP, isBlockChainSynced) +// val accumulatedPeers = accumulatedHPPeers ++: accumulatedIPPeers +// +// accumulatedPeers.foreach { p => +// logger.debug(s"Update uptime from $p") +// connectedPeers = connectedPeers.updateLastUptime(Map(p.socketAddress -> LastUptime(System.currentTimeMillis()))) +// } +// nodeViewSync ! PeersForSyncInfo(accumulatedPeers) +// +// context.system.scheduler.scheduleOnce(settings.network.syncInterval) { +// logger.debug("Scheduler once for SendLocalSyncInfo triggered") +// self ! SendLocalSyncInfo +// } +// +// case PeerFromCli(peer) => +// if (!blackList.contains(peer.getAddress) && !peersForConnection.contains(peer) && !connectedPeers.contains(peer) && !isSelf(peer)) { +// peersForConnection += (peer -> 0) +// knownPeers += peer.getAddress +// logger.info(s"Added peer: $peer to known peers. Current newPeers are: ${peersForConnection.mkString(",")}." + +// s" Current known peers are: ${knownPeers.mkString(",")}.") +// } +// +// case RemovePeerFromBlackList(peer) => blackList = blackList.remove(peer.getAddress) +// +// case FullBlockChainIsSynced => +// logger.info(s"Peers keeper got message: FullBlockChainIsSynced") +// context.become(workingBehaviour(isBlockChainSynced = true)) +// +// case msg => logger.info(s"Peers keeper got unhandled message: $msg.") +// } +// +// def banPeersLogic: Receive = { +// case BanPeer(peer, reason) => +// logger.info(s"Banning peer: ${peer.socketAddress} for $reason.") +// blackList = blackList.banPeer(reason, peer.socketAddress.getAddress) +// peer.handlerRef ! CloseConnection +// +// case BanPeerFromAPI(peer, reason) => +// logger.info(s"Got msg from API... Removing peer: $peer, reason: $reason") +// blackList = blackList.banPeer(reason, peer.getAddress) +// } +// +// //todo NPE in InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) +// def isSelf(address: InetSocketAddress): Boolean = Try(address == settings.network.bindAddress || +// settings.network.declaredAddress.contains(address) || +// InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) || +// InetAddress.getLoopbackAddress.getAddress.sameElements(address.getAddress.getAddress)).getOrElse(true) +// +// def filterByPriority(priority: PeersPriorityStatus)(address: InetSocketAddress, info: PeerInfo): Boolean = { +// val isTimeRangeConserved: Boolean = (System.currentTimeMillis() - info.lastUptime.time) > settings.network.syncInterval.toMillis +// val isNecessaryPriority: Boolean = info.peerPriorityStatus == priority +// logger.debug(s"findByPriorityForSync: peer: $address, isTimeRangeConserved: $isTimeRangeConserved," + +// s" isNecessaryPriority: $isNecessaryPriority") +// isTimeRangeConserved && isNecessaryPriority +// } +// +// def getConnectedPeers(add: InetSocketAddress, info: PeerInfo): ConnectedPeer = info.connectedPeer +// +// def getPeersForRemote(add: InetSocketAddress, info: PeerInfo): InetSocketAddress = add +// +// def getPeersForDM(address: InetSocketAddress, info: PeerInfo): (InetSocketAddress, (ConnectedPeer, HistoryComparisonResult, PeersPriorityStatus)) = +// address -> (info.connectedPeer, info.historyComparisonResult, info.peerPriorityStatus) +// +// def getAllPeers: (InetSocketAddress, PeerInfo) => Boolean = (_, _) => true +// +// def accumulatePeersForSync(peers: Seq[ConnectedPeer], isChainSynced: Boolean): Seq[ConnectedPeer] = peers match { +// case coll: Seq[_] if coll.nonEmpty && isChainSynced => +// logger.info(s"Peers collection for sync info non empty and block chain is synced. Sending to DM" + +// s" peers collection: ${coll.mkString(",")}.") +// coll +// case coll: Seq[_] if coll.nonEmpty => scala.util.Random.shuffle(coll).headOption.toSeq.map { p => +// logger.info(s"Peers collection for sync info non empty but block chain is not synced. Sending to DM" + +// s" peer for sync: $p.") +// p +// } +// case _ => +// logger.info(s"Peers collection for sync info message is empty.") +// Seq.empty[ConnectedPeer] +// +// } +//} +// object PeersKeeper { sealed trait PeerCommandHelper @@ -345,9 +345,9 @@ object PeersKeeper { case object GetKnownPeers - def props(settings: EncryAppSettings, - nodeViewSync: ActorRef, - dataHolder: ActorRef): Props = Props(new PeersKeeper(settings, nodeViewSync, dataHolder)) +// def props(settings: EncryAppSettings, +// nodeViewSync: ActorRef, +// dataHolder: ActorRef): Props = Props(new PeersKeeper(settings, nodeViewSync, dataHolder)) class PeersKeeperPriorityQueue(settings: ActorSystem.Settings, config: Config) extends UnboundedStablePriorityMailbox( diff --git a/src/test/scala/encry/network/BlackListTests.scala b/src/test/scala/encry/network/BlackListTests.scala index 77ba1eea69..d71faf7d77 100644 --- a/src/test/scala/encry/network/BlackListTests.scala +++ b/src/test/scala/encry/network/BlackListTests.scala @@ -14,100 +14,100 @@ import org.encryfoundation.common.network.BasicMessagesRepo.Handshake import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} import scala.concurrent.duration._ -class BlackListTests extends WordSpecLike - with Matchers - with BeforeAndAfterAll - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - implicit val system: ActorSystem = ActorSystem() - - override def afterAll(): Unit = system.terminate() - - val knowPeersSettings = testNetSettings.copy( - network = settings.network.copy( - knownPeers = List(new InetSocketAddress("172.16.11.11", 9001)), - connectOnlyWithKnownPeers = Some(true) - ), - blackList = settings.blackList.copy( - banTime = 2 seconds, - cleanupTime = 3 seconds - )) - - /* - Unit tests - */ - "Black list" should { - "temporary ban requested peer correctly" in { - val blackList: BlackList = BlackList(knowPeersSettings) - val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress - val newBL = blackList.banPeer(SemanticallyInvalidPersistentModifier, peer) - newBL.contains(peer) shouldBe true - } - "clean black list from peers with expired ban time which were banned by temporary ban" in { - val blackList: BlackList = BlackList(knowPeersSettings) - val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress - val newBL = blackList.banPeer(SyntacticallyInvalidPersistentModifier, peer) - Thread.sleep(2000) - val newBL1 = newBL.cleanupBlackList - newBL1.contains(peer) shouldBe false - } - "don't remove peer from black list before ban time expired" in { - val blackList: BlackList = BlackList(knowPeersSettings) - val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress - val newBL = blackList.banPeer(SentInvForPayload, peer) - val newBL1 = newBL.cleanupBlackList - newBL1.contains(peer) shouldBe true - } - } - - /* - Akka tests - */ - "Peers keeper" should { - "handle ban peer message correctly" in { - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) - val peerHandler: TestProbe = TestProbe() - val connectedPeer: ConnectedPeer = ConnectedPeer( - address, - peerHandler.ref, - Outgoing, - Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) - ) - peersKeeper ! BanPeer(connectedPeer, SpamSender) - peerHandler.expectMsg(CloseConnection) - peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe true - } - "cleanup black list by scheduler correctly" in { - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) - val peerHandler: TestProbe = TestProbe() - val connectedPeer: ConnectedPeer = ConnectedPeer( - address, - peerHandler.ref, - Outgoing, - Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) - ) - peersKeeper ! BanPeer(connectedPeer, SentPeersMessageWithoutRequest) - Thread.sleep(6000) - peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe false - } - "don't remove peer from black list before ban time expired" in { - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) - val peerHandler: TestProbe = TestProbe() - val connectedPeer: ConnectedPeer = ConnectedPeer( - address, - peerHandler.ref, - Outgoing, - Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) - ) - Thread.sleep(4000) - peersKeeper ! BanPeer(connectedPeer, CorruptedSerializedBytes) - Thread.sleep(2000) - peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe true - } - } -} \ No newline at end of file +//class BlackListTests extends WordSpecLike +// with Matchers +// with BeforeAndAfterAll +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// +// implicit val system: ActorSystem = ActorSystem() +// +// override def afterAll(): Unit = system.terminate() +// +// val knowPeersSettings = testNetSettings.copy( +// network = settings.network.copy( +// knownPeers = List(new InetSocketAddress("172.16.11.11", 9001)), +// connectOnlyWithKnownPeers = Some(true) +// ), +// blackList = settings.blackList.copy( +// banTime = 2 seconds, +// cleanupTime = 3 seconds +// )) +// +// /* +// Unit tests +// */ +// "Black list" should { +// "temporary ban requested peer correctly" in { +// val blackList: BlackList = BlackList(settings.blackList) +// val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress +// val newBL = blackList.banPeer(SemanticallyInvalidPersistentModifier, peer) +// newBL.contains(peer) shouldBe true +// } +// "clean black list from peers with expired ban time which were banned by temporary ban" in { +// val blackList: BlackList = BlackList(settings.blackList) +// val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress +// val newBL = blackList.banPeer(SyntacticallyInvalidPersistentModifier, peer) +// Thread.sleep(2000) +// val newBL1 = newBL.cleanupBlackList +// newBL1.contains(peer) shouldBe false +// } +// "don't remove peer from black list before ban time expired" in { +// val blackList: BlackList = BlackList(settings.blackList) +// val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress +// val newBL = blackList.banPeer(SentInvForPayload, peer) +// val newBL1 = newBL.cleanupBlackList +// newBL1.contains(peer) shouldBe true +// } +// } +// +// /* +// Akka tests +// */ +// "Peers keeper" should { +// "handle ban peer message correctly" in { +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) +// val peerHandler: TestProbe = TestProbe() +// val connectedPeer: ConnectedPeer = ConnectedPeer( +// address, +// peerHandler.ref, +// Outgoing, +// Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) +// ) +// peersKeeper ! BanPeer(connectedPeer, SpamSender) +// peerHandler.expectMsg(CloseConnection) +// peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe true +// } +// "cleanup black list by scheduler correctly" in { +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) +// val peerHandler: TestProbe = TestProbe() +// val connectedPeer: ConnectedPeer = ConnectedPeer( +// address, +// peerHandler.ref, +// Outgoing, +// Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) +// ) +// peersKeeper ! BanPeer(connectedPeer, SentPeersMessageWithoutRequest) +// Thread.sleep(6000) +// peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe false +// } +// "don't remove peer from black list before ban time expired" in { +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) +// val peerHandler: TestProbe = TestProbe() +// val connectedPeer: ConnectedPeer = ConnectedPeer( +// address, +// peerHandler.ref, +// Outgoing, +// Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) +// ) +// Thread.sleep(4000) +// peersKeeper ! BanPeer(connectedPeer, CorruptedSerializedBytes) +// Thread.sleep(2000) +// peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe true +// } +// } +//} \ No newline at end of file diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index bdb2126515..5f8160f949 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -14,28 +14,28 @@ import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, PeersNet import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} import scala.concurrent.duration._ -class ConnectWithNewPeerTests extends WordSpecLike - with Matchers - with BeforeAndAfterAll - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - implicit val system: ActorSystem = ActorSystem() - - override def afterAll(): Unit = system.terminate() - - val knowPeersSettings = testNetSettings.copy( - network = testNetSettings.network.copy( - knownPeers = List(new InetSocketAddress("172.16.11.11", 9001)), - connectOnlyWithKnownPeers = Some(true) - ), - blackList = testNetSettings.blackList.copy( - banTime = 2 seconds, - cleanupTime = 3 seconds - )) - - "Peers keeper" should { +//class ConnectWithNewPeerTests extends WordSpecLike +// with Matchers +// with BeforeAndAfterAll +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// +// implicit val system: ActorSystem = ActorSystem() +// +// override def afterAll(): Unit = system.terminate() +// +// val knowPeersSettings = testNetSettings.copy( +// network = testNetSettings.network.copy( +// knownPeers = List(new InetSocketAddress("172.16.11.11", 9001)), +// connectOnlyWithKnownPeers = Some(true) +// ), +// blackList = testNetSettings.blackList.copy( +// banTime = 2 seconds, +// cleanupTime = 3 seconds +// )) +// +// "Peers keeper" should { // "maintain outgoing connection process correctly" in { // /* Request first peer while current number of connections is 0 */ // val networkController = TestProbe() @@ -160,185 +160,185 @@ class ConnectWithNewPeerTests extends WordSpecLike // peersKeeper.underlyingActor.knownPeers.contains(availablePeers.head._1) shouldBe false // peersKeeper.underlyingActor.blackList.contains(availablePeers.head._1.getAddress) shouldBe true // } - "remove peer from available if it has been banned" in { - val networkController = TestProbe() - val nodeViewSync = TestProbe() - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, nodeViewSync.ref, TestProbe().ref)) - - val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.peersForConnection - - val peerHandler = TestProbe() - val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) - - networkController.send(peersKeeper, BanPeer(connectedPeer, ExpiredNumberOfConnections)) - networkController.send(peersKeeper, ConnectionStopped(availablePeers.head._1)) - - peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false - } - "filter peers from network message" in { - val networkController = TestProbe() - val nodeViewSync = TestProbe() - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, nodeViewSync.ref, TestProbe().ref)) - - val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.peersForConnection - - val peerHandler = TestProbe() - val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) - - val newPeerHandler = TestProbe() - val newConnectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.last._1, newPeerHandler.ref, Outgoing, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test-peer", Some(availablePeers.last._1), System.currentTimeMillis())) - - networkController.send(peersKeeper, BanPeer(connectedPeer, ExpiredNumberOfConnections)) - networkController.send(peersKeeper, ConnectionStopped(availablePeers.head._1)) - networkController.send(peersKeeper, HandshakedDone(newConnectedPeer)) - peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false - - val peer = new InetSocketAddress("172.16.28.98", 9023) - val peers = Seq(availablePeers.last._1, availablePeers.head._1, peer) - - networkController.send(peersKeeper, DataFromPeer(PeersNetworkMessage(peers), newConnectedPeer)) - peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false - peersKeeper.underlyingActor.peersForConnection.contains(peer) shouldBe true - } - "handle successful connection process" in { - val networkController = TestProbe() - val peersSenderProbe = TestProbe() - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - val connectedPeer: ConnectedPeer = ConnectedPeer(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test-peer", Some(testNetSettings.network.knownPeers.head), System.currentTimeMillis())) - - networkController.send(peersKeeper, RequestPeerForConnection) - networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) - peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true - peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true - - networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) - networkController.expectMsg( - ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) - - networkController.send(peersKeeper, HandshakedDone(connectedPeer)) - peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe true - } - "handle stop connection process" in { - val networkController = TestProbe() - val peersSenderProbe = TestProbe() - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - val connectedPeer: ConnectedPeer = ConnectedPeer(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test-peer", Some(testNetSettings.network.knownPeers.head), System.currentTimeMillis())) - - networkController.send(peersKeeper, RequestPeerForConnection) - networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) - peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true - peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true - - networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) - networkController.expectMsg( - ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) - - networkController.send(peersKeeper, HandshakedDone(connectedPeer)) - peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe true - - peersKeeper ! ConnectionStopped(testNetSettings.network.knownPeers.head) - peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe false - peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true - } - "handle failed connection process" in { - val networkController = TestProbe() - val peersSenderProbe = TestProbe() - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - - networkController.send(peersKeeper, RequestPeerForConnection) - networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) - peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true - peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true - - networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) - networkController.expectMsg( - ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) - - peersKeeper ! OutgoingConnectionFailed(testNetSettings.network.knownPeers.head) - peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe false - peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true - } - "handle incoming connections correctly while connection with only known peers false " + - "and incoming peer doesn't contains in black list and connected peers collection" in { - val networkController = TestProbe() - val remoteConnectionTestProbe: TestProbe = TestProbe() - val remoteAddress: InetSocketAddress = testNetSettings.network.knownPeers.head - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) - - networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) - networkController.expectMsg(ConnectionVerified(remoteAddress, remoteConnectionTestProbe.ref, Incoming)) - peersKeeper.stop() - } - "handle incoming connections correctly while connection with only known peers false " + - "and incoming peer contain in black list" in { - val networkController = TestProbe() - val remoteConnectionTestProbe: TestProbe = TestProbe() - val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.11", 9001) - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) - val connectedPeer: ConnectedPeer = ConnectedPeer(remoteAddress, remoteConnectionTestProbe.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test-peer", Some(remoteAddress), System.currentTimeMillis())) - - peersKeeper ! BanPeer(connectedPeer, SyntacticallyInvalidPersistentModifier) - networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) - networkController.expectNoMsg() - peersKeeper.stop() - } - "handle incoming connections correctly while connection with only known peers false " + - "and incoming peer contain in connected peers" in { - val networkController = TestProbe() - val remoteConnectionTestProbe: TestProbe = TestProbe() - val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.11", 9001) - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) - val connectedPeer: ConnectedPeer = ConnectedPeer(remoteAddress, remoteConnectionTestProbe.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test-peer", Some(remoteAddress), System.currentTimeMillis())) - - peersKeeper ! HandshakedDone(connectedPeer) - networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) - networkController.expectNoMsg() - peersKeeper.stop() - } - "handle incoming connections correctly while connection with only known peers true" in { - val networkController = TestProbe() - val remoteConnectionTestProbe: TestProbe = TestProbe() - val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.99", 9001) - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - - networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) - networkController.expectNoMsg() - peersKeeper.stop() - } - "handle incoming connections correctly while peer is equal to local address" in { - val networkController = TestProbe() - val remoteConnectionTestProbe: TestProbe = TestProbe() - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) - - networkController.send(peersKeeper, NewConnection( - new InetSocketAddress("0.0.0.0", 9001), remoteConnectionTestProbe.ref)) - networkController.expectNoMsg() - peersKeeper.stop() - } - "handle outgoing connection" in { - val networkController = TestProbe() - val remoteConnectionTestProbe: TestProbe = TestProbe() - val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) - - peersKeeper ! RequestPeerForConnection - peersKeeper.underlyingActor.outgoingConnections.contains(knowPeersSettings.network.knownPeers.head) shouldBe true - networkController.send(peersKeeper, NewConnection(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref)) - networkController.expectMsg( - ConnectionVerified(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref, Outgoing)) - } - } -} \ No newline at end of file +// "remove peer from available if it has been banned" in { +// val networkController = TestProbe() +// val nodeViewSync = TestProbe() +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, nodeViewSync.ref, TestProbe().ref)) +// +// val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.peersForConnection +// +// val peerHandler = TestProbe() +// val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) +// +// networkController.send(peersKeeper, BanPeer(connectedPeer, ExpiredNumberOfConnections)) +// networkController.send(peersKeeper, ConnectionStopped(availablePeers.head._1)) +// +// peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false +// } +// "filter peers from network message" in { +// val networkController = TestProbe() +// val nodeViewSync = TestProbe() +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, nodeViewSync.ref, TestProbe().ref)) +// +// val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.peersForConnection +// +// val peerHandler = TestProbe() +// val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) +// +// val newPeerHandler = TestProbe() +// val newConnectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.last._1, newPeerHandler.ref, Outgoing, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test-peer", Some(availablePeers.last._1), System.currentTimeMillis())) +// +// networkController.send(peersKeeper, BanPeer(connectedPeer, ExpiredNumberOfConnections)) +// networkController.send(peersKeeper, ConnectionStopped(availablePeers.head._1)) +// networkController.send(peersKeeper, HandshakedDone(newConnectedPeer)) +// peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false +// +// val peer = new InetSocketAddress("172.16.28.98", 9023) +// val peers = Seq(availablePeers.last._1, availablePeers.head._1, peer) +// +// networkController.send(peersKeeper, DataFromPeer(PeersNetworkMessage(peers), newConnectedPeer)) +// peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false +// peersKeeper.underlyingActor.peersForConnection.contains(peer) shouldBe true +// } +// "handle successful connection process" in { +// val networkController = TestProbe() +// val peersSenderProbe = TestProbe() +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) +// val connectedPeer: ConnectedPeer = ConnectedPeer(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test-peer", Some(testNetSettings.network.knownPeers.head), System.currentTimeMillis())) +// +// networkController.send(peersKeeper, RequestPeerForConnection) +// networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) +// peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true +// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true +// +// networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) +// networkController.expectMsg( +// ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) +// +// networkController.send(peersKeeper, HandshakedDone(connectedPeer)) +// peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe true +// } +// "handle stop connection process" in { +// val networkController = TestProbe() +// val peersSenderProbe = TestProbe() +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) +// val connectedPeer: ConnectedPeer = ConnectedPeer(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test-peer", Some(testNetSettings.network.knownPeers.head), System.currentTimeMillis())) +// +// networkController.send(peersKeeper, RequestPeerForConnection) +// networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) +// peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true +// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true +// +// networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) +// networkController.expectMsg( +// ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) +// +// networkController.send(peersKeeper, HandshakedDone(connectedPeer)) +// peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe true +// +// peersKeeper ! ConnectionStopped(testNetSettings.network.knownPeers.head) +// peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe false +// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true +// } +// "handle failed connection process" in { +// val networkController = TestProbe() +// val peersSenderProbe = TestProbe() +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) +// +// networkController.send(peersKeeper, RequestPeerForConnection) +// networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) +// peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true +// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true +// +// networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) +// networkController.expectMsg( +// ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) +// +// peersKeeper ! OutgoingConnectionFailed(testNetSettings.network.knownPeers.head) +// peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe false +// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true +// } +// "handle incoming connections correctly while connection with only known peers false " + +// "and incoming peer doesn't contains in black list and connected peers collection" in { +// val networkController = TestProbe() +// val remoteConnectionTestProbe: TestProbe = TestProbe() +// val remoteAddress: InetSocketAddress = testNetSettings.network.knownPeers.head +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) +// +// networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) +// networkController.expectMsg(ConnectionVerified(remoteAddress, remoteConnectionTestProbe.ref, Incoming)) +// peersKeeper.stop() +// } +// "handle incoming connections correctly while connection with only known peers false " + +// "and incoming peer contain in black list" in { +// val networkController = TestProbe() +// val remoteConnectionTestProbe: TestProbe = TestProbe() +// val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.11", 9001) +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) +// val connectedPeer: ConnectedPeer = ConnectedPeer(remoteAddress, remoteConnectionTestProbe.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test-peer", Some(remoteAddress), System.currentTimeMillis())) +// +// peersKeeper ! BanPeer(connectedPeer, SyntacticallyInvalidPersistentModifier) +// networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) +// networkController.expectNoMsg() +// peersKeeper.stop() +// } +// "handle incoming connections correctly while connection with only known peers false " + +// "and incoming peer contain in connected peers" in { +// val networkController = TestProbe() +// val remoteConnectionTestProbe: TestProbe = TestProbe() +// val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.11", 9001) +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) +// val connectedPeer: ConnectedPeer = ConnectedPeer(remoteAddress, remoteConnectionTestProbe.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test-peer", Some(remoteAddress), System.currentTimeMillis())) +// +// peersKeeper ! HandshakedDone(connectedPeer) +// networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) +// networkController.expectNoMsg() +// peersKeeper.stop() +// } +// "handle incoming connections correctly while connection with only known peers true" in { +// val networkController = TestProbe() +// val remoteConnectionTestProbe: TestProbe = TestProbe() +// val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.99", 9001) +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) +// +// networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) +// networkController.expectNoMsg() +// peersKeeper.stop() +// } +// "handle incoming connections correctly while peer is equal to local address" in { +// val networkController = TestProbe() +// val remoteConnectionTestProbe: TestProbe = TestProbe() +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) +// +// networkController.send(peersKeeper, NewConnection( +// new InetSocketAddress("0.0.0.0", 9001), remoteConnectionTestProbe.ref)) +// networkController.expectNoMsg() +// peersKeeper.stop() +// } +// "handle outgoing connection" in { +// val networkController = TestProbe() +// val remoteConnectionTestProbe: TestProbe = TestProbe() +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) +// +// peersKeeper ! RequestPeerForConnection +// peersKeeper.underlyingActor.outgoingConnections.contains(knowPeersSettings.network.knownPeers.head) shouldBe true +// networkController.send(peersKeeper, NewConnection(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref)) +// networkController.expectMsg( +// ConnectionVerified(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref, Outgoing)) +// } +// } +//} \ No newline at end of file diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala b/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala index 0bb95839cc..03d54c59cb 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala @@ -17,41 +17,41 @@ import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.mutable import scala.collection.mutable.WrappedArray -object DMUtils extends InstanceFactory { - - def initialiseDeliveryManager(isBlockChainSynced: Boolean, - isMining: Boolean, - settings: EncryAppSettings) - (implicit actorSystem: ActorSystem): (TestActorRef[DeliveryManager], History) = { - val history: History = generateDummyHistory(settings) - val deliveryManager: TestActorRef[DeliveryManager] = - TestActorRef[DeliveryManager](DeliveryManager - .props(None, TestProbe().ref, TestProbe().ref, TestProbe().ref, TestProbe().ref, TestProbe().ref, settings)) - deliveryManager ! UpdatedHistory(history) - if (isMining) deliveryManager ! StartMining - else deliveryManager ! DisableMining - if (isBlockChainSynced) deliveryManager ! FullBlockChainIsSynced - (deliveryManager, history) - } - - def generateBlocks(qty: Int, history: History): (History, List[Block]) = - (0 until qty).foldLeft(history, List.empty[Block]) { - case ((prevHistory, blocks), _) => - val block: Block = generateNextBlock(prevHistory) - prevHistory.append(block.header) - prevHistory.append(block.payload) - val a = prevHistory.reportModifierIsValid(block) - (a, blocks :+ block) - } - - def toKey(id: ModifierId): WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) - - def createPeer(port: Int, - host: String, - settings: EncryAppSettings)(implicit system: ActorSystem): (InetSocketAddress, ConnectedPeer) = { - val address = new InetSocketAddress(host, port) - val peer: ConnectedPeer = ConnectedPeer(address, TestProbe().ref, Incoming, - Handshake(protocolToBytes(settings.network.appVersion), host, Some(address), System.currentTimeMillis())) - (address, peer) - } -} \ No newline at end of file +//object DMUtils extends InstanceFactory { +// +// def initialiseDeliveryManager(isBlockChainSynced: Boolean, +// isMining: Boolean, +// settings: EncryAppSettings) +// (implicit actorSystem: ActorSystem): (TestActorRef[DeliveryManager], History) = { +// val history: History = generateDummyHistory(settings) +// val deliveryManager: TestActorRef[DeliveryManager] = +// TestActorRef[DeliveryManager](DeliveryManager +// .props(None, TestProbe().ref, TestProbe().ref, TestProbe().ref, TestProbe().ref, TestProbe().ref, settings)) +// deliveryManager ! UpdatedHistory(history) +// if (isMining) deliveryManager ! StartMining +// else deliveryManager ! DisableMining +// if (isBlockChainSynced) deliveryManager ! FullBlockChainIsSynced +// (deliveryManager, history) +// } +// +// def generateBlocks(qty: Int, history: History): (History, List[Block]) = +// (0 until qty).foldLeft(history, List.empty[Block]) { +// case ((prevHistory, blocks), _) => +// val block: Block = generateNextBlock(prevHistory) +// prevHistory.append(block.header) +// prevHistory.append(block.payload) +// val a = prevHistory.reportModifierIsValid(block) +// (a, blocks :+ block) +// } +// +// def toKey(id: ModifierId): WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) +// +// def createPeer(port: Int, +// host: String, +// settings: EncryAppSettings)(implicit system: ActorSystem): (InetSocketAddress, ConnectedPeer) = { +// val address = new InetSocketAddress(host, port) +// val peer: ConnectedPeer = ConnectedPeer(address, TestProbe().ref, Incoming, +// Handshake(protocolToBytes(settings.network.appVersion), host, Some(address), System.currentTimeMillis())) +// (address, peer) +// } +//} \ No newline at end of file diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala index 410373e2b3..fb358c98b8 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala @@ -2,7 +2,7 @@ package encry.network.DeliveryManagerTests import java.net.InetSocketAddress -import encry.network.DeliveryManagerTests.DMUtils.{createPeer, generateBlocks, initialiseDeliveryManager} +//import encry.network.DeliveryManagerTests.DMUtils.{createPeer, generateBlocks, initialiseDeliveryManager} import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestKit} import encry.consensus.HistoryConsensus @@ -10,7 +10,7 @@ import encry.consensus.HistoryConsensus.{Equal, Older, Younger} import encry.modifiers.InstanceFactory import encry.network.DeliveryManager import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.NodeViewSynchronizer.ReceivableMessages.RequestFromLocal +//import encry.network.NodeViewSynchronizer.ReceivableMessages.RequestFromLocal import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus import encry.network.PeersKeeper.UpdatedPeersCollection @@ -21,225 +21,225 @@ import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMess import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -class DeliveryManagerPriorityTests extends WordSpecLike - with BeforeAndAfterAll - with Matchers - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") - - override def afterAll: Unit = TestKit.shutdownActorSystem(system) - - def initialiseState: (TestActorRef[DeliveryManager], ConnectedPeer, ConnectedPeer, ConnectedPeer, - ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, - List[Block], List[ModifierId]) = { - val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = true, isMining = true, testNetSettings) - val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) - val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) - val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) - val (_: InetSocketAddress, cp4: ConnectedPeer) = createPeer(9004, "172.16.13.13", testNetSettings) - val (_: InetSocketAddress, cp5: ConnectedPeer) = createPeer(9005, "172.16.13.14", testNetSettings) - val (_: InetSocketAddress, cp6: ConnectedPeer) = createPeer(9006, "172.16.13.15", testNetSettings) - val (_: InetSocketAddress, cp7: ConnectedPeer) = createPeer(9007, "172.16.13.16", testNetSettings) - val (_: InetSocketAddress, cp8: ConnectedPeer) = createPeer(9008, "172.16.13.17", testNetSettings) - val (_: InetSocketAddress, cp9: ConnectedPeer) = createPeer(9009, "172.16.13.18", testNetSettings) - val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 - val headersIds: List[ModifierId] = blocks.map(_.header.id) - (deliveryManager, cp1, cp2, cp3, cp4, cp5, cp6,cp7, cp8, cp9, blocks, headersIds) - } - - "Delivery Manager" should { - /** - * This test simulates DeliveryManager behaviour connected with updating nodes priority. - * - * Test expected behavior is: - * Send handshakedPeer to the Delivery Manager from cp1 for cp1. - * Send RequestFromLocal for N modifiers to the Delivery Manager. - * Delivery manager have to use requestModifier, send request to N modifiers to cp1 and put this N modifiers in expectedModifiersCollection. - * Receive less than 1/2 of this modifiers during 1 attempt. - * When period of updating priorities will expire, delivery manager will mark cp1 as BadNode. - * - */ - "mark peer as BadNode with BadPriority (1)" in { - val (deliveryManager, cp1, _, _, _, _, _, _, _, _, _, headersIds) = initialiseState - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic - assert(result.contains(cp1.socketAddress)) - assert(result(cp1.socketAddress) == BadNode) - deliveryManager.stop() - } - - /** - * This test simulates DeliveryManager behaviour connected with updating nodes priority - * - * Test expected behavior is: - * Send handshakedPeer to the Delivery Manager from cp1. - * Send RequestFromLocal for N modifiers to the Delivery Manager for cp1. - * Delivery manager have to use requestModifier, send request to N modifiers to cp1 and put this N modifiers in expectedModifiersCollection. - * Receive more than 3\4 of this modifiers during 1 attempt. - * When period of updating priorities will expire, delivery manager will mark cp1 as BestNode. - */ - "mark peer as HighPriorityNode with HighPriority (4)" in { - val (deliveryManager, cp1, _, _, _, _, _, _, _, _, blocks, headersIds) = initialiseState - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.map(block => block.header.id -> block.header.bytes).toMap), cp1) - val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic - - assert(result.contains(cp1.socketAddress)) - assert(result(cp1.socketAddress) == HighPriority) - deliveryManager.stop() - } - - /** - * This test simulates DeliveryManager behaviour connected with updating nodes priority - * - * Test expected behavior is: - * Send handshakedPeer to the Delivery Manager from cp1. - * Send RequestFromLocal for N modifiers to the Delivery Manager for cp1. - * Delivery manager have to use requestModifier, send request to N modifiers to cp1 and put this N modifiers in expectedModifiersCollection. - * Receive more than 1\2 and less than 3\4 of this modifiers during 1 attempt. - * When period of updating priorities will expire, delivery manager will mark cp1 as LowPriorityNode. - */ - "mark peer as LowPriorityNode with LowPriority (3)" in { - val (deliveryManager, cp1, _, _, _, _, _, _, _, _, blocks, headersIds) = initialiseState - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.take(6).map(block => block.header.id -> block.header.bytes).toMap), cp1) - val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic - assert(result.contains(cp1.socketAddress)) - assert(result(cp1.socketAddress) == LowPriority) - deliveryManager.stop() - } - - /** - * This test simulates DeliveryManager behavior connected with updating several nodes priority active in one time - * - * Test expected behavior is: - * Send handshakedPeer to the Delivery Manager from cp1, cp2, cp3, cp4, cp5, cp6. - * Send RequestFromLocal for N modifiers to the Delivery Manager for cp1, cp2, cp3, cp4, cp5, cp6. - * Delivery manager have to use requestModifier, send request to N modifiers to cp1 and put this N modifiers in expectedModifiersCollection. - * Receive more than 3\4 requested modifiers from cp1 and cp4. - * Receive less than 3\4 but more than 1\2 requested modifiers from cp2 and cp5. - * Receive less than 1\2 requested modifiers from cp3 and cp6. - * When period of updating priorities will expire, delivery manager will mark cp1 and cp4 as HighPriorityNode. - * When period of updating priorities will expire, delivery manager will mark cp2 and cp5 as LowPriorityNode. - * When period of updating priorities will expire, delivery manager will mark cp3 and cp6 as BadNode. - */ - "correctly choose peer priority while several peers are available" in { - val (deliveryManager, cp1, cp2, cp3, cp4, cp5, cp6, cp7, cp8, cp9, blocks, headersIds) = initialiseState - val updatedPeersCollection = - Map( - cp1.socketAddress -> (cp1, Older, InitialPriority), - cp1.socketAddress -> (cp2, Younger, InitialPriority), - cp1.socketAddress -> (cp3, Equal, InitialPriority), - cp1.socketAddress -> (cp4, Older, InitialPriority), - cp1.socketAddress -> (cp5, Younger, InitialPriority), - cp1.socketAddress -> (cp6, Equal, InitialPriority), - cp1.socketAddress -> (cp7, Older, InitialPriority), - cp1.socketAddress -> (cp8, Younger, InitialPriority), - cp1.socketAddress -> (cp9, Equal, InitialPriority) - ) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp4, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp5, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp6, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp7, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp8, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp9, Header.modifierTypeId, headersIds) - - val headerBytes = HeaderProtoSerializer.toProto(blocks.head.header).toByteArray - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.map(block => block.header.id -> headerBytes).toMap), cp1) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.map(block => block.header.id -> headerBytes).toMap), cp2) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.map(block => block.header.id -> headerBytes).toMap), cp3) - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.take(5).map(block => block.header.id -> headerBytes).toMap), cp4) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.take(5).map(block => block.header.id -> headerBytes).toMap), cp5) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.take(5).map(block => block.header.id -> headerBytes).toMap), cp6) - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.take(2).map(block => block.header.id -> headerBytes).toMap), cp7) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.take(2).map(block => block.header.id -> headerBytes).toMap), cp8) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.take(2).map(block => block.header.id -> headerBytes).toMap), cp9) - - val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic - - assert(result.contains(cp1.socketAddress)) - assert(result(cp1.socketAddress) == HighPriority) - - //todo fix spam after it fix test -// assert(result.contains(cp2.socketAddress)) -// assert(result(cp2.socketAddress) == HighPriority()) - -// assert(result.contains(cp3.socketAddress)) -// assert(result(cp3.socketAddress) == HighPriority()) - -// assert(result.contains(cp4.socketAddress)) -// assert(result(cp4.socketAddress) == LowPriority()) +//class DeliveryManagerPriorityTests extends WordSpecLike +// with BeforeAndAfterAll +// with Matchers +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { // -// assert(result.contains(cp5.socketAddress)) -// assert(result(cp5.socketAddress) == LowPriority()) +// implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") // -// assert(result.contains(cp6.socketAddress)) -// assert(result(cp6.socketAddress) == LowPriority()) +// override def afterAll: Unit = TestKit.shutdownActorSystem(system) // -// assert(result.contains(cp7.socketAddress)) -// assert(result(cp7.socketAddress) == BadNode()) +// def initialiseState: (TestActorRef[DeliveryManager], ConnectedPeer, ConnectedPeer, ConnectedPeer, +// ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, +// List[Block], List[ModifierId]) = { +// val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = true, isMining = true, testNetSettings) +// val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) +// val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) +// val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) +// val (_: InetSocketAddress, cp4: ConnectedPeer) = createPeer(9004, "172.16.13.13", testNetSettings) +// val (_: InetSocketAddress, cp5: ConnectedPeer) = createPeer(9005, "172.16.13.14", testNetSettings) +// val (_: InetSocketAddress, cp6: ConnectedPeer) = createPeer(9006, "172.16.13.15", testNetSettings) +// val (_: InetSocketAddress, cp7: ConnectedPeer) = createPeer(9007, "172.16.13.16", testNetSettings) +// val (_: InetSocketAddress, cp8: ConnectedPeer) = createPeer(9008, "172.16.13.17", testNetSettings) +// val (_: InetSocketAddress, cp9: ConnectedPeer) = createPeer(9009, "172.16.13.18", testNetSettings) +// val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 +// val headersIds: List[ModifierId] = blocks.map(_.header.id) +// (deliveryManager, cp1, cp2, cp3, cp4, cp5, cp6,cp7, cp8, cp9, blocks, headersIds) +// } // -// assert(result.contains(cp8.socketAddress)) -// assert(result(cp8.socketAddress) == BadNode()) +// "Delivery Manager" should { +// /** +// * This test simulates DeliveryManager behaviour connected with updating nodes priority. +// * +// * Test expected behavior is: +// * Send handshakedPeer to the Delivery Manager from cp1 for cp1. +// * Send RequestFromLocal for N modifiers to the Delivery Manager. +// * Delivery manager have to use requestModifier, send request to N modifiers to cp1 and put this N modifiers in expectedModifiersCollection. +// * Receive less than 1/2 of this modifiers during 1 attempt. +// * When period of updating priorities will expire, delivery manager will mark cp1 as BadNode. +// * +// */ +// "mark peer as BadNode with BadPriority (1)" in { +// val (deliveryManager, cp1, _, _, _, _, _, _, _, _, _, headersIds) = initialiseState +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic +// assert(result.contains(cp1.socketAddress)) +// assert(result(cp1.socketAddress) == BadNode) +// deliveryManager.stop() +// } // -// assert(result.contains(cp9.socketAddress)) -// assert(result(cp9.socketAddress) == BadNode()) - - deliveryManager.stop() - } - - /** - * This test simulates DeliveryManager behavior connected with updating node priority while receiving spam modifiers - * - * Test expected behavior is: - * Send handshakedPeer to the Delivery Manager from cp1. - * Receive unexpected modifiers from cp1. - * cp1 priority must stay as InitialPriority. - */ - "not increment modifiers which will be putted in spam collection" in { - val (deliveryManager, cp1, _, _, _, _, _, _, _, _, blocks, _) = initialiseState - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId, blocks.map(block => block.header.id -> block.header.bytes).toMap), cp1) - val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic - assert(result.contains(cp1.socketAddress)) - assert(result(cp1.socketAddress) == BadNode) - deliveryManager.stop() - } - } -} \ No newline at end of file +// /** +// * This test simulates DeliveryManager behaviour connected with updating nodes priority +// * +// * Test expected behavior is: +// * Send handshakedPeer to the Delivery Manager from cp1. +// * Send RequestFromLocal for N modifiers to the Delivery Manager for cp1. +// * Delivery manager have to use requestModifier, send request to N modifiers to cp1 and put this N modifiers in expectedModifiersCollection. +// * Receive more than 3\4 of this modifiers during 1 attempt. +// * When period of updating priorities will expire, delivery manager will mark cp1 as BestNode. +// */ +// "mark peer as HighPriorityNode with HighPriority (4)" in { +// val (deliveryManager, cp1, _, _, _, _, _, _, _, _, blocks, headersIds) = initialiseState +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.map(block => block.header.id -> block.header.bytes).toMap), cp1) +// val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic +// +// assert(result.contains(cp1.socketAddress)) +// assert(result(cp1.socketAddress) == HighPriority) +// deliveryManager.stop() +// } +// +// /** +// * This test simulates DeliveryManager behaviour connected with updating nodes priority +// * +// * Test expected behavior is: +// * Send handshakedPeer to the Delivery Manager from cp1. +// * Send RequestFromLocal for N modifiers to the Delivery Manager for cp1. +// * Delivery manager have to use requestModifier, send request to N modifiers to cp1 and put this N modifiers in expectedModifiersCollection. +// * Receive more than 1\2 and less than 3\4 of this modifiers during 1 attempt. +// * When period of updating priorities will expire, delivery manager will mark cp1 as LowPriorityNode. +// */ +// "mark peer as LowPriorityNode with LowPriority (3)" in { +// val (deliveryManager, cp1, _, _, _, _, _, _, _, _, blocks, headersIds) = initialiseState +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.take(6).map(block => block.header.id -> block.header.bytes).toMap), cp1) +// val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic +// assert(result.contains(cp1.socketAddress)) +// assert(result(cp1.socketAddress) == LowPriority) +// deliveryManager.stop() +// } +// +// /** +// * This test simulates DeliveryManager behavior connected with updating several nodes priority active in one time +// * +// * Test expected behavior is: +// * Send handshakedPeer to the Delivery Manager from cp1, cp2, cp3, cp4, cp5, cp6. +// * Send RequestFromLocal for N modifiers to the Delivery Manager for cp1, cp2, cp3, cp4, cp5, cp6. +// * Delivery manager have to use requestModifier, send request to N modifiers to cp1 and put this N modifiers in expectedModifiersCollection. +// * Receive more than 3\4 requested modifiers from cp1 and cp4. +// * Receive less than 3\4 but more than 1\2 requested modifiers from cp2 and cp5. +// * Receive less than 1\2 requested modifiers from cp3 and cp6. +// * When period of updating priorities will expire, delivery manager will mark cp1 and cp4 as HighPriorityNode. +// * When period of updating priorities will expire, delivery manager will mark cp2 and cp5 as LowPriorityNode. +// * When period of updating priorities will expire, delivery manager will mark cp3 and cp6 as BadNode. +// */ +// "correctly choose peer priority while several peers are available" in { +// val (deliveryManager, cp1, cp2, cp3, cp4, cp5, cp6, cp7, cp8, cp9, blocks, headersIds) = initialiseState +// val updatedPeersCollection = +// Map( +// cp1.socketAddress -> (cp1, Older, InitialPriority), +// cp1.socketAddress -> (cp2, Younger, InitialPriority), +// cp1.socketAddress -> (cp3, Equal, InitialPriority), +// cp1.socketAddress -> (cp4, Older, InitialPriority), +// cp1.socketAddress -> (cp5, Younger, InitialPriority), +// cp1.socketAddress -> (cp6, Equal, InitialPriority), +// cp1.socketAddress -> (cp7, Older, InitialPriority), +// cp1.socketAddress -> (cp8, Younger, InitialPriority), +// cp1.socketAddress -> (cp9, Equal, InitialPriority) +// ) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp4, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp5, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp6, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp7, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp8, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp9, Header.modifierTypeId, headersIds) +// +// val headerBytes = HeaderProtoSerializer.toProto(blocks.head.header).toByteArray +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.map(block => block.header.id -> headerBytes).toMap), cp1) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.map(block => block.header.id -> headerBytes).toMap), cp2) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.map(block => block.header.id -> headerBytes).toMap), cp3) +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.take(5).map(block => block.header.id -> headerBytes).toMap), cp4) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.take(5).map(block => block.header.id -> headerBytes).toMap), cp5) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.take(5).map(block => block.header.id -> headerBytes).toMap), cp6) +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.take(2).map(block => block.header.id -> headerBytes).toMap), cp7) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.take(2).map(block => block.header.id -> headerBytes).toMap), cp8) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.take(2).map(block => block.header.id -> headerBytes).toMap), cp9) +// +// val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic +// +// assert(result.contains(cp1.socketAddress)) +// assert(result(cp1.socketAddress) == HighPriority) +// +// //todo fix spam after it fix test +//// assert(result.contains(cp2.socketAddress)) +//// assert(result(cp2.socketAddress) == HighPriority()) +// +//// assert(result.contains(cp3.socketAddress)) +//// assert(result(cp3.socketAddress) == HighPriority()) +// +//// assert(result.contains(cp4.socketAddress)) +//// assert(result(cp4.socketAddress) == LowPriority()) +//// +//// assert(result.contains(cp5.socketAddress)) +//// assert(result(cp5.socketAddress) == LowPriority()) +//// +//// assert(result.contains(cp6.socketAddress)) +//// assert(result(cp6.socketAddress) == LowPriority()) +//// +//// assert(result.contains(cp7.socketAddress)) +//// assert(result(cp7.socketAddress) == BadNode()) +//// +//// assert(result.contains(cp8.socketAddress)) +//// assert(result(cp8.socketAddress) == BadNode()) +//// +//// assert(result.contains(cp9.socketAddress)) +//// assert(result(cp9.socketAddress) == BadNode()) +// +// deliveryManager.stop() +// } +// +// /** +// * This test simulates DeliveryManager behavior connected with updating node priority while receiving spam modifiers +// * +// * Test expected behavior is: +// * Send handshakedPeer to the Delivery Manager from cp1. +// * Receive unexpected modifiers from cp1. +// * cp1 priority must stay as InitialPriority. +// */ +// "not increment modifiers which will be putted in spam collection" in { +// val (deliveryManager, cp1, _, _, _, _, _, _, _, _, blocks, _) = initialiseState +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId, blocks.map(block => block.header.id -> block.header.bytes).toMap), cp1) +// val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic +// assert(result.contains(cp1.socketAddress)) +// assert(result(cp1.socketAddress) == BadNode) +// deliveryManager.stop() +// } +// } +//} \ No newline at end of file diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala index 97acbe4976..106005b9ff 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala @@ -1,187 +1,187 @@ -package encry.network.DeliveryManagerTests - -import java.net.InetSocketAddress -import akka.actor.ActorSystem -import akka.testkit.{TestActorRef, TestProbe} -import encry.consensus.HistoryConsensus -import encry.consensus.HistoryConsensus.Older -import encry.modifiers.InstanceFactory -import encry.network.DeliveryManager -import encry.network.DeliveryManagerTests.DMUtils._ -import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.NodeViewSynchronizer.ReceivableMessages._ -import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} -import encry.network.PeersKeeper.UpdatedPeersCollection -import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus.InitialPriority -import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus -import encry.settings.TestNetSettings -import encry.view.history.History -import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer} -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage} -import org.encryfoundation.common.utils.TaggedTypes.ModifierId -import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -import scala.concurrent.duration._ -import scala.collection.mutable.WrappedArray - -class DeliveryManagerReRequestModifiesSpec extends WordSpecLike - with BeforeAndAfterAll - with Matchers - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") - - override def afterAll(): Unit = system.terminate() - - def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], - ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte], History) = { - val (deliveryManager, history) = - initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) - val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) - val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) - val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) - val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 - val headersIds: List[ModifierId] = blocks.map(_.header.id) - val headersAsKey = headersIds.map(toKey) - (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey, history) - } - - "ReRequestModifies" should { - "re-ask necessary modifier several times (number of attempts from testNetSettings) and remove modifier from " + - "expectedModifiers collection after all attempts will expire" in { - val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val header: ModifierId = headersIds.head - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header)) - handler1.expectMsgAllOf( - testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)) - ) - //this thread sleep is using for expecting modifier removal - Thread.sleep(6000) - - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty).isEmpty) - deliveryManager.stop() - } - "not re-ask unnecessary modifiers" in { - val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val header: ModifierId = headersIds.head - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header)) - - //await one re-ask - handler1.expectMsgAllOf( - testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)) - ) - - val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, - Map(header -> headerBytes)), cp1) - deliveryManager.stop() - } - "not re-ask modifiers which were applied to the history" in { - val (deliveryManager, _, _, _, blocks, headerIds, _, history) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) - - handler1.expectMsg(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(headerIds.head))) - - val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, - Map(headerIds.head -> headerBytes)), cp1) - - history.append(blocks.head.header) - val uHistory: History = history.reportModifierIsValid(blocks.head.header) - - deliveryManager ! UpdatedHistory(uHistory) - - deliveryManager ! SemanticallySuccessfulModifier(blocks.head.header) - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) - - assert(deliveryManager.underlyingActor.expectedModifiers - .getOrElse(cp1.socketAddress, Map.empty).isEmpty) - deliveryManager.stop() - } - "remove peer from expectedModifiers if expected modifiers collection from this peer is empty" in { - val (deliveryManager, cp1, _, _, _, headerIds, _, _) = initialiseState() - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) - //this thread sleep is using for expecting modifier removal - Thread.sleep((testNetSettings.network.maxDeliveryChecks * testNetSettings.network.deliveryTimeout._1) * 1000) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty).isEmpty) - assert(deliveryManager.underlyingActor.expectedModifiers - .getOrElse(cp1.socketAddress, Map.empty) == Map.empty) - deliveryManager.stop() - } - "not re-ask transactions" in { - val (deliveryManager, _, _, _, _, _, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val transactions: Seq[ModifierId] = genValidPaymentTxs(1).map(_.id) - - deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, transactions) - - handler1.expectMsgAllOf( - RequestModifiersNetworkMessage(Transaction.modifierTypeId -> transactions) - ) - handler1.expectNoMsg(10.seconds) - assert(deliveryManager.underlyingActor.expectedModifiers - .getOrElse(cp1.socketAddress, Map.empty) == Map.empty) - deliveryManager.stop() - } - } -} \ No newline at end of file +//package encry.network.DeliveryManagerTests +// +//import java.net.InetSocketAddress +//import akka.actor.ActorSystem +//import akka.testkit.{TestActorRef, TestProbe} +//import encry.consensus.HistoryConsensus +//import encry.consensus.HistoryConsensus.Older +//import encry.modifiers.InstanceFactory +//import encry.network.DeliveryManager +////import encry.network.DeliveryManagerTests.DMUtils._ +//import encry.network.NetworkController.ReceivableMessages.DataFromPeer +//import encry.network.NodeViewSynchronizer.ReceivableMessages._ +//import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} +//import encry.network.PeersKeeper.UpdatedPeersCollection +//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus.InitialPriority +//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus +//import encry.settings.TestNetSettings +//import encry.view.history.History +//import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer} +//import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +//import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage} +//import org.encryfoundation.common.utils.TaggedTypes.ModifierId +//import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +//import scala.concurrent.duration._ +//import scala.collection.mutable.WrappedArray +// +//class DeliveryManagerReRequestModifiesSpec extends WordSpecLike +// with BeforeAndAfterAll +// with Matchers +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// +// implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") +// +// override def afterAll(): Unit = system.terminate() +// +// def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], +// ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte], History) = { +// val (deliveryManager, history) = +// initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) +// val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) +// val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) +// val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) +// val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 +// val headersIds: List[ModifierId] = blocks.map(_.header.id) +// val headersAsKey = headersIds.map(toKey) +// (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey, history) +// } +// +// "ReRequestModifies" should { +// "re-ask necessary modifier several times (number of attempts from testNetSettings) and remove modifier from " + +// "expectedModifiers collection after all attempts will expire" in { +// val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val header: ModifierId = headersIds.head +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header)) +// handler1.expectMsgAllOf( +// testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)) +// ) +// //this thread sleep is using for expecting modifier removal +// Thread.sleep(6000) +// +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty).isEmpty) +// deliveryManager.stop() +// } +// "not re-ask unnecessary modifiers" in { +// val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val header: ModifierId = headersIds.head +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header)) +// +// //await one re-ask +// handler1.expectMsgAllOf( +// testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)) +// ) +// +// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, +// Map(header -> headerBytes)), cp1) +// deliveryManager.stop() +// } +// "not re-ask modifiers which were applied to the history" in { +// val (deliveryManager, _, _, _, blocks, headerIds, _, history) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) +// +// handler1.expectMsg(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(headerIds.head))) +// +// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, +// Map(headerIds.head -> headerBytes)), cp1) +// +// history.append(blocks.head.header) +// val uHistory: History = history.reportModifierIsValid(blocks.head.header) +// +// deliveryManager ! UpdatedHistory(uHistory) +// +// deliveryManager ! SemanticallySuccessfulModifier(blocks.head.header) +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) +// +// assert(deliveryManager.underlyingActor.expectedModifiers +// .getOrElse(cp1.socketAddress, Map.empty).isEmpty) +// deliveryManager.stop() +// } +// "remove peer from expectedModifiers if expected modifiers collection from this peer is empty" in { +// val (deliveryManager, cp1, _, _, _, headerIds, _, _) = initialiseState() +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) +// //this thread sleep is using for expecting modifier removal +// Thread.sleep((testNetSettings.network.maxDeliveryChecks * testNetSettings.network.deliveryTimeout._1) * 1000) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty).isEmpty) +// assert(deliveryManager.underlyingActor.expectedModifiers +// .getOrElse(cp1.socketAddress, Map.empty) == Map.empty) +// deliveryManager.stop() +// } +// "not re-ask transactions" in { +// val (deliveryManager, _, _, _, _, _, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val transactions: Seq[ModifierId] = genValidPaymentTxs(1).map(_.id) +// +// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, transactions) +// +// handler1.expectMsgAllOf( +// RequestModifiersNetworkMessage(Transaction.modifierTypeId -> transactions) +// ) +// handler1.expectNoMsg(10.seconds) +// assert(deliveryManager.underlyingActor.expectedModifiers +// .getOrElse(cp1.socketAddress, Map.empty) == Map.empty) +// deliveryManager.stop() +// } +// } +//} \ No newline at end of file diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala index 190f11c8d4..0ab92b2abf 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala @@ -1,320 +1,320 @@ package encry.network.DeliveryManagerTests import java.net.InetSocketAddress - -import akka.actor.ActorSystem -import akka.testkit.{TestActorRef, TestKit, TestProbe} -import encry.consensus.HistoryConsensus -import encry.consensus.HistoryConsensus.{Fork, Older, Younger} -import encry.modifiers.InstanceFactory -import encry.network.DeliveryManager -import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.NodeViewSynchronizer.ReceivableMessages.RequestFromLocal -import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} -import encry.settings.TestNetSettings -import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -import encry.network.DeliveryManagerTests.DMUtils._ -import encry.network.PeersKeeper.UpdatedPeersCollection -import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus -import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ -import encry.view.NodeViewHolder.DownloadRequest -import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} -import org.encryfoundation.common.network.SyncInfo -import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.ModifierId - -import scala.collection.mutable.WrappedArray - -class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll - with Matchers - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") - - override def afterAll(): Unit = TestKit.shutdownActorSystem(system) - - def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], - ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { - val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) - val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) - val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) - val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) - val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 - val headersIds: List[ModifierId] = blocks.map(_.header.id) - val headersAsKey = headersIds.map(toKey) - (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) - } - - "RequestModifies" should { - "handle uniq modifiers from RequestFromLocal message correctly" in { - val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.size == headersIds.size) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.forall(elem => headersAsKey.contains(elem))) - deliveryManager.stop() - } - "not handle repeating modifiers from RequestFromLocal message" in { - val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.size == headersIds.size) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.forall(elem => headersAsKey.contains(elem))) - deliveryManager.stop() - } - "Delivery Manager should handle received modifier which were requested correctly" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.isEmpty) - assert(deliveryManager.underlyingActor.receivedModifiers.size == blocks.size) - assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) - deliveryManager.stop() - } - "Delivery manager should not handle repeating modifiers" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) - assert(deliveryManager.underlyingActor.receivedModifiers.size == headersIds.size) - assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) - deliveryManager.stop() - } - "handle priority request for payload correctly" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId -> blocks.map(k => k.header.id -> Array.emptyByteArray).toMap), cp1) - headersIds.foreach(id => - deliveryManager ! DownloadRequest(Payload.modifierTypeId, blocks.find(block => - block.id.sameElements(id)).get.payload.id)) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .size == blocks.size) - deliveryManager.stop() - } - "choose correct peer in priority request" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val address2 = new InetSocketAddress("123.123.123.124", 9001) - val handler2: TestProbe = TestProbe() - val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.124", Some(address2), System.currentTimeMillis())) - - val address3 = new InetSocketAddress("123.123.123.125", 9001) - val handler3: TestProbe = TestProbe() - val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.125", Some(address3), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map( - address1 -> (cp1, Older, InitialPriority), - address2 -> (cp2, Older, InitialPriority), - address3 -> (cp3, Older, InitialPriority) - ) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val header: Header = blocks.head.header - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) - deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) - deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3) - - deliveryManager ! DownloadRequest(Payload.modifierTypeId, header.payloadId) - - handler1.expectMsgAnyOf( - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), - RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), - SyncInfoNetworkMessage(SyncInfo(List())) - ) - - handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - deliveryManager.stop() - } - "not ask modifiers while block chain is not synced from Younger nodes" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) - - val address2 = new InetSocketAddress("123.123.123.124", 9001) - val handler2: TestProbe = TestProbe() - val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.124", Some(address2), System.currentTimeMillis())) - - val address3 = new InetSocketAddress("123.123.123.125", 9001) - val handler3: TestProbe = TestProbe() - val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.125", Some(address3), System.currentTimeMillis())) - - val updatedPeersCollection = - Map( - address2 -> (cp2, Younger, InitialPriority), - address3 -> (cp3, Fork, InitialPriority) - ) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - - val header: Header = blocks.head.header - - deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) - deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) - - handler2.expectNoMsg() - handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - deliveryManager.stop() - } - "not ask modifiers from peer which is not contained in status tracker" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val address2 = new InetSocketAddress("123.123.123.124", 9001) - val handler2: TestProbe = TestProbe() - val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.124", Some(address2), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address2 -> (cp2, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val header: Header = blocks.head.header - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) - deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) - - handler1.expectNoMsg() - handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - deliveryManager.stop() - } - "not ask transactions while block chain is not synced" in { - val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) - val txs: Seq[Transaction] = genInvalidPaymentTxs(1) - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) - - handler1.expectNoMsg() - deliveryManager.stop() - } - "not ask transaction while node is not mining" in { - val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) - val txs: Seq[Transaction] = genInvalidPaymentTxs(1) - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) - - handler1.expectNoMsg() - deliveryManager.stop() - } - "not re-ask modifiers which already have been received" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val header: Header = blocks.head.header - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) - - val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(header).toByteArray - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> headerBytes)), cp1) - - handler1.expectMsgAllOf( - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)) - ) - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) - - handler1.expectNoMsg() - - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.isEmpty) - assert(deliveryManager.underlyingActor.receivedModifiers.size == 1) - assert(deliveryManager.underlyingActor.receivedModifiers.contains(toKey(header.id))) - deliveryManager.stop() - } - } -} \ No newline at end of file +// +//import akka.actor.ActorSystem +//import akka.testkit.{TestActorRef, TestKit, TestProbe} +//import encry.consensus.HistoryConsensus +//import encry.consensus.HistoryConsensus.{Fork, Older, Younger} +//import encry.modifiers.InstanceFactory +//import encry.network.DeliveryManager +//import encry.network.NetworkController.ReceivableMessages.DataFromPeer +//import encry.network.NodeViewSynchronizer.ReceivableMessages.RequestFromLocal +//import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} +//import encry.settings.TestNetSettings +//import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +//import encry.network.DeliveryManagerTests.DMUtils._ +//import encry.network.PeersKeeper.UpdatedPeersCollection +//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus +//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ +//import encry.view.NodeViewHolder.DownloadRequest +//import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} +//import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +//import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} +//import org.encryfoundation.common.network.SyncInfo +//import org.encryfoundation.common.utils.Algos +//import org.encryfoundation.common.utils.TaggedTypes.ModifierId +// +//import scala.collection.mutable.WrappedArray + +//class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll +// with Matchers +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// +// implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") +// +// override def afterAll(): Unit = TestKit.shutdownActorSystem(system) +// +// def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], +// ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { +// val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) +// val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) +// val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) +// val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) +// val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 +// val headersIds: List[ModifierId] = blocks.map(_.header.id) +// val headersAsKey = headersIds.map(toKey) +// (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) +// } +// +// "RequestModifies" should { +// "handle uniq modifiers from RequestFromLocal message correctly" in { +// val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.size == headersIds.size) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.forall(elem => headersAsKey.contains(elem))) +// deliveryManager.stop() +// } +// "not handle repeating modifiers from RequestFromLocal message" in { +// val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.size == headersIds.size) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.forall(elem => headersAsKey.contains(elem))) +// deliveryManager.stop() +// } +// "Delivery Manager should handle received modifier which were requested correctly" in { +// val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.isEmpty) +// assert(deliveryManager.underlyingActor.receivedModifiers.size == blocks.size) +// assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) +// deliveryManager.stop() +// } +// "Delivery manager should not handle repeating modifiers" in { +// val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) +// assert(deliveryManager.underlyingActor.receivedModifiers.size == headersIds.size) +// assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) +// deliveryManager.stop() +// } +// "handle priority request for payload correctly" in { +// val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId -> blocks.map(k => k.header.id -> Array.emptyByteArray).toMap), cp1) +// headersIds.foreach(id => +// deliveryManager ! DownloadRequest(Payload.modifierTypeId, blocks.find(block => +// block.id.sameElements(id)).get.payload.id)) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .size == blocks.size) +// deliveryManager.stop() +// } +// "choose correct peer in priority request" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val address2 = new InetSocketAddress("123.123.123.124", 9001) +// val handler2: TestProbe = TestProbe() +// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.124", Some(address2), System.currentTimeMillis())) +// +// val address3 = new InetSocketAddress("123.123.123.125", 9001) +// val handler3: TestProbe = TestProbe() +// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.125", Some(address3), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map( +// address1 -> (cp1, Older, InitialPriority), +// address2 -> (cp2, Older, InitialPriority), +// address3 -> (cp3, Older, InitialPriority) +// ) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) +// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) +// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3) +// +// deliveryManager ! DownloadRequest(Payload.modifierTypeId, header.payloadId) +// +// handler1.expectMsgAnyOf( +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), +// RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), +// SyncInfoNetworkMessage(SyncInfo(List())) +// ) +// +// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// deliveryManager.stop() +// } +// "not ask modifiers while block chain is not synced from Younger nodes" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) +// +// val address2 = new InetSocketAddress("123.123.123.124", 9001) +// val handler2: TestProbe = TestProbe() +// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.124", Some(address2), System.currentTimeMillis())) +// +// val address3 = new InetSocketAddress("123.123.123.125", 9001) +// val handler3: TestProbe = TestProbe() +// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.125", Some(address3), System.currentTimeMillis())) +// +// val updatedPeersCollection = +// Map( +// address2 -> (cp2, Younger, InitialPriority), +// address3 -> (cp3, Fork, InitialPriority) +// ) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) +// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) +// +// handler2.expectNoMsg() +// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// deliveryManager.stop() +// } +// "not ask modifiers from peer which is not contained in status tracker" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val address2 = new InetSocketAddress("123.123.123.124", 9001) +// val handler2: TestProbe = TestProbe() +// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.124", Some(address2), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address2 -> (cp2, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) +// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) +// +// handler1.expectNoMsg() +// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// deliveryManager.stop() +// } +// "not ask transactions while block chain is not synced" in { +// val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) +// val txs: Seq[Transaction] = genInvalidPaymentTxs(1) +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) +// +// handler1.expectNoMsg() +// deliveryManager.stop() +// } +// "not ask transaction while node is not mining" in { +// val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) +// val txs: Seq[Transaction] = genInvalidPaymentTxs(1) +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) +// +// handler1.expectNoMsg() +// deliveryManager.stop() +// } +// "not re-ask modifiers which already have been received" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) +// +// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(header).toByteArray +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> headerBytes)), cp1) +// +// handler1.expectMsgAllOf( +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)) +// ) +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) +// +// handler1.expectNoMsg() +// +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.isEmpty) +// assert(deliveryManager.underlyingActor.receivedModifiers.size == 1) +// assert(deliveryManager.underlyingActor.receivedModifiers.contains(toKey(header.id))) +// deliveryManager.stop() +// } +// } +//} \ No newline at end of file diff --git a/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala b/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala index d432fad199..2b8ffd799c 100644 --- a/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala +++ b/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala @@ -11,7 +11,7 @@ import encry.network.NodeViewSynchronizer.ReceivableMessages.{ ChangedHistory, U import encry.network.PeerConnectionHandler.{ ConnectedPeer, Outgoing } import encry.network.PeersKeeper.BanPeer import encry.settings.TestNetSettings -import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote +//import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote import encry.view.history.History import org.encryfoundation.common.crypto.equihash.EquihashSolution import org.encryfoundation.common.modifiers.history.{ @@ -27,205 +27,205 @@ import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpec import scorex.crypto.hash.Digest32 import scorex.utils.Random -class DownloadedModifiersValidatorTests - extends WordSpecLike - with Matchers - with BeforeAndAfterAll - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - implicit val system: ActorSystem = ActorSystem() - - override def afterAll(): Unit = system.terminate() - - "DownloadedModifiersValidatorTests" should { - "find too old header by height" in { - val nodeViewHolder = TestProbe() - val peersKeeper = TestProbe() - val nodeViewSync = TestProbe() - val mempool = TestProbe() - - val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( - DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, - nodeViewHolder.ref, - peersKeeper.ref, - nodeViewSync.ref, - mempool.ref, - None, - settings) - ) - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) - val peerHandler: TestProbe = TestProbe() - val connectedPeer: ConnectedPeer = ConnectedPeer( - address, - peerHandler.ref, - Outgoing, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test node", - Some(address), - System.currentTimeMillis()) - ) - - val (history, _) = generateBlocks(200, generateDummyHistory(testNetSettings)) - downloadedModifiersValidator ! UpdatedHistory(history) - val invalidHeader = generateGenesisBlock(Height @@ 1) - - val mods: Map[ModifierId, Array[Byte]] = List(invalidHeader) - .map( - b => b.header.id -> HeaderProtoSerializer.toProto(b.header).toByteArray - ) - .toMap - import scala.concurrent.duration._ - downloadedModifiersValidator ! ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods) - peersKeeper.expectMsgPF(10.seconds) { - case BanPeer(connected, _) => connected == connectedPeer - } - val validHeightHeader = generateGenesisBlock(Height @@ 200) - - val mods1: Map[ModifierId, Array[Byte]] = List(validHeightHeader) - .map( - b => b.header.id -> HeaderProtoSerializer.toProto(b.header).toByteArray - ) - .toMap - downloadedModifiersValidator ! ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods1) - nodeViewHolder.expectMsgPF(10.seconds) { - case ModifierFromRemote(mod) => mod == validHeightHeader.header - } - } - "find corrupted header" in { - val nodeViewHolder = TestProbe() - val peersKeeper = TestProbe() - val deliveryManager = TestProbe() - val nodeViewSync = TestProbe() - val mempool = TestProbe() - - val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( - DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, - nodeViewHolder.ref, - peersKeeper.ref, - nodeViewSync.ref, - mempool.ref, - None, - settings) - ) - val history: History = generateDummyHistory(testNetSettings) - - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) - val peerHandler: TestProbe = TestProbe() - val connectedPeer: ConnectedPeer = ConnectedPeer( - address, - peerHandler.ref, - Outgoing, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test node", - Some(address), - System.currentTimeMillis()) - ) - - val timestamp1 = System.currentTimeMillis() - Thread.sleep(1000) - val timestamp2 = System.currentTimeMillis() - - val header_first: Header = Header( - 1.toByte, - ModifierId @@ Random.randomBytes(), - Digest32 @@ Random.randomBytes(), - timestamp2, - 2, - scala.util.Random.nextLong(), - testNetSettings.constants.InitialDifficulty, - EquihashSolution(Seq(1, 3)), - Random.randomBytes() - ) - val header_second: Header = Header( - 1.toByte, - header_first.id, - Digest32 @@ Random.randomBytes(), - timestamp1, - 1, - scala.util.Random.nextLong(), - testNetSettings.constants.InitialDifficulty, - EquihashSolution(Seq(1, 3)), - Random.randomBytes() - ) - - history.append(header_first) - - nodeViewSync.send(downloadedModifiersValidator, UpdatedHistory(history)) - - /* Header */ - val mods = Seq(header_second).map(x => x.id -> HeaderProtoSerializer.toProto(x).toByteArray.reverse).toMap - val msg = ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods) - - deliveryManager.send(downloadedModifiersValidator, msg) - peersKeeper.expectMsg(BanPeer(connectedPeer, CorruptedSerializedBytes)) - nodeViewHolder.expectNoMsg() - nodeViewSync.expectMsg(InvalidModifier(header_second.id)) - } - "find corrupted payload" in { - val nodeViewHolder = TestProbe() - val peersKeeper = TestProbe() - val deliveryManager = TestProbe() - val nodeViewSync = TestProbe() - val mempool = TestProbe() - - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) - val peerHandler: TestProbe = TestProbe() - val connectedPeer: ConnectedPeer = ConnectedPeer( - address, - peerHandler.ref, - Outgoing, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "test node", - Some(address), - System.currentTimeMillis()) - ) - - val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( - DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, - nodeViewHolder.ref, - peersKeeper.ref, - nodeViewSync.ref, - mempool.ref, - None, - settings) - ) - val history: History = generateDummyHistory(testNetSettings) - - val historyWith10Blocks = (0 until 10).foldLeft(history, Seq.empty[Block]) { - case ((prevHistory, blocks), _) => - val block: Block = generateNextBlock(prevHistory) - prevHistory.append(block.header) - prevHistory.append(block.payload) - (prevHistory.reportModifierIsValid(block), blocks :+ block) - } - - val payload = Payload(ModifierId @@ scorex.utils.Random.randomBytes(), Seq(coinbaseTransaction)) - - nodeViewSync.send(downloadedModifiersValidator, UpdatedHistory(historyWith10Blocks._1)) - - val bytes = PayloadProtoSerializer.toProto(payload).toByteArray - - val mods: Map[ModifierId, Array[Byte]] = (historyWith10Blocks._2.map( - b => b.payload.id -> PayloadProtoSerializer.toProto(b.payload).toByteArray.reverse - ) :+ (payload.id -> bytes)).toMap - - deliveryManager - .send(downloadedModifiersValidator, ModifiersForValidating(connectedPeer, Payload.modifierTypeId, mods)) - - peersKeeper.expectMsg(BanPeer(connectedPeer, CorruptedSerializedBytes)) - nodeViewHolder.expectMsg(ModifierFromRemote(payload)) - } - } - - def generateBlocks(qty: Int, history: History): (History, List[Block]) = - (0 until qty).foldLeft(history, List.empty[Block]) { - case ((prevHistory, blocks), _) => - val block: Block = generateNextBlock(prevHistory) - prevHistory.append(block.header) - prevHistory.append(block.payload) - val a = prevHistory.reportModifierIsValid(block) - (a, blocks :+ block) - } -} +//class DownloadedModifiersValidatorTests +// extends WordSpecLike +// with Matchers +// with BeforeAndAfterAll +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// +// implicit val system: ActorSystem = ActorSystem() +// +// override def afterAll(): Unit = system.terminate() +// +// "DownloadedModifiersValidatorTests" should { +// "find too old header by height" in { +// val nodeViewHolder = TestProbe() +// val peersKeeper = TestProbe() +// val nodeViewSync = TestProbe() +// val mempool = TestProbe() +// +// val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( +// DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, +// nodeViewHolder.ref, +// peersKeeper.ref, +// nodeViewSync.ref, +// mempool.ref, +// None, +// settings) +// ) +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) +// val peerHandler: TestProbe = TestProbe() +// val connectedPeer: ConnectedPeer = ConnectedPeer( +// address, +// peerHandler.ref, +// Outgoing, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test node", +// Some(address), +// System.currentTimeMillis()) +// ) +// +// val (history, _) = generateBlocks(200, generateDummyHistory(testNetSettings)) +// downloadedModifiersValidator ! UpdatedHistory(history) +// val invalidHeader = generateGenesisBlock(Height @@ 1) +// +// val mods: Map[ModifierId, Array[Byte]] = List(invalidHeader) +// .map( +// b => b.header.id -> HeaderProtoSerializer.toProto(b.header).toByteArray +// ) +// .toMap +// import scala.concurrent.duration._ +// downloadedModifiersValidator ! ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods) +// peersKeeper.expectMsgPF(10.seconds) { +// case BanPeer(connected, _) => connected == connectedPeer +// } +// val validHeightHeader = generateGenesisBlock(Height @@ 200) +// +// val mods1: Map[ModifierId, Array[Byte]] = List(validHeightHeader) +// .map( +// b => b.header.id -> HeaderProtoSerializer.toProto(b.header).toByteArray +// ) +// .toMap +// downloadedModifiersValidator ! ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods1) +// nodeViewHolder.expectMsgPF(10.seconds) { +// case ModifierFromRemote(mod) => mod == validHeightHeader.header +// } +// } +// "find corrupted header" in { +// val nodeViewHolder = TestProbe() +// val peersKeeper = TestProbe() +// val deliveryManager = TestProbe() +// val nodeViewSync = TestProbe() +// val mempool = TestProbe() +// +// val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( +// DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, +// nodeViewHolder.ref, +// peersKeeper.ref, +// nodeViewSync.ref, +// mempool.ref, +// None, +// settings) +// ) +// val history: History = generateDummyHistory(testNetSettings) +// +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) +// val peerHandler: TestProbe = TestProbe() +// val connectedPeer: ConnectedPeer = ConnectedPeer( +// address, +// peerHandler.ref, +// Outgoing, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test node", +// Some(address), +// System.currentTimeMillis()) +// ) +// +// val timestamp1 = System.currentTimeMillis() +// Thread.sleep(1000) +// val timestamp2 = System.currentTimeMillis() +// +// val header_first: Header = Header( +// 1.toByte, +// ModifierId @@ Random.randomBytes(), +// Digest32 @@ Random.randomBytes(), +// timestamp2, +// 2, +// scala.util.Random.nextLong(), +// testNetSettings.constants.InitialDifficulty, +// EquihashSolution(Seq(1, 3)), +// Random.randomBytes() +// ) +// val header_second: Header = Header( +// 1.toByte, +// header_first.id, +// Digest32 @@ Random.randomBytes(), +// timestamp1, +// 1, +// scala.util.Random.nextLong(), +// testNetSettings.constants.InitialDifficulty, +// EquihashSolution(Seq(1, 3)), +// Random.randomBytes() +// ) +// +// history.append(header_first) +// +// nodeViewSync.send(downloadedModifiersValidator, UpdatedHistory(history)) +// +// /* Header */ +// val mods = Seq(header_second).map(x => x.id -> HeaderProtoSerializer.toProto(x).toByteArray.reverse).toMap +// val msg = ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods) +// +// deliveryManager.send(downloadedModifiersValidator, msg) +// peersKeeper.expectMsg(BanPeer(connectedPeer, CorruptedSerializedBytes)) +// nodeViewHolder.expectNoMsg() +// nodeViewSync.expectMsg(InvalidModifier(header_second.id)) +// } +// "find corrupted payload" in { +// val nodeViewHolder = TestProbe() +// val peersKeeper = TestProbe() +// val deliveryManager = TestProbe() +// val nodeViewSync = TestProbe() +// val mempool = TestProbe() +// +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) +// val peerHandler: TestProbe = TestProbe() +// val connectedPeer: ConnectedPeer = ConnectedPeer( +// address, +// peerHandler.ref, +// Outgoing, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "test node", +// Some(address), +// System.currentTimeMillis()) +// ) +// +// val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( +// DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, +// nodeViewHolder.ref, +// peersKeeper.ref, +// nodeViewSync.ref, +// mempool.ref, +// None, +// settings) +// ) +// val history: History = generateDummyHistory(testNetSettings) +// +// val historyWith10Blocks = (0 until 10).foldLeft(history, Seq.empty[Block]) { +// case ((prevHistory, blocks), _) => +// val block: Block = generateNextBlock(prevHistory) +// prevHistory.append(block.header) +// prevHistory.append(block.payload) +// (prevHistory.reportModifierIsValid(block), blocks :+ block) +// } +// +// val payload = Payload(ModifierId @@ scorex.utils.Random.randomBytes(), Seq(coinbaseTransaction)) +// +// nodeViewSync.send(downloadedModifiersValidator, UpdatedHistory(historyWith10Blocks._1)) +// +// val bytes = PayloadProtoSerializer.toProto(payload).toByteArray +// +// val mods: Map[ModifierId, Array[Byte]] = (historyWith10Blocks._2.map( +// b => b.payload.id -> PayloadProtoSerializer.toProto(b.payload).toByteArray.reverse +// ) :+ (payload.id -> bytes)).toMap +// +// deliveryManager +// .send(downloadedModifiersValidator, ModifiersForValidating(connectedPeer, Payload.modifierTypeId, mods)) +// +// peersKeeper.expectMsg(BanPeer(connectedPeer, CorruptedSerializedBytes)) +// nodeViewHolder.expectMsg(ModifierFromRemote(payload)) +// } +// } +// +// def generateBlocks(qty: Int, history: History): (History, List[Block]) = +// (0 until qty).foldLeft(history, List.empty[Block]) { +// case ((prevHistory, blocks), _) => +// val block: Block = generateNextBlock(prevHistory) +// prevHistory.append(block.header) +// prevHistory.append(block.payload) +// val a = prevHistory.reportModifierIsValid(block) +// (a, blocks :+ block) +// } +//} diff --git a/src/test/scala/encry/view/history/HistoryComparisionResultTest.scala b/src/test/scala/encry/view/history/HistoryComparisionResultTest.scala index 265dfff90f..d58e74ce0e 100644 --- a/src/test/scala/encry/view/history/HistoryComparisionResultTest.scala +++ b/src/test/scala/encry/view/history/HistoryComparisionResultTest.scala @@ -1,120 +1,120 @@ -package encry.view.history - -import encry.consensus.HistoryConsensus._ -import encry.modifiers.InstanceFactory -import encry.network.DeliveryManagerTests.DMUtils.generateBlocks -import encry.settings.{EncryAppSettings, TestNetSettings} -import org.encryfoundation.common.modifiers.history.Block -import org.encryfoundation.common.network.SyncInfo -import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} - -class HistoryComparisionResultTest extends WordSpecLike - with Matchers - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - "History Reader" should { - "mark history as Equal where our best header is the same as other history best header" in { - val history: History = generateDummyHistory(testNetSettings) - val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 - val syncInfo: SyncInfo = SyncInfo(blocks.map(_.header.id)) - - val updatedHistory: History = blocks.foldLeft(history) { case (hst, block) => - hst.append(block.header) - hst.updateIdsForSyncInfo() - hst.append(block.payload) - hst.reportModifierIsValid(block) - } - - val comparisonResult = updatedHistory.compare(syncInfo) - assert(comparisonResult == Equal) - } - - "mark history as Older where our best header is in other history best chain, but not at the last position" in { - val history: History = generateDummyHistory(testNetSettings) - val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 - val syncInfo: SyncInfo = SyncInfo(blocks.map(_.header.id)) - - val updatedHistory: History = blocks.take(50).foldLeft(history) { case (hst, block) => - hst.append(block.header) - hst.updateIdsForSyncInfo() - hst.append(block.payload) - hst.reportModifierIsValid(block) - } - - val comparisonResult = updatedHistory.compare(syncInfo) - assert(comparisonResult == Older) - } - - "mark history as Younger when comparing history is empty" in { - val history: History = generateDummyHistory(testNetSettings) - val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 - val syncInfo: SyncInfo = SyncInfo(Seq.empty) - - val updatedHistory: History = blocks.foldLeft(history) { case (hst, block) => - hst.append(block.header) - hst.updateIdsForSyncInfo() - hst.append(block.payload) - hst.reportModifierIsValid(block) - } - - val comparisonResult = updatedHistory.compare(syncInfo) - assert(comparisonResult == Younger) - } - - "mark history as Younger when our history contains all other history but other history " + - "doesn't contain our last 70 headers" in { - val history: History = generateDummyHistory(testNetSettings) - val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 - val syncInfo: SyncInfo = SyncInfo(blocks.take(30).map(_.header.id)) - - val updatedHistory: History = blocks.foldLeft(history) { case (hst, block) => - hst.append(block.header) - hst.updateIdsForSyncInfo() - hst.append(block.payload) - hst.reportModifierIsValid(block) - } - - val comparisonResult = updatedHistory.compare(syncInfo) - assert(comparisonResult == Younger) - } - - "mark history as Fork when we have same point in histories" in { - val history: History = generateDummyHistory(testNetSettings) - - val fork = genForkOn(100, 1000, 25, 30, testNetSettings) - - val syncInfo: SyncInfo = SyncInfo( - fork._1.take(25).map(_.header.id) ++: fork._2.map(_.header.id) - ) - - val updatedHistory: History = fork._1.take(30).foldLeft(history) { case (hst, block) => - hst.append(block.header) - hst.updateIdsForSyncInfo() - hst.append(block.payload) - hst.reportModifierIsValid(block) - } - - val comparisonResult = updatedHistory.compare(syncInfo) - assert(comparisonResult == Fork) - } - - "mark history as Equal where both nodes do not keep any blocks" in { - val history: History = generateDummyHistory(testNetSettings) - val syncInfo: SyncInfo = SyncInfo(Seq.empty) - - val comparisonResult = history.compare(syncInfo) - assert(comparisonResult == Equal) - } - - "mark history as Older " in { - val history: History = generateDummyHistory(testNetSettings) - val syncInfo: SyncInfo = SyncInfo( - generateBlocks(30, generateDummyHistory(testNetSettings))._2.map(_.header.id)) - - val comparisonResult = history.compare(syncInfo) - assert(comparisonResult == Older) - } - } -} \ No newline at end of file +//package encry.view.history +// +//import encry.consensus.HistoryConsensus._ +//import encry.modifiers.InstanceFactory +//import encry.network.DeliveryManagerTests.DMUtils.generateBlocks +//import encry.settings.{EncryAppSettings, TestNetSettings} +//import org.encryfoundation.common.modifiers.history.Block +//import org.encryfoundation.common.network.SyncInfo +//import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} +// +//class HistoryComparisionResultTest extends WordSpecLike +// with Matchers +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// +// "History Reader" should { +// "mark history as Equal where our best header is the same as other history best header" in { +// val history: History = generateDummyHistory(testNetSettings) +// val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 +// val syncInfo: SyncInfo = SyncInfo(blocks.map(_.header.id)) +// +// val updatedHistory: History = blocks.foldLeft(history) { case (hst, block) => +// hst.append(block.header) +// hst.updateIdsForSyncInfo() +// hst.append(block.payload) +// hst.reportModifierIsValid(block) +// } +// +// val comparisonResult = updatedHistory.compare(syncInfo) +// assert(comparisonResult == Equal) +// } +// +// "mark history as Older where our best header is in other history best chain, but not at the last position" in { +// val history: History = generateDummyHistory(testNetSettings) +// val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 +// val syncInfo: SyncInfo = SyncInfo(blocks.map(_.header.id)) +// +// val updatedHistory: History = blocks.take(50).foldLeft(history) { case (hst, block) => +// hst.append(block.header) +// hst.updateIdsForSyncInfo() +// hst.append(block.payload) +// hst.reportModifierIsValid(block) +// } +// +// val comparisonResult = updatedHistory.compare(syncInfo) +// assert(comparisonResult == Older) +// } +// +// "mark history as Younger when comparing history is empty" in { +// val history: History = generateDummyHistory(testNetSettings) +// val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 +// val syncInfo: SyncInfo = SyncInfo(Seq.empty) +// +// val updatedHistory: History = blocks.foldLeft(history) { case (hst, block) => +// hst.append(block.header) +// hst.updateIdsForSyncInfo() +// hst.append(block.payload) +// hst.reportModifierIsValid(block) +// } +// +// val comparisonResult = updatedHistory.compare(syncInfo) +// assert(comparisonResult == Younger) +// } +// +// "mark history as Younger when our history contains all other history but other history " + +// "doesn't contain our last 70 headers" in { +// val history: History = generateDummyHistory(testNetSettings) +// val blocks: List[Block] = generateBlocks(100, generateDummyHistory(testNetSettings))._2 +// val syncInfo: SyncInfo = SyncInfo(blocks.take(30).map(_.header.id)) +// +// val updatedHistory: History = blocks.foldLeft(history) { case (hst, block) => +// hst.append(block.header) +// hst.updateIdsForSyncInfo() +// hst.append(block.payload) +// hst.reportModifierIsValid(block) +// } +// +// val comparisonResult = updatedHistory.compare(syncInfo) +// assert(comparisonResult == Younger) +// } +// +// "mark history as Fork when we have same point in histories" in { +// val history: History = generateDummyHistory(testNetSettings) +// +// val fork = genForkOn(100, 1000, 25, 30, testNetSettings) +// +// val syncInfo: SyncInfo = SyncInfo( +// fork._1.take(25).map(_.header.id) ++: fork._2.map(_.header.id) +// ) +// +// val updatedHistory: History = fork._1.take(30).foldLeft(history) { case (hst, block) => +// hst.append(block.header) +// hst.updateIdsForSyncInfo() +// hst.append(block.payload) +// hst.reportModifierIsValid(block) +// } +// +// val comparisonResult = updatedHistory.compare(syncInfo) +// assert(comparisonResult == Fork) +// } +// +// "mark history as Equal where both nodes do not keep any blocks" in { +// val history: History = generateDummyHistory(testNetSettings) +// val syncInfo: SyncInfo = SyncInfo(Seq.empty) +// +// val comparisonResult = history.compare(syncInfo) +// assert(comparisonResult == Equal) +// } +// +// "mark history as Older " in { +// val history: History = generateDummyHistory(testNetSettings) +// val syncInfo: SyncInfo = SyncInfo( +// generateBlocks(30, generateDummyHistory(testNetSettings))._2.map(_.header.id)) +// +// val comparisonResult = history.compare(syncInfo) +// assert(comparisonResult == Older) +// } +// } +//} \ No newline at end of file diff --git a/src/test/scala/encry/view/history/ModifiersValidationTest.scala b/src/test/scala/encry/view/history/ModifiersValidationTest.scala index e879864ea8..722258dc54 100644 --- a/src/test/scala/encry/view/history/ModifiersValidationTest.scala +++ b/src/test/scala/encry/view/history/ModifiersValidationTest.scala @@ -1,45 +1,45 @@ -package encry.view.history - -import encry.modifiers.InstanceFactory -import encry.network.DeliveryManagerTests.DMUtils.generateBlocks -import encry.settings.{EncryAppSettings, TestNetSettings} -import org.encryfoundation.common.modifiers.history.Block -import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} - -class ModifiersValidationTest extends WordSpecLike - with Matchers - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - "Modifiers validator" should { - "validate genesis block" in { - val newHistory: History = generateDummyHistory(testNetSettings) - val genesisBlock: Block = generateGenesisBlock(testNetSettings.constants.GenesisHeight) - newHistory.testApplicable(genesisBlock.header).isRight shouldBe true - newHistory.append(genesisBlock.header) - val updatedHistory: History = newHistory.reportModifierIsValid(genesisBlock.header) - updatedHistory.testApplicable(genesisBlock.payload).isRight shouldBe true - } - "reject incorrect modifiers" in { - val blocks: List[Block] = generateBlocks(2, generateDummyHistory(testNetSettings))._2 - val newHistory: History = generateDummyHistory(testNetSettings) - blocks.take(1).foldLeft(newHistory) { case (history, block) => - history.testApplicable(block.header).isRight shouldBe true - history.append(block.header) - history.reportModifierIsValid(block.header) - history.testApplicable(block.payload).isRight shouldBe true - history.append(block.payload) - history.reportModifierIsValid(block) - } - blocks.takeRight(1).foldLeft(newHistory) { case (history, block) => - history.testApplicable(block.header).isRight shouldBe false - history.append(block.header) - history.reportModifierIsValid(block.header) - history.testApplicable(block.payload).isRight shouldBe true - history.append(block.payload) - history.reportModifierIsValid(block) - } - } - } -} \ No newline at end of file +//package encry.view.history +// +//import encry.modifiers.InstanceFactory +//import encry.network.DeliveryManagerTests.DMUtils.generateBlocks +//import encry.settings.{EncryAppSettings, TestNetSettings} +//import org.encryfoundation.common.modifiers.history.Block +//import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} +// +//class ModifiersValidationTest extends WordSpecLike +// with Matchers +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// +// "Modifiers validator" should { +// "validate genesis block" in { +// val newHistory: History = generateDummyHistory(testNetSettings) +// val genesisBlock: Block = generateGenesisBlock(testNetSettings.constants.GenesisHeight) +// newHistory.testApplicable(genesisBlock.header).isRight shouldBe true +// newHistory.append(genesisBlock.header) +// val updatedHistory: History = newHistory.reportModifierIsValid(genesisBlock.header) +// updatedHistory.testApplicable(genesisBlock.payload).isRight shouldBe true +// } +// "reject incorrect modifiers" in { +// val blocks: List[Block] = generateBlocks(2, generateDummyHistory(testNetSettings))._2 +// val newHistory: History = generateDummyHistory(testNetSettings) +// blocks.take(1).foldLeft(newHistory) { case (history, block) => +// history.testApplicable(block.header).isRight shouldBe true +// history.append(block.header) +// history.reportModifierIsValid(block.header) +// history.testApplicable(block.payload).isRight shouldBe true +// history.append(block.payload) +// history.reportModifierIsValid(block) +// } +// blocks.takeRight(1).foldLeft(newHistory) { case (history, block) => +// history.testApplicable(block.header).isRight shouldBe false +// history.append(block.header) +// history.reportModifierIsValid(block.header) +// history.testApplicable(block.payload).isRight shouldBe true +// history.append(block.payload) +// history.reportModifierIsValid(block) +// } +// } +// } +//} \ No newline at end of file From dca7c6f516ab7c1a63b773b6a525997833b579b5 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 15:15:47 +0300 Subject: [PATCH 032/177] Merge branch 'EC-859' of /Users/timofey/idea/encry/core with conflicts. --- .../scala/encry/nvg/IntermediaryNVH.scala | 15 +- .../scala/encry/nvg/ModifiersValidator.scala | 2 +- .../encry/nvg/NetworkMessagesProcessor.scala | 30 +- src/main/scala/encry/nvg/NodeViewHolder.scala | 23 +- .../DeliveryManagerRequestModifiesSpec.scala | 633 +++++++++--------- 5 files changed, 351 insertions(+), 352 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index d338e100fb..9a8beaf0fa 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -7,7 +7,7 @@ import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.DownloadedModifiersValidator.InvalidModifier -import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal } import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } import encry.network.NodeViewSynchronizer.ReceivableMessages.{ @@ -82,17 +82,16 @@ class IntermediaryNVH( networkMessagesProcessor ! newReader case msg @ BanPeer(_, _) => intermediaryNetwork ! msg case msg @ InvalidModifier(_) => intermediaryNetwork ! msg - case msg @ FastSyncDone => intermediaryNetwork ! msg - case msg @ DownloadRequest(_, _, _) => intermediaryNetwork ! msg + case msg @ DownloadRequest(_, _) => intermediaryNetwork ! msg case msg @ OtherNodeSyncingStatus(_, _, _) => intermediaryNetwork ! msg case msg @ RequestFromLocal(_, _, _) => intermediaryNetwork ! msg - case msg @ ModifiersNetworkMessage(_, _) => intermediaryNetwork ! msg - case msg @ SendToNetwork(_, _) => intermediaryNetwork ! msg - case msg @ IdsForRequest(_) => intermediaryNetwork ! msg + case msg @ ResponseFromLocal(_, _, _) => intermediaryNetwork ! msg + case msg @ BroadcastModifier(_, _) => intermediaryNetwork ! msg case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync case msg @ TreeChunks(_, _) => //+ to fast sync - case msg @ HeaderChainIsSynced => intermediaryNetwork ! msg - case msg @ FullBlockChainIsSynced => intermediaryNetwork ! msg //+ to miner + case msg @ FastSyncDone => + case msg @ HeaderChainIsSynced => + case msg @ FullBlockChainIsSynced => //+ to miner case msg @ DisableMining => //+ to miner case msg @ StartMining => //+ to miner case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index 215f1ec2a2..cb91e6d0d1 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -44,7 +44,7 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings } else if (!syntacticValidation) { logger.info(s"Modifier ${modifier.encodedId} is syntactically invalid.") context.parent ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) - context.parent ! InvalidModifier(modifierId) + context.parent ! SyntacticallyInvalidPersistentModifier(modifierId) } else preSemanticValidation.leftMap { case IllegalHeight(error) => diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 7ed1c56f3e..767cdbb013 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -1,23 +1,26 @@ package encry.nvg -import akka.actor.{Actor, Cancellable, Props} +import akka.actor.{ Actor, Cancellable, Props } import com.typesafe.scalalogging.StrictLogging -import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Younger} -import encry.network.Broadcast +import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } import cats.syntax.option._ import encry.network.DeliveryManager.CheckPayloadsToDownload -import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal } import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.NodeViewSynchronizer.ReceivableMessages.{OtherNodeSyncingStatus, SemanticallySuccessfulModifier} -import encry.network.PeersKeeper.SendToNetwork -import encry.nvg.NetworkMessagesProcessor.IdsForRequest +import encry.network.NodeViewSynchronizer.ReceivableMessages.{ OtherNodeSyncingStatus, SemanticallySuccessfulModifier } +import encry.nvg.NodeViewHolder.DownloadRequest import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} -import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.network.BasicMessagesRepo.{ + InvNetworkMessage, + ModifiersNetworkMessage, + RequestModifiersNetworkMessage, + SyncInfoNetworkMessage +} import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -44,7 +47,7 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St case block: Block if historyReader.isFullChainSynced => List(block.header, block.payload).foreach { mod: PersistentModifier => logger.info(s"Going to broadcast inv for modifier of type ${mod.modifierTypeId} with id: ${mod.encodedId}.") - context.parent ! SendToNetwork(InvNetworkMessage(mod.modifierTypeId -> Seq(mod.id)), Broadcast) + context.parent ! BroadcastModifier(mod.modifierTypeId, mod.id) } modifiersRequestCache = Map( block.encodedId -> toProto(block.header), @@ -100,18 +103,18 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St typeId match { case h if h == Header.modifierTypeId => - context.parent ! ModifiersNetworkMessage(typeId -> getModsForRemote(unrequestedModifiers, historyReader)) + context.parent ! ResponseFromLocal(remote, typeId, getModsForRemote(unrequestedModifiers, historyReader)) case _ => getModsForRemote(unrequestedModifiers, historyReader).foreach { case (id: ModifierId, bytes: Array[Byte]) => - context.parent ! ModifiersNetworkMessage(typeId -> Map(id -> bytes)) + context.parent ! ResponseFromLocal(remote, typeId, Map(id -> bytes)) } } } case CheckPayloadsToDownload => val newIds: Seq[ModifierId] = historyReader.payloadsIdsToDownload(settings.network.networkChunkSize) logger.debug(s"newIds: ${newIds.map(elem => Algos.encode(elem)).mkString(",")}") - if (newIds.nonEmpty) context.parent ! IdsForRequest(newIds.toList) + if (newIds.nonEmpty) context.parent ! DownloadRequest(Payload.modifierTypeId, newIds.toList) val nextCheckModsScheduler: Cancellable = context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)(self ! CheckPayloadsToDownload) context.become(workingCycle(nextCheckModsScheduler.some)) @@ -125,6 +128,5 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St } object NetworkMessagesProcessor { - final case class IdsForRequest(ids: List[ModifierId]) extends AnyVal def props(settings: EncryAppSettings): Props = Props(new NetworkMessagesProcessor(settings)) } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index a8f8aae222..60bfe7705c 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -2,18 +2,18 @@ package encry.nvg import java.io.File -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{DisableMining, StartMining} +import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} -import encry.nvg.NodeViewHolder.{DownloadRequest, NodeView, UpdateInformation} +import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } +import encry.nvg.NodeViewHolder.{ DownloadRequest, NodeView, UpdateInformation } import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag @@ -23,7 +23,7 @@ import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.storage.HistoryStorage -import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} +import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } import encry.view.mempool.MemoryPool.RolledBackTransactions import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree @@ -31,14 +31,14 @@ import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } -import scala.collection.{IndexedSeq, Seq, mutable} +import scala.collection.{ mutable, IndexedSeq, Seq } import scala.concurrent.duration._ -import scala.util.{Failure, Success, Try} +import scala.util.{ Failure, Success, Try } class NodeViewHolder( settings: EncryAppSettings, @@ -171,7 +171,7 @@ class NodeViewHolder( s"Previous modifier is ${previousModifier.map(Algos.encode)}." ) if (tid != Payload.modifierTypeId || (nodeView.history.isFullChainSynced && tid == Payload.modifierTypeId)) - context.parent ! DownloadRequest(tid, id, previousModifier) + context.parent ! DownloadRequest(tid, List(id)) else logger.info( s"Ignore sending download request for modifier ${Algos.encode(id)} because full chain is not synced." @@ -501,8 +501,7 @@ object NodeViewHolder { final case class DownloadRequest( modifierTypeId: ModifierTypeId, - modifierId: ModifierId, - previousModifier: Option[ModifierId] = none + modifierIds: List[ModifierId] ) extends NodeViewHolderEvent final case class UpdateInformation( diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala index 0ab92b2abf..067b867ecd 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala @@ -1,320 +1,319 @@ package encry.network.DeliveryManagerTests import java.net.InetSocketAddress -// -//import akka.actor.ActorSystem -//import akka.testkit.{TestActorRef, TestKit, TestProbe} -//import encry.consensus.HistoryConsensus -//import encry.consensus.HistoryConsensus.{Fork, Older, Younger} -//import encry.modifiers.InstanceFactory -//import encry.network.DeliveryManager -//import encry.network.NetworkController.ReceivableMessages.DataFromPeer -//import encry.network.NodeViewSynchronizer.ReceivableMessages.RequestFromLocal -//import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} -//import encry.settings.TestNetSettings -//import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -//import encry.network.DeliveryManagerTests.DMUtils._ -//import encry.network.PeersKeeper.UpdatedPeersCollection -//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus -//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ -//import encry.view.NodeViewHolder.DownloadRequest -//import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} -//import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -//import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} -//import org.encryfoundation.common.network.SyncInfo -//import org.encryfoundation.common.utils.Algos -//import org.encryfoundation.common.utils.TaggedTypes.ModifierId -// -//import scala.collection.mutable.WrappedArray - -//class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll -// with Matchers -// with InstanceFactory -// with OneInstancePerTest -// with TestNetSettings { -// -// implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") -// -// override def afterAll(): Unit = TestKit.shutdownActorSystem(system) -// -// def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], -// ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { -// val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) -// val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) -// val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) -// val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) -// val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 -// val headersIds: List[ModifierId] = blocks.map(_.header.id) -// val headersAsKey = headersIds.map(toKey) -// (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) -// } -// -// "RequestModifies" should { -// "handle uniq modifiers from RequestFromLocal message correctly" in { -// val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.size == headersIds.size) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.forall(elem => headersAsKey.contains(elem))) -// deliveryManager.stop() -// } -// "not handle repeating modifiers from RequestFromLocal message" in { -// val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.size == headersIds.size) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.forall(elem => headersAsKey.contains(elem))) -// deliveryManager.stop() -// } -// "Delivery Manager should handle received modifier which were requested correctly" in { -// val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( -// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.isEmpty) -// assert(deliveryManager.underlyingActor.receivedModifiers.size == blocks.size) -// assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) -// deliveryManager.stop() -// } -// "Delivery manager should not handle repeating modifiers" in { -// val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( -// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( -// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) -// assert(deliveryManager.underlyingActor.receivedModifiers.size == headersIds.size) -// assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) -// deliveryManager.stop() -// } -// "handle priority request for payload correctly" in { -// val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( -// Header.modifierTypeId -> blocks.map(k => k.header.id -> Array.emptyByteArray).toMap), cp1) -// headersIds.foreach(id => -// deliveryManager ! DownloadRequest(Payload.modifierTypeId, blocks.find(block => -// block.id.sameElements(id)).get.payload.id)) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .size == blocks.size) -// deliveryManager.stop() -// } -// "choose correct peer in priority request" in { -// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val address2 = new InetSocketAddress("123.123.123.124", 9001) -// val handler2: TestProbe = TestProbe() -// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.124", Some(address2), System.currentTimeMillis())) -// -// val address3 = new InetSocketAddress("123.123.123.125", 9001) -// val handler3: TestProbe = TestProbe() -// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.125", Some(address3), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map( -// address1 -> (cp1, Older, InitialPriority), -// address2 -> (cp2, Older, InitialPriority), -// address3 -> (cp3, Older, InitialPriority) -// ) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val header: Header = blocks.head.header -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) -// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) -// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) -// -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1) -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2) -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3) -// -// deliveryManager ! DownloadRequest(Payload.modifierTypeId, header.payloadId) -// -// handler1.expectMsgAnyOf( -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), -// RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), -// SyncInfoNetworkMessage(SyncInfo(List())) -// ) -// -// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) -// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) -// deliveryManager.stop() -// } -// "not ask modifiers while block chain is not synced from Younger nodes" in { -// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) -// -// val address2 = new InetSocketAddress("123.123.123.124", 9001) -// val handler2: TestProbe = TestProbe() -// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.124", Some(address2), System.currentTimeMillis())) -// -// val address3 = new InetSocketAddress("123.123.123.125", 9001) -// val handler3: TestProbe = TestProbe() -// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.125", Some(address3), System.currentTimeMillis())) -// -// val updatedPeersCollection = -// Map( -// address2 -> (cp2, Younger, InitialPriority), -// address3 -> (cp3, Fork, InitialPriority) -// ) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// -// val header: Header = blocks.head.header -// -// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) -// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) -// -// handler2.expectNoMsg() -// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) -// deliveryManager.stop() -// } -// "not ask modifiers from peer which is not contained in status tracker" in { -// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val address2 = new InetSocketAddress("123.123.123.124", 9001) -// val handler2: TestProbe = TestProbe() -// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.124", Some(address2), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address2 -> (cp2, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val header: Header = blocks.head.header -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) -// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) -// -// handler1.expectNoMsg() -// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) -// deliveryManager.stop() -// } -// "not ask transactions while block chain is not synced" in { -// val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) -// val txs: Seq[Transaction] = genInvalidPaymentTxs(1) -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) -// -// handler1.expectNoMsg() -// deliveryManager.stop() -// } -// "not ask transaction while node is not mining" in { -// val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) -// val txs: Seq[Transaction] = genInvalidPaymentTxs(1) -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) -// -// handler1.expectNoMsg() -// deliveryManager.stop() -// } -// "not re-ask modifiers which already have been received" in { -// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val header: Header = blocks.head.header -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) -// -// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(header).toByteArray -// -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> headerBytes)), cp1) -// -// handler1.expectMsgAllOf( -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)) -// ) -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) -// -// handler1.expectNoMsg() -// -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.isEmpty) -// assert(deliveryManager.underlyingActor.receivedModifiers.size == 1) -// assert(deliveryManager.underlyingActor.receivedModifiers.contains(toKey(header.id))) -// deliveryManager.stop() -// } -// } -//} \ No newline at end of file + +import akka.actor.ActorSystem +import akka.testkit.{TestActorRef, TestKit, TestProbe} +import encry.consensus.HistoryConsensus +import encry.consensus.HistoryConsensus.{Fork, Older, Younger} +import encry.modifiers.InstanceFactory +import encry.network.DeliveryManager +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} +import encry.settings.TestNetSettings +import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +import encry.network.DeliveryManagerTests.DMUtils._ +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.PeersKeeper.UpdatedPeersCollection +import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus +import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ +import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} +import org.encryfoundation.common.network.SyncInfo +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.ModifierId + +import scala.collection.mutable.WrappedArray + +class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll + with Matchers + with InstanceFactory + with OneInstancePerTest + with TestNetSettings { + + implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") + + override def afterAll(): Unit = TestKit.shutdownActorSystem(system) + + def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], + ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { + val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) + val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) + val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) + val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) + val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 + val headersIds: List[ModifierId] = blocks.map(_.header.id) + val headersAsKey = headersIds.map(toKey) + (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) + } + + "RequestModifies" should { + "handle uniq modifiers from RequestFromLocal message correctly" in { + val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) + assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) + .keys.size == headersIds.size) + assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) + .keys.forall(elem => headersAsKey.contains(elem))) + deliveryManager.stop() + } + "not handle repeating modifiers from RequestFromLocal message" in { + val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) + assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) + .keys.size == headersIds.size) + assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) + .keys.forall(elem => headersAsKey.contains(elem))) + deliveryManager.stop() + } + "Delivery Manager should handle received modifier which were requested correctly" in { + val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) + val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray + deliveryManager ! DataFromPeer(ModifiersNetworkMessage( + Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) + assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) + .keys.isEmpty) + assert(deliveryManager.underlyingActor.receivedModifiers.size == blocks.size) + assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) + deliveryManager.stop() + } + "Delivery manager should not handle repeating modifiers" in { + val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) + val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray + deliveryManager ! DataFromPeer(ModifiersNetworkMessage( + Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) + deliveryManager ! DataFromPeer(ModifiersNetworkMessage( + Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) + assert(deliveryManager.underlyingActor.receivedModifiers.size == headersIds.size) + assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) + deliveryManager.stop() + } + "handle priority request for payload correctly" in { + val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) + deliveryManager ! DataFromPeer(ModifiersNetworkMessage( + Header.modifierTypeId -> blocks.map(k => k.header.id -> Array.emptyByteArray).toMap), cp1) + headersIds.foreach(id => + deliveryManager ! DownloadRequest(Payload.modifierTypeId, blocks.find(block => + block.id.sameElements(id)).get.payload.id)) + assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) + .size == blocks.size) + deliveryManager.stop() + } + "choose correct peer in priority request" in { + val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val address2 = new InetSocketAddress("123.123.123.124", 9001) + val handler2: TestProbe = TestProbe() + val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.124", Some(address2), System.currentTimeMillis())) + + val address3 = new InetSocketAddress("123.123.123.125", 9001) + val handler3: TestProbe = TestProbe() + val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.125", Some(address3), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map( + address1 -> (cp1, Older, InitialPriority), + address2 -> (cp2, Older, InitialPriority), + address3 -> (cp3, Older, InitialPriority) + ) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + val header: Header = blocks.head.header + + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) + deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) + deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) + + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1) + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2) + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3) + + deliveryManager ! DownloadRequest(Payload.modifierTypeId, header.payloadId) + + handler1.expectMsgAnyOf( + RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), + RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), + SyncInfoNetworkMessage(SyncInfo(List())) + ) + + handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) + handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) + deliveryManager.stop() + } + "not ask modifiers while block chain is not synced from Younger nodes" in { + val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) + + val address2 = new InetSocketAddress("123.123.123.124", 9001) + val handler2: TestProbe = TestProbe() + val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.124", Some(address2), System.currentTimeMillis())) + + val address3 = new InetSocketAddress("123.123.123.125", 9001) + val handler3: TestProbe = TestProbe() + val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.125", Some(address3), System.currentTimeMillis())) + + val updatedPeersCollection = + Map( + address2 -> (cp2, Younger, InitialPriority), + address3 -> (cp3, Fork, InitialPriority) + ) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + + val header: Header = blocks.head.header + + deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) + deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) + + handler2.expectNoMsg() + handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) + deliveryManager.stop() + } + "not ask modifiers from peer which is not contained in status tracker" in { + val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val address2 = new InetSocketAddress("123.123.123.124", 9001) + val handler2: TestProbe = TestProbe() + val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.124", Some(address2), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address2 -> (cp2, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + val header: Header = blocks.head.header + + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) + deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) + + handler1.expectNoMsg() + handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) + deliveryManager.stop() + } + "not ask transactions while block chain is not synced" in { + val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) + val txs: Seq[Transaction] = genInvalidPaymentTxs(1) + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address1 -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) + + handler1.expectNoMsg() + deliveryManager.stop() + } + "not ask transaction while node is not mining" in { + val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) + val txs: Seq[Transaction] = genInvalidPaymentTxs(1) + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address1 -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) + + handler1.expectNoMsg() + deliveryManager.stop() + } + "not re-ask modifiers which already have been received" in { + val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address1 -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + val header: Header = blocks.head.header + + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) + + val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(header).toByteArray + + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> headerBytes)), cp1) + + handler1.expectMsgAllOf( + RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)) + ) + + deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) + + handler1.expectNoMsg() + + assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) + .keys.isEmpty) + assert(deliveryManager.underlyingActor.receivedModifiers.size == 1) + assert(deliveryManager.underlyingActor.receivedModifiers.contains(toKey(header.id))) + deliveryManager.stop() + } + } +} \ No newline at end of file From 3ad6215d16562d9dd824769e94fa0bec99980797 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 15:25:30 +0300 Subject: [PATCH 033/177] added messages --- .../DownloadedModifiersValidator.scala | 1 - .../scala/encry/nvg/IntermediaryNVH.scala | 46 +++++++++---------- .../scala/encry/nvg/ModifiersValidator.scala | 12 +++-- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/main/scala/encry/network/DownloadedModifiersValidator.scala b/src/main/scala/encry/network/DownloadedModifiersValidator.scala index 3c4651e727..156bdc9009 100644 --- a/src/main/scala/encry/network/DownloadedModifiersValidator.scala +++ b/src/main/scala/encry/network/DownloadedModifiersValidator.scala @@ -66,7 +66,6 @@ object DownloadedModifiersValidator { typeId: ModifierTypeId, modifiers: Map[ModifierId, Array[Byte]]) - final case class InvalidModifier(ids: ModifierId) extends AnyVal // def props(modifierIdSize: Int, // nodeViewHolder: ActorRef, diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 9a8beaf0fa..f46f3baf63 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -6,7 +6,6 @@ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.DownloadedModifiersValidator.InvalidModifier import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal } import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } @@ -15,11 +14,11 @@ import encry.network.NodeViewSynchronizer.ReceivableMessages.{ RollbackFailed, RollbackSucceed, SemanticallyFailedModification, - SemanticallySuccessfulModifier + SemanticallySuccessfulModifier, + SyntacticallyFailedModification } import encry.network.PeersKeeper.{ BanPeer, SendToNetwork } -import encry.nvg.ModifiersValidator.ModifierForValidation -import encry.nvg.NetworkMessagesProcessor.IdsForRequest +import encry.nvg.ModifiersValidator.{ InvalidNetworkBytes, ModifierForValidation } import encry.nvg.NodeViewHolder.{ DownloadRequest, UpdateHistoryReader } import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage @@ -80,25 +79,26 @@ class IntermediaryNVH( case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! newReader - case msg @ BanPeer(_, _) => intermediaryNetwork ! msg - case msg @ InvalidModifier(_) => intermediaryNetwork ! msg - case msg @ DownloadRequest(_, _) => intermediaryNetwork ! msg - case msg @ OtherNodeSyncingStatus(_, _, _) => intermediaryNetwork ! msg - case msg @ RequestFromLocal(_, _, _) => intermediaryNetwork ! msg - case msg @ ResponseFromLocal(_, _, _) => intermediaryNetwork ! msg - case msg @ BroadcastModifier(_, _) => intermediaryNetwork ! msg - case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync - case msg @ TreeChunks(_, _) => //+ to fast sync - case msg @ FastSyncDone => - case msg @ HeaderChainIsSynced => - case msg @ FullBlockChainIsSynced => //+ to miner - case msg @ DisableMining => //+ to miner - case msg @ StartMining => //+ to miner - case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder - case msg @ RolledBackTransactions(_) => //+ to memory pool - case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) - case msg @ RollbackSucceed(_) => - case msg @ RollbackFailed(_) => + case msg @ BanPeer(_, _) => intermediaryNetwork ! msg + case msg @ InvalidNetworkBytes(_) => intermediaryNetwork ! msg + case msg @ DownloadRequest(_, _) => intermediaryNetwork ! msg + case msg @ OtherNodeSyncingStatus(_, _, _) => intermediaryNetwork ! msg + case msg @ RequestFromLocal(_, _, _) => intermediaryNetwork ! msg + case msg @ ResponseFromLocal(_, _, _) => intermediaryNetwork ! msg + case msg @ BroadcastModifier(_, _) => intermediaryNetwork ! msg + case msg @ SyntacticallyFailedModification(_, _) => intermediaryNetwork ! msg + case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync + case msg @ TreeChunks(_, _) => //+ to fast sync + case msg @ FastSyncDone => + case msg @ HeaderChainIsSynced => + case msg @ FullBlockChainIsSynced => //+ to miner + case msg @ DisableMining => //+ to miner + case msg @ StartMining => //+ to miner + case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder + case msg @ RolledBackTransactions(_) => //+ to memory pool + case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) + case msg @ RollbackSucceed(_) => + case msg @ RollbackFailed(_) => case msg @ SemanticallySuccessfulModifier(_) => intermediaryNetwork ! msg networkMessagesProcessor ! msg diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index cb91e6d0d1..73d44dcfc2 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -14,9 +14,9 @@ import encry.network.BlackList.BanReason.{ PreSemanticInvalidModifier, SyntacticallyInvalidPersistentModifier } -import encry.network.DownloadedModifiersValidator.InvalidModifier +import encry.network.NodeViewSynchronizer.ReceivableMessages.SyntacticallyFailedModification import encry.network.PeersKeeper.BanPeer -import encry.nvg.ModifiersValidator.{ ModifierForValidation, ValidatedModifier } +import encry.nvg.ModifiersValidator.{ InvalidNetworkBytes, ModifierForValidation, ValidatedModifier } import encry.settings.EncryAppSettings import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier @@ -32,7 +32,7 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings case Left(error) => logger.info(s"Modifier ${Algos.encode(modifierId)} is incorrect cause: ${error.getMessage}.") context.parent ! BanPeer(remote, CorruptedSerializedBytes) - context.parent ! InvalidModifier(modifierId) + context.parent ! InvalidNetworkBytes(modifierId) case Right(modifier) => val preSemanticValidation: Either[PreSemanticValidationException, Unit] = isPreSemanticValid(modifier, reader, settings) @@ -44,13 +44,13 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings } else if (!syntacticValidation) { logger.info(s"Modifier ${modifier.encodedId} is syntactically invalid.") context.parent ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) - context.parent ! SyntacticallyInvalidPersistentModifier(modifierId) + context.parent ! SyntacticallyFailedModification(modifier, List.empty) } else preSemanticValidation.leftMap { case IllegalHeight(error) => logger.info(s"Modifier ${modifier.encodedId} is invalid cause: $error.") context.parent ! BanPeer(remote, PreSemanticInvalidModifier(error)) - context.parent ! InvalidModifier(modifierId) + context.parent ! SyntacticallyFailedModification(modifier, List.empty) } } } @@ -102,6 +102,8 @@ object ModifiersValidator { final case class ValidatedModifier(modifier: PersistentModifier) extends AnyVal + final case class InvalidNetworkBytes(id: ModifierId) extends AnyVal + def props(nodeViewHolderRef: ActorRef, settings: EncryAppSettings): Props = Props(new ModifiersValidator(nodeViewHolderRef, settings)) } From 88e357395841d1591f198cdb7b356423e8d61961 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 5 Mar 2020 16:00:49 +0300 Subject: [PATCH 034/177] made source in reqFromLocal optional --- src/main/scala/encry/network/Messages.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/encry/network/Messages.scala b/src/main/scala/encry/network/Messages.scala index faf0078240..10f881290e 100644 --- a/src/main/scala/encry/network/Messages.scala +++ b/src/main/scala/encry/network/Messages.scala @@ -8,7 +8,7 @@ object Messages { sealed trait MessageToNetwork object MessageToNetwork { - final case class RequestFromLocal(source: InetSocketAddress, + final case class RequestFromLocal(source: Option[InetSocketAddress], modifierTypeId: ModifierTypeId, modifierIds: List[ModifierId]) extends MessageToNetwork final case class SendSyncInfo(syncInfo: SyncInfo) extends MessageToNetwork From db4d2ff2848b568a1a175227bf9b80c6e26713b3 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 16:11:46 +0300 Subject: [PATCH 035/177] history reader implemented added sync info scheduler removed response from local message added nvg start --- src/main/scala/encry/Starter.scala | 21 +++++++---- .../scala/encry/network/DeliveryManager.scala | 3 +- .../scala/encry/nvg/IntermediaryNVH.scala | 37 +++++++++---------- .../scala/encry/nvg/ModifiersValidator.scala | 6 +-- .../encry/nvg/NetworkMessagesProcessor.scala | 17 +++++++-- src/main/scala/encry/nvg/NodeViewHolder.scala | 24 ++++++------ .../encry/view/history/HistoryReader.scala | 35 ++++++++++-------- 7 files changed, 80 insertions(+), 63 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 25b5fd71e7..90980f3225 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -20,6 +20,7 @@ import encry.cli.ConsoleListener.{StartListening, prompt} import encry.local.miner.Miner import encry.local.miner.Miner.StartMining import encry.network.{NetworkRouter, NodeViewSynchronizer} +import encry.nvg.IntermediaryNVH import encry.settings._ import encry.stats.StatsSender import encry.utils.{Mnemonic, NetworkTimeProvider} @@ -415,12 +416,12 @@ class Starter(settings: EncryAppSettings, .props(newSettings, timeProvider, miner, influxRef) .withDispatcher("mempool-dispatcher") ) - val nodeViewHolder: ActorRef = context.system.actorOf( - NodeViewHolder - .props(memoryPool, influxRef, dataHolderForApi, newSettings) - .withDispatcher("nvh-dispatcher"), - "nodeViewHolder" - ) +// val nodeViewHolder: ActorRef = context.system.actorOf( +// NodeViewHolder +// .props(memoryPool, influxRef, dataHolderForApi, newSettings) +// .withDispatcher("nvh-dispatcher"), +// "nodeViewHolder" +// ) if (nodePass.nonEmpty) dataHolderForApi ! PassForStorage(nodePass) @@ -431,13 +432,19 @@ class Starter(settings: EncryAppSettings, // "nodeViewSynchronizer" // ) - context.system.actorOf( + val networkRouter = context.system.actorOf( NetworkRouter .props(networkSettings, settings.blackList) .withDispatcher("nvsh-dispatcher"), "nodeViewSynchronizer" ) + val nvhRouter = context.system.actorOf( + IntermediaryNVH.props(settings, networkRouter, timeProvider, influxRef) + ) + + + if (newSettings.node.mining) miner ! StartMining if (newSettings.node.useCli) { context.system diff --git a/src/main/scala/encry/network/DeliveryManager.scala b/src/main/scala/encry/network/DeliveryManager.scala index 3fcd742e84..36774ef7ea 100644 --- a/src/main/scala/encry/network/DeliveryManager.scala +++ b/src/main/scala/encry/network/DeliveryManager.scala @@ -20,14 +20,13 @@ import scala.collection.{IndexedSeq, mutable} import scala.util.Random import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import com.typesafe.config.Config -import encry.network.DownloadedModifiersValidator.{InvalidModifier, ModifiersForValidating} +import encry.network.DownloadedModifiersValidator.{ModifiersForValidating} import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.PeersKeeper.ConnectionStatusMessages.ConnectionStopped import encry.network.PeersKeeper._ import encry.network.PrioritiesCalculator.AccumulatedPeersStatistic import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus.BadNode -import encry.nvg.NodeViewHolder.DownloadRequest import encry.view.mempool.MemoryPool.{StartTransactionsValidation, StopTransactionsValidation} import org.encryfoundation.common.modifiers.history.{Block, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index f46f3baf63..4806d992e6 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -1,25 +1,18 @@ package encry.nvg -import akka.actor.{ Actor, ActorRef } +import akka.actor.{ Actor, ActorRef, Props } import akka.routing.BalancingPool import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal } +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } -import encry.network.NodeViewSynchronizer.ReceivableMessages.{ - OtherNodeSyncingStatus, - RollbackFailed, - RollbackSucceed, - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification -} -import encry.network.PeersKeeper.{ BanPeer, SendToNetwork } -import encry.nvg.ModifiersValidator.{ InvalidNetworkBytes, ModifierForValidation } -import encry.nvg.NodeViewHolder.{ DownloadRequest, UpdateHistoryReader } +import encry.network.NodeViewSynchronizer.ReceivableMessages._ +import encry.network.PeersKeeper.BanPeer +import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation } +import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider @@ -33,7 +26,6 @@ import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, - ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage } @@ -76,17 +68,17 @@ class IntermediaryNVH( logger.info(s"Got modifier ${Algos.encode(modifierId)} of type $typeId from $remote for validation.") modifiersValidatorRouter ! ModifierForValidation(historyReader, modifierId, typeId, modifierBytes, remote) case msg @ DataFromPeer(_, _) => networkMessagesProcessor ! msg - case UpdateHistoryReader(newReader: HistoryReader) => + case msg @ UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader - networkMessagesProcessor ! newReader + networkMessagesProcessor ! msg case msg @ BanPeer(_, _) => intermediaryNetwork ! msg - case msg @ InvalidNetworkBytes(_) => intermediaryNetwork ! msg - case msg @ DownloadRequest(_, _) => intermediaryNetwork ! msg + case msg @ InvalidModifierBytes(_) => intermediaryNetwork ! msg case msg @ OtherNodeSyncingStatus(_, _, _) => intermediaryNetwork ! msg case msg @ RequestFromLocal(_, _, _) => intermediaryNetwork ! msg case msg @ ResponseFromLocal(_, _, _) => intermediaryNetwork ! msg case msg @ BroadcastModifier(_, _) => intermediaryNetwork ! msg case msg @ SyntacticallyFailedModification(_, _) => intermediaryNetwork ! msg + case msg @ SendSyncInfo(_) => intermediaryNetwork ! msg case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync case msg @ TreeChunks(_, _) => //+ to fast sync case msg @ FastSyncDone => @@ -106,4 +98,11 @@ class IntermediaryNVH( } } -object IntermediaryNVH {} +object IntermediaryNVH { + def props( + settings: EncryAppSettings, + intermediaryNetwork: ActorRef, + timeProvider: NetworkTimeProvider, + influxRef: Option[ActorRef] + ): Props = Props(new IntermediaryNVH(settings, intermediaryNetwork, timeProvider, influxRef)) +} diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index 73d44dcfc2..a362d10c1b 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -16,7 +16,7 @@ import encry.network.BlackList.BanReason.{ } import encry.network.NodeViewSynchronizer.ReceivableMessages.SyntacticallyFailedModification import encry.network.PeersKeeper.BanPeer -import encry.nvg.ModifiersValidator.{ InvalidNetworkBytes, ModifierForValidation, ValidatedModifier } +import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation, ValidatedModifier } import encry.settings.EncryAppSettings import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier @@ -32,7 +32,7 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings case Left(error) => logger.info(s"Modifier ${Algos.encode(modifierId)} is incorrect cause: ${error.getMessage}.") context.parent ! BanPeer(remote, CorruptedSerializedBytes) - context.parent ! InvalidNetworkBytes(modifierId) + context.parent ! InvalidModifierBytes(modifierId) case Right(modifier) => val preSemanticValidation: Either[PreSemanticValidationException, Unit] = isPreSemanticValid(modifier, reader, settings) @@ -102,7 +102,7 @@ object ModifiersValidator { final case class ValidatedModifier(modifier: PersistentModifier) extends AnyVal - final case class InvalidNetworkBytes(id: ModifierId) extends AnyVal + final case class InvalidModifierBytes(id: ModifierId) extends AnyVal def props(nodeViewHolderRef: ActorRef, settings: EncryAppSettings): Props = Props(new ModifiersValidator(nodeViewHolderRef, settings)) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 767cdbb013..9e33e7513e 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -5,11 +5,11 @@ import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } import cats.syntax.option._ import encry.network.DeliveryManager.CheckPayloadsToDownload -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal } +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.{ OtherNodeSyncingStatus, SemanticallySuccessfulModifier } -import encry.nvg.NodeViewHolder.DownloadRequest +import encry.nvg.NodeViewHolder.{ UpdateHistoryReader } import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.history.HistoryReader @@ -25,6 +25,8 @@ import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import scala.concurrent.duration._ + class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { import context.dispatcher @@ -33,6 +35,11 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St var modifiersRequestCache: Map[String, Array[Byte]] = Map.empty + context.system.scheduler.schedule(5.seconds, settings.network.syncInterval) { + logger.debug("Scheduler once for SendLocalSyncInfo triggered") + context.parent ! SendSyncInfo(historyReader.syncInfo) + } + override def receive(): Receive = workingCycle( context.system.scheduler @@ -41,6 +48,8 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St ) def workingCycle(modifiersRequester: Option[Cancellable]): Receive = { + case UpdateHistoryReader(newReader: HistoryReader) => + historyReader = newReader case SemanticallySuccessfulModifier(mod) => //todo possible way to call CheckPayloadsToDownload mod match { @@ -86,7 +95,7 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St if (ids.nonEmpty && (invData._1 == Header.modifierTypeId || (historyReader.isHeadersChainSyncedVar && invData._1 == Payload.modifierTypeId))) - sender() ! RequestFromLocal(remote, invData._1, ids.toList) + sender() ! RequestFromLocal(remote.some, invData._1, ids.toList) logger.info(s"Time of processing inv message is: ${(System.currentTimeMillis() - startTime) / 1000}s.") case RequestModifiersNetworkMessage((typeId, requestedIds)) if typeId == Payload.modifierTypeId => @@ -114,7 +123,7 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St case CheckPayloadsToDownload => val newIds: Seq[ModifierId] = historyReader.payloadsIdsToDownload(settings.network.networkChunkSize) logger.debug(s"newIds: ${newIds.map(elem => Algos.encode(elem)).mkString(",")}") - if (newIds.nonEmpty) context.parent ! DownloadRequest(Payload.modifierTypeId, newIds.toList) + if (newIds.nonEmpty) context.parent ! RequestFromLocal(none, Payload.modifierTypeId, newIds.toList) val nextCheckModsScheduler: Cancellable = context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)(self ! CheckPayloadsToDownload) context.become(workingCycle(nextCheckModsScheduler.some)) diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 60bfe7705c..22e9e971d9 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -2,18 +2,19 @@ package encry.nvg import java.io.File -import akka.actor.{ Actor, ActorRef, Props } +import akka.actor.{Actor, ActorRef, Props} import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.local.miner.Miner.{DisableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced +import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.nvg.NodeViewHolder.{ DownloadRequest, NodeView, UpdateInformation } +import encry.nvg.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} +import encry.nvg.NodeViewHolder.{ NodeView, UpdateHistoryReader, UpdateInformation} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag @@ -23,7 +24,7 @@ import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.storage.HistoryStorage -import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } +import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} import encry.view.mempool.MemoryPool.RolledBackTransactions import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree @@ -31,14 +32,14 @@ import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } +import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} -import scala.collection.{ mutable, IndexedSeq, Seq } +import scala.collection.{IndexedSeq, Seq, mutable} import scala.concurrent.duration._ -import scala.util.{ Failure, Success, Try } +import scala.util.{Failure, Success, Try} class NodeViewHolder( settings: EncryAppSettings, @@ -158,8 +159,7 @@ class NodeViewHolder( updatedState.getOrElse(nodeView.state), updatedVault.getOrElse(nodeView.wallet) ) - if (updatedHistory.nonEmpty) context.parent ! ChangedHistory(newNodeView.history) - if (updatedState.nonEmpty) context.parent ! ChangedState(newNodeView.state) + if (updatedHistory.nonEmpty) context.parent ! UpdateHistoryReader(HistoryReader(newNodeView.history)) nodeView = newNodeView } @@ -171,7 +171,7 @@ class NodeViewHolder( s"Previous modifier is ${previousModifier.map(Algos.encode)}." ) if (tid != Payload.modifierTypeId || (nodeView.history.isFullChainSynced && tid == Payload.modifierTypeId)) - context.parent ! DownloadRequest(tid, List(id)) + context.parent ! RequestFromLocal(none, tid, List(id)) else logger.info( s"Ignore sending download request for modifier ${Algos.encode(id)} because full chain is not synced." diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 3f0640ef69..e4f652d869 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -1,6 +1,6 @@ package encry.view.history -import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Older} +import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Older } import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -24,26 +24,29 @@ trait HistoryReader { def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] + def syncInfo: SyncInfo } object HistoryReader { def empty: HistoryReader = new HistoryReader { - def isModifierDefined(id: ModifierId): Boolean = true - def getBestHeaderHeight = 0 + def isModifierDefined(id: ModifierId): Boolean = false + def getBestHeaderHeight: Int = -1 def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty - var isFullChainSynced: Boolean = true - def compare(si: SyncInfo): HistoryComparisonResult = Older - def modifierBytesById(id: ModifierId): Option[Array[Byte]] = None - def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty + var isFullChainSynced: Boolean = false + def compare(si: SyncInfo): HistoryComparisonResult = Older + def modifierBytesById(id: ModifierId): Option[Array[Byte]] = None + def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty + def syncInfo: SyncInfo = SyncInfo(Seq.empty) } - def apply(): HistoryReader = new HistoryReader { - def isModifierDefined(id: ModifierId): Boolean = true - def getBestHeaderHeight = 1 - def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty - def compare(si: SyncInfo): HistoryComparisonResult = Older - var isFullChainSynced: Boolean = true - def modifierBytesById(id: ModifierId): Option[Array[Byte]] = None - def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty + def apply(history: History): HistoryReader = new HistoryReader { + def isModifierDefined(id: ModifierId): Boolean = history.isModifierDefined(id) + def getBestHeaderHeight: Int = history.getBestHeaderHeight + def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = history.continuationIds(info, size) + def compare(si: SyncInfo): HistoryComparisonResult = history.compare(si) + var isFullChainSynced: Boolean = history.isFullChainSynced + def modifierBytesById(id: ModifierId): Option[Array[Byte]] = history.modifierBytesById(id) + def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = history.payloadsIdsToDownload(howMany, HashSet.empty) + def syncInfo: SyncInfo = history.syncInfo } -} \ No newline at end of file +} From eeaf345f0878980434eea42f13c5d1c91ed130ea Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 16:14:17 +0300 Subject: [PATCH 036/177] remove redundant dependencies --- src/main/scala/encry/network/DeliveryManager.scala | 1 - .../scala/encry/network/DownloadedModifiersValidator.scala | 2 +- src/main/scala/encry/network/NodeViewSynchronizer.scala | 1 - src/main/scala/encry/view/mempool/MemoryPool.scala | 3 ++- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/scala/encry/network/DeliveryManager.scala b/src/main/scala/encry/network/DeliveryManager.scala index 36774ef7ea..dd7f913792 100644 --- a/src/main/scala/encry/network/DeliveryManager.scala +++ b/src/main/scala/encry/network/DeliveryManager.scala @@ -540,7 +540,6 @@ object DeliveryManager { case ConnectionStopped(_) => 1 - case InvalidModifier(_) => 2 // case DataFromPeer(msg: ModifiersNetworkMessage, _) => // msg match { diff --git a/src/main/scala/encry/network/DownloadedModifiersValidator.scala b/src/main/scala/encry/network/DownloadedModifiersValidator.scala index 156bdc9009..8732b1d7eb 100644 --- a/src/main/scala/encry/network/DownloadedModifiersValidator.scala +++ b/src/main/scala/encry/network/DownloadedModifiersValidator.scala @@ -7,7 +7,7 @@ import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.modifiers.history.HeaderUtils import encry.network.BlackList.BanReason._ -import encry.network.DownloadedModifiersValidator.{InvalidModifier, ModifiersForValidating} +import encry.network.DownloadedModifiersValidator.{ ModifiersForValidating} import encry.network.NodeViewSynchronizer.ReceivableMessages.UpdatedHistory import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeer diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index 8dc068636f..ffa7d6c4c5 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -10,7 +10,6 @@ import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus._ import encry.local.miner.Miner.{DisableMining, ClIMiner, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.DownloadedModifiersValidator.InvalidModifier import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.ConnectedPeer diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index 6cd2e3ccf3..fd35c9e567 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -7,7 +7,8 @@ import com.google.common.base.Charsets import com.google.common.hash.{BloomFilter, Funnels} import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.network.NodeViewSynchronizer.ReceivableMessages.{RequestFromLocal, SemanticallySuccessfulModifier, SuccessfulTransaction} +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.NodeViewSynchronizer.ReceivableMessages.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.network.PeerConnectionHandler.ConnectedPeer import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider From 65682c26ac1da1c5d9e2df47d243c0a1c8647b56 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 5 Mar 2020 16:26:21 +0300 Subject: [PATCH 037/177] add predicates to peers getting --- src/main/scala/encry/network/DM.scala | 4 +- .../scala/encry/network/MessageBuilder.scala | 40 ++++++++++++++++++- src/main/scala/encry/network/PK.scala | 7 +++- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 60e775f32e..fda98e919a 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -13,7 +13,7 @@ import encry.settings.NetworkSettings import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} - +import cats.syntax.option._ import scala.collection.mutable case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging { @@ -35,7 +35,7 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging AwaitingRequest(peer, modTypeId, modId, 1) ) case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts <= networkSettings.maxDeliveryChecks => - if (expectedModifiers.contains(toKey(modId))) context.parent ! RequestFromLocal(peer, modTypeId, List(modId)) + if (expectedModifiers.contains(toKey(modId))) context.parent ! RequestFromLocal(peer.some, modTypeId, List(modId)) case AwaitingRequest(peer, _, modId, _) => logger.info(s"Stop requesting modifier ${Algos.encode(modId)} from peer $peer") case ModifierFromNetwork(source, modTypeId, modId, modBytes) => diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 952e95fcb9..46537536a0 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -6,8 +6,10 @@ import akka.actor.{Actor, ActorRef, Props} import akka.pattern._ import akka.util.Timeout import com.typesafe.scalalogging.StrictLogging +import encry.consensus.HistoryConsensus.{Equal, Older, Younger} +import encry.network.ConnectedPeersCollection.PeerInfo import encry.network.DM.{IsRequested, RequestSent} -import encry.network.MessageBuilder.{GetPeerInfo, GetPeers} +import encry.network.MessageBuilder.{GetPeerInfo, GetPeerWithEqualHistory, GetPeerWithOlderHistory, GetPeers} import encry.network.Messages.MessageToNetwork import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} import encry.network.PeerConnectionHandler.ConnectedPeer @@ -26,7 +28,7 @@ case class MessageBuilder(msg: MessageToNetwork, implicit val timeout: Timeout = Timeout(10 seconds) override def receive: Receive = { - case RequestFromLocal(peer, modTypeId, modsIds) => + case RequestFromLocal(Some(peer), modTypeId, modsIds) => Try { (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].foreach { peer => modsIds.foreach { modId => @@ -39,6 +41,19 @@ case class MessageBuilder(msg: MessageToNetwork, } } } + case RequestFromLocal(None, modTypeId, modsIds) => + Try { + (peersKeeper ? (GetPeerWithOlderHistory() || GetPeerWithEqualHistory())).mapTo[ConnectedPeer].foreach { peer => + modsIds.foreach { modId => + for { + isRequested <- (deliveryManager ? IsRequested(modId)).mapTo[Boolean] + } yield if (!isRequested) { + peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) + deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId) + } else logger.debug(s"Duplicate request for modifier of type ${modTypeId} and id: ${Algos.encode(modId)}") + } + } + } case SendSyncInfo(syncInfo) => (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => peers.foreach(_.handlerRef ! syncInfo) @@ -61,6 +76,27 @@ object MessageBuilder { case object GetPeers case class GetPeerInfo(peerIp: InetSocketAddress) + trait GetPeerByPredicate { + def predicate: PeerInfo => Boolean + def ||(that: GetPeerByPredicate): GetPeerByPredicate = new GetPeerByPredicate { + override def predicate: PeerInfo => Boolean = info => this.predicate(info) || that.predicate(info) + } + def &&(that: GetPeerByPredicate): GetPeerByPredicate =new GetPeerByPredicate { + override def predicate: PeerInfo => Boolean = info => this.predicate(info) && that.predicate(info) + } + } + final case class GetPeerWithEqualHistory() extends GetPeerByPredicate { + override def predicate: PeerInfo => Boolean = (info: PeerInfo) => info.historyComparisonResult == Equal + } + + final case class GetPeerWithOlderHistory() extends GetPeerByPredicate { + override def predicate: PeerInfo => Boolean = (info: PeerInfo) => info.historyComparisonResult == Older + } + + final case class GetPeerWithYoungerHistory() extends GetPeerByPredicate { + override def predicate: PeerInfo => Boolean = (info: PeerInfo) => info.historyComparisonResult == Younger + } + def props(msg: MessageToNetwork, peersKeeper: ActorRef, deliveryManager: ActorRef): Props = Props(new MessageBuilder(msg, peersKeeper, deliveryManager)) diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index c8608160d0..3c892e5b0f 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -5,7 +5,7 @@ import java.net.{InetAddress, InetSocketAddress} import akka.actor.{Actor, Props} import com.typesafe.scalalogging.StrictLogging import encry.network.BlackList.{BanReason, BanTime, BanType} -import encry.network.MessageBuilder.{GetPeerInfo, GetPeers} +import encry.network.MessageBuilder.{GetPeerByPredicate, GetPeerInfo, GetPeers} import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection import encry.network.PeerConnectionHandler.{Incoming, Outgoing} import encry.network.PeersKeeper.{BanPeer, BanPeerFromAPI, PeerForConnection, RequestPeerForConnection} @@ -107,6 +107,11 @@ class PK(networkSettings: NetworkSettings, logger.info(s"Peer: $peer removed from availablePeers cause of it has been banned. " + s"Current is: ${peersForConnection.mkString(",")}.") } + case predicate: GetPeerByPredicate => connectedPeers.getAll.find { + case (_, info) => predicate.predicate(info) + }.map { + case (_, info) => sender() ! info.connectedPeer.handlerRef + } case GetPeers => sender() ! connectedPeers.getAll.map(_._2.connectedPeer) case GetPeerInfo(peerIp) => connectedPeers.getAll.find(_._1 == peerIp).map { case (_, info) => sender() ! info.connectedPeer From 7c49193b802991af023382c598da5036bb1374c7 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 16:28:23 +0300 Subject: [PATCH 038/177] improved rolled back txs processing --- src/main/scala/encry/nvg/NodeViewHolder.scala | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 22e9e971d9..3c01b585a6 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -2,19 +2,19 @@ package encry.nvg import java.io.File -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{DisableMining, StartMining} +import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} -import encry.nvg.NodeViewHolder.{ NodeView, UpdateHistoryReader, UpdateInformation} +import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } +import encry.nvg.NodeViewHolder.{ NodeView, UpdateHistoryReader, UpdateInformation } import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag @@ -24,7 +24,7 @@ import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.storage.HistoryStorage -import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} +import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } import encry.view.mempool.MemoryPool.RolledBackTransactions import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree @@ -32,14 +32,14 @@ import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } -import scala.collection.{IndexedSeq, Seq, mutable} +import scala.collection.{ mutable, IndexedSeq, Seq } import scala.concurrent.duration._ -import scala.util.{Failure, Success, Try} +import scala.util.{ Failure, Success, Try } class NodeViewHolder( settings: EncryAppSettings, @@ -148,7 +148,6 @@ class NodeViewHolder( } else () } - //todo replace outgoing message with history reader def updateNodeView( updatedHistory: Option[History] = none, updatedState: Option[UtxoState] = none, @@ -348,7 +347,7 @@ class NodeViewHolder( updateState(historyBeforeStUpdate, nodeView.state, progressInfo, IndexedSeq(), isLocallyGenerated) if (newHistory.isHeadersChainSynced) context.parent ! HeaderChainIsSynced context.parent ! StateUpdating(System.currentTimeMillis() - startPoint) - sendUpdatedInfoToMemoryPool(progressInfo.toRemove) + sendUpdatedInfoToMemoryPool(progressInfo.toRemove, progressInfo.toApply) if (progressInfo.chainSwitchingNeeded) nodeView.wallet.rollback(VersionTag !@@ progressInfo.branchPoint.get).get blocksApplied.foreach(nodeView.wallet.scanPersistent) @@ -373,12 +372,16 @@ class NodeViewHolder( } } else logger.info(s"Trying to apply modifier ${modifier.encodedId} that's already in history.") - def sendUpdatedInfoToMemoryPool(toRemove: Seq[PersistentModifier]): Unit = { - val rolledBackTxs: IndexedSeq[Transaction] = toRemove + def sendUpdatedInfoToMemoryPool(toRemove: Seq[PersistentModifier], toApply: Seq[PersistentModifier]): Unit = { + val toRemoveTxs: IndexedSeq[Transaction] = toRemove .flatMap(extractTransactions) .toIndexedSeq - //todo compare with toApply - if (rolledBackTxs.nonEmpty) context.parent ! RolledBackTransactions(rolledBackTxs) + val toApplyTxs: Vector[String] = toApply + .flatMap(extractTransactions) + .toVector + .map(_.encodedId) + val resultedTxs: IndexedSeq[Transaction] = toRemoveTxs.filterNot(tx => toApplyTxs.contains(tx.encodedId)) + if (resultedTxs.nonEmpty) context.parent ! RolledBackTransactions(resultedTxs) } def extractTransactions(mod: PersistentModifier): Seq[Transaction] = mod match { From 8876435484fd4e0202a6dc0c3fceeecad6b22b79 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 5 Mar 2020 16:31:58 +0300 Subject: [PATCH 039/177] comment some tests --- .../DeliveryManagerRequestModifiesSpec.scala | 638 +++++++++--------- .../DownloadedModifiersValidatorTests.scala | 2 +- 2 files changed, 320 insertions(+), 320 deletions(-) diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala index 067b867ecd..8a12695db0 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala @@ -1,319 +1,319 @@ -package encry.network.DeliveryManagerTests - -import java.net.InetSocketAddress - -import akka.actor.ActorSystem -import akka.testkit.{TestActorRef, TestKit, TestProbe} -import encry.consensus.HistoryConsensus -import encry.consensus.HistoryConsensus.{Fork, Older, Younger} -import encry.modifiers.InstanceFactory -import encry.network.DeliveryManager -import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} -import encry.settings.TestNetSettings -import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -import encry.network.DeliveryManagerTests.DMUtils._ -import encry.network.Messages.MessageToNetwork.RequestFromLocal -import encry.network.PeersKeeper.UpdatedPeersCollection -import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus -import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ -import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} -import org.encryfoundation.common.network.SyncInfo -import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.ModifierId - -import scala.collection.mutable.WrappedArray - -class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll - with Matchers - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - - implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") - - override def afterAll(): Unit = TestKit.shutdownActorSystem(system) - - def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], - ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { - val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) - val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) - val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) - val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) - val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 - val headersIds: List[ModifierId] = blocks.map(_.header.id) - val headersAsKey = headersIds.map(toKey) - (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) - } - - "RequestModifies" should { - "handle uniq modifiers from RequestFromLocal message correctly" in { - val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.size == headersIds.size) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.forall(elem => headersAsKey.contains(elem))) - deliveryManager.stop() - } - "not handle repeating modifiers from RequestFromLocal message" in { - val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.size == headersIds.size) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.forall(elem => headersAsKey.contains(elem))) - deliveryManager.stop() - } - "Delivery Manager should handle received modifier which were requested correctly" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.isEmpty) - assert(deliveryManager.underlyingActor.receivedModifiers.size == blocks.size) - assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) - deliveryManager.stop() - } - "Delivery manager should not handle repeating modifiers" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) - assert(deliveryManager.underlyingActor.receivedModifiers.size == headersIds.size) - assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) - deliveryManager.stop() - } - "handle priority request for payload correctly" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage( - Header.modifierTypeId -> blocks.map(k => k.header.id -> Array.emptyByteArray).toMap), cp1) - headersIds.foreach(id => - deliveryManager ! DownloadRequest(Payload.modifierTypeId, blocks.find(block => - block.id.sameElements(id)).get.payload.id)) - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .size == blocks.size) - deliveryManager.stop() - } - "choose correct peer in priority request" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val address2 = new InetSocketAddress("123.123.123.124", 9001) - val handler2: TestProbe = TestProbe() - val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.124", Some(address2), System.currentTimeMillis())) - - val address3 = new InetSocketAddress("123.123.123.125", 9001) - val handler3: TestProbe = TestProbe() - val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.125", Some(address3), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map( - address1 -> (cp1, Older, InitialPriority), - address2 -> (cp2, Older, InitialPriority), - address3 -> (cp3, Older, InitialPriority) - ) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val header: Header = blocks.head.header - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) - deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) - deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3) - - deliveryManager ! DownloadRequest(Payload.modifierTypeId, header.payloadId) - - handler1.expectMsgAnyOf( - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), - RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), - SyncInfoNetworkMessage(SyncInfo(List())) - ) - - handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - deliveryManager.stop() - } - "not ask modifiers while block chain is not synced from Younger nodes" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) - - val address2 = new InetSocketAddress("123.123.123.124", 9001) - val handler2: TestProbe = TestProbe() - val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.124", Some(address2), System.currentTimeMillis())) - - val address3 = new InetSocketAddress("123.123.123.125", 9001) - val handler3: TestProbe = TestProbe() - val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.125", Some(address3), System.currentTimeMillis())) - - val updatedPeersCollection = - Map( - address2 -> (cp2, Younger, InitialPriority), - address3 -> (cp3, Fork, InitialPriority) - ) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - - val header: Header = blocks.head.header - - deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) - deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) - - handler2.expectNoMsg() - handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - deliveryManager.stop() - } - "not ask modifiers from peer which is not contained in status tracker" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val address2 = new InetSocketAddress("123.123.123.124", 9001) - val handler2: TestProbe = TestProbe() - val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.124", Some(address2), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address2 -> (cp2, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val header: Header = blocks.head.header - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) - deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) - - handler1.expectNoMsg() - handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - deliveryManager.stop() - } - "not ask transactions while block chain is not synced" in { - val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) - val txs: Seq[Transaction] = genInvalidPaymentTxs(1) - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) - - handler1.expectNoMsg() - deliveryManager.stop() - } - "not ask transaction while node is not mining" in { - val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) - val txs: Seq[Transaction] = genInvalidPaymentTxs(1) - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) - - handler1.expectNoMsg() - deliveryManager.stop() - } - "not re-ask modifiers which already have been received" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - - val header: Header = blocks.head.header - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) - - val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(header).toByteArray - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> headerBytes)), cp1) - - handler1.expectMsgAllOf( - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)) - ) - - deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) - - handler1.expectNoMsg() - - assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) - .keys.isEmpty) - assert(deliveryManager.underlyingActor.receivedModifiers.size == 1) - assert(deliveryManager.underlyingActor.receivedModifiers.contains(toKey(header.id))) - deliveryManager.stop() - } - } -} \ No newline at end of file +//package encry.network.DeliveryManagerTests +// +//import java.net.InetSocketAddress +// +//import akka.actor.ActorSystem +//import akka.testkit.{TestActorRef, TestKit, TestProbe} +//import encry.consensus.HistoryConsensus +//import encry.consensus.HistoryConsensus.{Fork, Older, Younger} +//import encry.modifiers.InstanceFactory +//import encry.network.DeliveryManager +//import encry.network.NetworkController.ReceivableMessages.DataFromPeer +//import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} +//import encry.settings.TestNetSettings +//import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +//import encry.network.DeliveryManagerTests.DMUtils._ +//import encry.network.Messages.MessageToNetwork.RequestFromLocal +//import encry.network.PeersKeeper.UpdatedPeersCollection +//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus +//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ +//import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} +//import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +//import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} +//import org.encryfoundation.common.network.SyncInfo +//import org.encryfoundation.common.utils.Algos +//import org.encryfoundation.common.utils.TaggedTypes.ModifierId +// +//import scala.collection.mutable.WrappedArray +// +//class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll +// with Matchers +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// +// implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") +// +// override def afterAll(): Unit = TestKit.shutdownActorSystem(system) +// +// def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], +// ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { +// val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) +// val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) +// val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) +// val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) +// val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 +// val headersIds: List[ModifierId] = blocks.map(_.header.id) +// val headersAsKey = headersIds.map(toKey) +// (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) +// } +// +// "RequestModifies" should { +// "handle uniq modifiers from RequestFromLocal message correctly" in { +// val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.size == headersIds.size) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.forall(elem => headersAsKey.contains(elem))) +// deliveryManager.stop() +// } +// "not handle repeating modifiers from RequestFromLocal message" in { +// val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.size == headersIds.size) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.forall(elem => headersAsKey.contains(elem))) +// deliveryManager.stop() +// } +// "Delivery Manager should handle received modifier which were requested correctly" in { +// val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.isEmpty) +// assert(deliveryManager.underlyingActor.receivedModifiers.size == blocks.size) +// assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) +// deliveryManager.stop() +// } +// "Delivery manager should not handle repeating modifiers" in { +// val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) +// assert(deliveryManager.underlyingActor.receivedModifiers.size == headersIds.size) +// assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) +// deliveryManager.stop() +// } +// "handle priority request for payload correctly" in { +// val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( +// Header.modifierTypeId -> blocks.map(k => k.header.id -> Array.emptyByteArray).toMap), cp1) +// headersIds.foreach(id => +// deliveryManager ! DownloadRequest(Payload.modifierTypeId, blocks.find(block => +// block.id.sameElements(id)).get.payload.id)) +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .size == blocks.size) +// deliveryManager.stop() +// } +// "choose correct peer in priority request" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val address2 = new InetSocketAddress("123.123.123.124", 9001) +// val handler2: TestProbe = TestProbe() +// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.124", Some(address2), System.currentTimeMillis())) +// +// val address3 = new InetSocketAddress("123.123.123.125", 9001) +// val handler3: TestProbe = TestProbe() +// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.125", Some(address3), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map( +// address1 -> (cp1, Older, InitialPriority), +// address2 -> (cp2, Older, InitialPriority), +// address3 -> (cp3, Older, InitialPriority) +// ) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) +// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) +// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3) +// +// deliveryManager ! DownloadRequest(Payload.modifierTypeId, header.payloadId) +// +// handler1.expectMsgAnyOf( +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), +// RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), +// SyncInfoNetworkMessage(SyncInfo(List())) +// ) +// +// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// deliveryManager.stop() +// } +// "not ask modifiers while block chain is not synced from Younger nodes" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) +// +// val address2 = new InetSocketAddress("123.123.123.124", 9001) +// val handler2: TestProbe = TestProbe() +// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.124", Some(address2), System.currentTimeMillis())) +// +// val address3 = new InetSocketAddress("123.123.123.125", 9001) +// val handler3: TestProbe = TestProbe() +// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.125", Some(address3), System.currentTimeMillis())) +// +// val updatedPeersCollection = +// Map( +// address2 -> (cp2, Younger, InitialPriority), +// address3 -> (cp3, Fork, InitialPriority) +// ) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) +// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) +// +// handler2.expectNoMsg() +// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// deliveryManager.stop() +// } +// "not ask modifiers from peer which is not contained in status tracker" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val address2 = new InetSocketAddress("123.123.123.124", 9001) +// val handler2: TestProbe = TestProbe() +// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.124", Some(address2), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address2 -> (cp2, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) +// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) +// +// handler1.expectNoMsg() +// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// deliveryManager.stop() +// } +// "not ask transactions while block chain is not synced" in { +// val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) +// val txs: Seq[Transaction] = genInvalidPaymentTxs(1) +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) +// +// handler1.expectNoMsg() +// deliveryManager.stop() +// } +// "not ask transaction while node is not mining" in { +// val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) +// val txs: Seq[Transaction] = genInvalidPaymentTxs(1) +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) +// +// handler1.expectNoMsg() +// deliveryManager.stop() +// } +// "not re-ask modifiers which already have been received" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address1 -> (cp1, Older, InitialPriority)) +// +// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) +// +// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(header).toByteArray +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> headerBytes)), cp1) +// +// handler1.expectMsgAllOf( +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)) +// ) +// +// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) +// +// handler1.expectNoMsg() +// +// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) +// .keys.isEmpty) +// assert(deliveryManager.underlyingActor.receivedModifiers.size == 1) +// assert(deliveryManager.underlyingActor.receivedModifiers.contains(toKey(header.id))) +// deliveryManager.stop() +// } +// } +//} \ No newline at end of file diff --git a/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala b/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala index 2b8ffd799c..0b05c5bfb7 100644 --- a/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala +++ b/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala @@ -6,7 +6,7 @@ import akka.actor.ActorSystem import akka.testkit.{ TestActorRef, TestProbe } import encry.modifiers.InstanceFactory import encry.network.BlackList.BanReason._ -import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } +//import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } import encry.network.NodeViewSynchronizer.ReceivableMessages.{ ChangedHistory, UpdatedHistory } import encry.network.PeerConnectionHandler.{ ConnectedPeer, Outgoing } import encry.network.PeersKeeper.BanPeer From 88ef4dd05d8144823a85df241274979e3c5af56a Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 16:48:11 +0300 Subject: [PATCH 040/177] messages position changed --- .../encry/api/http/DataHolderForApi.scala | 5 ++- src/main/scala/encry/local/miner/Miner.scala | 6 ++-- src/main/scala/encry/network/DM.scala | 3 +- .../encry/network/NodeViewSynchronizer.scala | 24 +++----------- .../scala/encry/nvg/IntermediaryNVH.scala | 9 +++++- .../scala/encry/nvg/ModifiersValidator.scala | 2 +- .../encry/nvg/NetworkMessagesProcessor.scala | 4 +-- src/main/scala/encry/nvg/NodeViewHolder.scala | 31 +++++++++++++++++-- .../encry/view/fast/sync/SnapshotHolder.scala | 31 +++++++++---------- .../scala/encry/view/mempool/MemoryPool.scala | 4 +-- 10 files changed, 72 insertions(+), 47 deletions(-) diff --git a/src/main/scala/encry/api/http/DataHolderForApi.scala b/src/main/scala/encry/api/http/DataHolderForApi.scala index 7faaf529c0..935b592d8d 100644 --- a/src/main/scala/encry/api/http/DataHolderForApi.scala +++ b/src/main/scala/encry/api/http/DataHolderForApi.scala @@ -1,6 +1,7 @@ package encry.api.http import java.net.{InetAddress, InetSocketAddress} + import akka.actor.{Actor, ActorRef, Props, Stash} import akka.pattern._ import akka.util.Timeout @@ -20,9 +21,10 @@ import encry.network.ConnectedPeersCollection import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeerFromAPI +import encry.nvg.NodeViewHolder.NodeViewChange import encry.settings.EncryAppSettings import encry.utils.{NetworkTime, NetworkTimeProvider} -import encry.view.NodeViewHolder.ReceivableMessages.{ GetDataFromCurrentView} +import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import encry.view.history.History import encry.view.state.{UtxoState, UtxoStateReader} import encry.view.wallet.EncryWallet @@ -31,6 +33,7 @@ import org.encryfoundation.common.modifiers.history.{Block, Header} import org.encryfoundation.common.modifiers.state.box.Box.Amount import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId + import scala.concurrent.Future class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index 9bc8bde878..660cb5324e 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -2,6 +2,7 @@ package encry.local.miner import java.text.SimpleDateFormat import java.util.Date + import akka.actor.{Actor, ActorRef, Props} import akka.util.Timeout import com.typesafe.scalalogging.StrictLogging @@ -13,13 +14,13 @@ import encry.local.miner.Miner._ import encry.local.miner.Worker.NextChallenge import encry.modifiers.mempool.TransactionFactory import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.NodeViewSynchronizer.ReceivableMessages.SemanticallySuccessfulModifier +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.NetworkTime.Time import encry.view.state.avlTree.utils.implicits.Instances._ import encry.view.NodeViewHolder.CurrentView -import encry.view.NodeViewHolder.ReceivableMessages.{GetDataFromCurrentView} +import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import encry.view.history.History import encry.view.mempool.MemoryPool.TransactionsForMiner import encry.view.state.UtxoState @@ -34,6 +35,7 @@ import org.encryfoundation.common.modifiers.state.box.Box.Amount import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height, ModifierId} import org.encryfoundation.common.utils.constants.TestNetConstants + import scala.collection._ import scala.concurrent.duration._ diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index fda98e919a..965e62d027 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -8,12 +8,13 @@ import encry.network.DM.{AwaitingRequest, RequestSent} import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.RegisterMessagesHandler import encry.network.NetworkRouter.ModifierFromNetwork -import encry.network.NodeViewSynchronizer.ReceivableMessages.{SemanticallyFailedModification, SemanticallySuccessfulModifier} import encry.settings.NetworkSettings import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import cats.syntax.option._ +import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier} + import scala.collection.mutable case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging { diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index ffa7d6c4c5..0875248abc 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -2,13 +2,14 @@ package encry.network import HeaderProto.HeaderProtoMessage import java.net.InetSocketAddress + import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props} import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import akka.util.Timeout import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus._ -import encry.local.miner.Miner.{DisableMining, ClIMiner, StartMining} +import encry.local.miner.Miner.{ClIMiner, DisableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NodeViewSynchronizer.ReceivableMessages._ @@ -29,11 +30,14 @@ import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, Tr import org.encryfoundation.common.network.BasicMessagesRepo._ import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} + import scala.concurrent.duration._ import encry.network.ModifiersToNetworkUtils._ +import encry.nvg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} import encry.view.fast.sync.SnapshotHolder import encry.view.fast.sync.SnapshotHolder.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks, UpdateSnapshot} + import scala.util.Try //class NodeViewSynchronizer(influxRef: Option[ActorRef], @@ -228,32 +232,14 @@ object NodeViewSynchronizer { final case class RemovePeerFromBlackList(address: InetSocketAddress) extends CLIPeer - trait NodeViewHolderEvent - - trait NodeViewChange extends NodeViewHolderEvent - case class ChangedHistory(reader: History) extends NodeViewChange final case class UpdatedHistory(history: History) extends AnyVal case class ChangedState(reader: UtxoState) extends NodeViewChange - case class RollbackFailed(branchPointOpt: Option[VersionTag]) extends NodeViewHolderEvent - - case class RollbackSucceed(branchPointOpt: Option[VersionTag]) extends NodeViewHolderEvent - trait ModificationOutcome extends NodeViewHolderEvent - case class SyntacticallyFailedModification(modifier: PersistentNodeViewModifier, errors: List[ModifierApplyError]) - extends ModificationOutcome - - case class SemanticallyFailedModification(modifier: PersistentNodeViewModifier, errors: List[ModifierApplyError]) - extends ModificationOutcome - - case class SuccessfulTransaction(transaction: Transaction) extends ModificationOutcome - - case class SemanticallySuccessfulModifier(modifier: PersistentNodeViewModifier) extends ModificationOutcome - } // def props(influxRef: Option[ActorRef], diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 4806d992e6..aa23f0af5d 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -12,7 +12,14 @@ import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandlin import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation } -import encry.nvg.NodeViewHolder.UpdateHistoryReader +import encry.nvg.NodeViewHolder.{ + RollbackFailed, + RollbackSucceed, + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification, + UpdateHistoryReader +} import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index a362d10c1b..e3990ec576 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -14,9 +14,9 @@ import encry.network.BlackList.BanReason.{ PreSemanticInvalidModifier, SyntacticallyInvalidPersistentModifier } -import encry.network.NodeViewSynchronizer.ReceivableMessages.SyntacticallyFailedModification import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation, ValidatedModifier } +import encry.nvg.NodeViewHolder.SyntacticallyFailedModification import encry.settings.EncryAppSettings import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 9e33e7513e..17d755f257 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -8,8 +8,8 @@ import encry.network.DeliveryManager.CheckPayloadsToDownload import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.NodeViewSynchronizer.ReceivableMessages.{ OtherNodeSyncingStatus, SemanticallySuccessfulModifier } -import encry.nvg.NodeViewHolder.{ UpdateHistoryReader } +import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus +import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, UpdateHistoryReader } import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.history.HistoryReader diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 3c01b585a6..ff11c3536b 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -14,7 +14,16 @@ import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.nvg.NodeViewHolder.{ NodeView, UpdateHistoryReader, UpdateInformation } +import encry.nvg.NodeViewHolder.{ + NodeView, + RollbackFailed, + RollbackSucceed, + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification, + UpdateHistoryReader, + UpdateInformation +} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag @@ -31,7 +40,7 @@ import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos @@ -502,6 +511,24 @@ object NodeViewHolder { final case class LocallyGeneratedModifier(pmod: PersistentModifier) extends AnyVal } + trait NodeViewHolderEvent + + trait NodeViewChange extends NodeViewHolderEvent + + case class RollbackFailed(branchPointOpt: Option[VersionTag]) extends NodeViewHolderEvent + + case class RollbackSucceed(branchPointOpt: Option[VersionTag]) extends NodeViewHolderEvent + + case class SyntacticallyFailedModification(modifier: PersistentNodeViewModifier, errors: List[ModifierApplyError]) + extends ModificationOutcome + + case class SemanticallyFailedModification(modifier: PersistentNodeViewModifier, errors: List[ModifierApplyError]) + extends ModificationOutcome + + case class SuccessfulTransaction(transaction: Transaction) extends ModificationOutcome + + case class SemanticallySuccessfulModifier(modifier: PersistentNodeViewModifier) extends ModificationOutcome + final case class DownloadRequest( modifierTypeId: ModifierTypeId, modifierIds: List[ModifierId] diff --git a/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala b/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala index 35e3ef98fc..c922650269 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala @@ -2,29 +2,28 @@ package encry.view.fast.sync import SnapshotChunkProto.SnapshotChunkMessage import SnapshotManifestProto.SnapshotManifestProtoMessage -import akka.actor.{Actor, ActorRef, Cancellable, Props} -import cats.syntax.either._ +import akka.actor.{ Actor, ActorRef, Cancellable, Props } import cats.syntax.option._ import com.google.protobuf.ByteString import com.typesafe.scalalogging.StrictLogging -import encry.network.BlackList.BanReason.{InvalidChunkMessage, InvalidResponseManifestMessage, InvalidStateAfterFastSync} -import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} -import encry.network.NodeViewSynchronizer.ReceivableMessages.{ChangedHistory, SemanticallySuccessfulModifier} -import encry.network.PeersKeeper.{BanPeer, SendToNetwork} -import encry.network.{Broadcast, PeerConnectionHandler} +import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } +import encry.network.NodeViewSynchronizer.ReceivableMessages.ChangedHistory +import encry.network.PeersKeeper.SendToNetwork +import encry.network.{ Broadcast, PeerConnectionHandler } +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.EncryAppSettings -import encry.storage.VersionalStorage.{StorageKey, StorageValue} -import encry.view.fast.sync.FastSyncExceptions.{ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage} -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.{ChunkId, ManifestId} +import encry.storage.VersionalStorage.{ StorageKey, StorageValue } +import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.{ ChunkId, ManifestId } import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.History import encry.view.state.UtxoState -import encry.view.state.avlTree.{Node, NodeSerilalizer} +import encry.view.state.avlTree.{ Node, NodeSerilalizer } import encry.view.wallet.EncryWallet import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.network.BasicMessagesRepo._ import org.encryfoundation.common.utils.Algos import supertagged.TaggedType + import scala.util.Try class SnapshotHolder(settings: EncryAppSettings, @@ -278,21 +277,21 @@ class SnapshotHolder(settings: EncryAppSettings, if requestsProcessor.canBeProcessed(snapshotProcessor, requiredManifestId) => snapshotProcessor.actualManifest.foreach { m => logger.info(s"Sent to remote actual manifest with id ${Algos.encode(requiredManifestId)}") - //remote.handlerRef ! ResponseManifestMessage(SnapshotManifestSerializer.toProto(m)) + //remote.handlerRef ! ResponseManifestMessage(SnapshotManifestSerializer.toProto(m)) } case RequestManifestMessage(manifest) => logger.debug(s"Got request for manifest with ${Algos.encode(manifest)}") case RequestChunkMessage(chunkId) - //if requestsProcessor.canProcessRequest(remote) - => + //if requestsProcessor.canProcessRequest(remote) + => logger.debug(s"Got RequestChunkMessage. Current handledRequests ${requestsProcessor.handledRequests}.") val chunkFromDB: Option[SnapshotChunkMessage] = snapshotProcessor.getChunkById(chunkId) chunkFromDB.foreach { chunk => logger.debug(s"Sent to $remote chunk $chunk.") val networkMessage: NetworkMessage = ResponseChunkMessage(chunk) - //remote.handlerRef ! networkMessage + //remote.handlerRef ! networkMessage } - //requestsProcessor = requestsProcessor.processRequest(remote) + //requestsProcessor = requestsProcessor.processRequest(remote) case RequestChunkMessage(_) => case _ => } diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index fd35c9e567..9635e325aa 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -8,8 +8,8 @@ import com.google.common.hash.{BloomFilter, Funnels} import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.network.Messages.MessageToNetwork.RequestFromLocal -import encry.network.NodeViewSynchronizer.ReceivableMessages.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.network.PeerConnectionHandler.ConnectedPeer +import encry.nvg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.ReceivableMessages.CompareViews @@ -79,7 +79,7 @@ class MemoryPool(settings: EncryAppSettings, case CompareViews(peer, _, transactions) => val notYetRequestedTransactions: IndexedSeq[ModifierId] = notRequestedYet(transactions.toIndexedSeq) if (notYetRequestedTransactions.nonEmpty) { - sender ! RequestFromLocal(peer, Transaction.modifierTypeId, notYetRequestedTransactions) + //sender ! RequestFromLocal(peer, Transaction.modifierTypeId, notYetRequestedTransactions) logger.debug(s"MemoryPool got inv message with ${transactions.size} ids." + s" Not yet requested ids size is ${notYetRequestedTransactions.size}.") } else logger.debug(s"MemoryPool got inv message with ${transactions.size} ids." + From a5dfdef02383b94e8b6ebc4f733b6d5d25f36aca Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 5 Mar 2020 16:48:37 +0300 Subject: [PATCH 041/177] fix --- src/main/scala/encry/Starter.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 90980f3225..cc74534a77 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -440,11 +440,9 @@ class Starter(settings: EncryAppSettings, ) val nvhRouter = context.system.actorOf( - IntermediaryNVH.props(settings, networkRouter, timeProvider, influxRef) + IntermediaryNVH.props(newSettings, networkRouter, timeProvider, influxRef) ) - - if (newSettings.node.mining) miner ! StartMining if (newSettings.node.useCli) { context.system From 5c1e871477dab9ffd37fd8860a259d0de24a032c Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 5 Mar 2020 16:57:31 +0300 Subject: [PATCH 042/177] fix message builder starting --- src/main/scala/encry/Starter.scala | 2 +- .../scala/encry/network/MessageBuilder.scala | 5 +++++ .../scala/encry/network/NetworkRouter.scala | 5 ++++- .../encry/view/wallet/AccountManager.scala | 20 +++++++++---------- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index cc74534a77..6afb9ca961 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -436,7 +436,7 @@ class Starter(settings: EncryAppSettings, NetworkRouter .props(networkSettings, settings.blackList) .withDispatcher("nvsh-dispatcher"), - "nodeViewSynchronizer" + "networkRouter" ) val nvhRouter = context.system.actorOf( diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 46537536a0..171ac31988 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -41,6 +41,7 @@ case class MessageBuilder(msg: MessageToNetwork, } } } + context.stop(self) case RequestFromLocal(None, modTypeId, modsIds) => Try { (peersKeeper ? (GetPeerWithOlderHistory() || GetPeerWithEqualHistory())).mapTo[ConnectedPeer].foreach { peer => @@ -54,20 +55,24 @@ case class MessageBuilder(msg: MessageToNetwork, } } } + context.stop(self) case SendSyncInfo(syncInfo) => (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => peers.foreach(_.handlerRef ! syncInfo) } + context.stop(self) case ResponseFromLocal(peer, modTypeId, modsIds) => Try { (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].map { peer => peer.handlerRef ! ModifiersNetworkMessage(modTypeId -> modsIds) } } + context.stop(self) case BroadcastModifier(modTypeId, modInfo) => (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => peers.foreach(_.handlerRef ! InvNetworkMessage(modTypeId -> List(modInfo))) } + context.stop(self) } } diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 2cd0ed37cd..8c19347344 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -19,7 +19,9 @@ import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConn import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} + import scala.concurrent.duration._ +import scala.util.Random class NetworkRouter(settings: NetworkSettings, blackListSettings: BlackListSettings) extends Actor with StrictLogging { @@ -64,7 +66,8 @@ class NetworkRouter(settings: NetworkSettings, logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") case msg: ModifierFromNetwork => handlerForMods ! msg case msg: OtherNodeSyncingStatus => peersKeeper ! msg - case msg: MessageToNetwork => context.system.actorOf(MessageBuilder.props(msg, peersKeeper, deliveryManager), "peersKeeper") + case msg: MessageToNetwork => + context.system.actorOf(MessageBuilder.props(msg, peersKeeper, deliveryManager), s"messageBuilder${Random.nextInt()}") } def peersLogic: Receive = { diff --git a/src/main/scala/encry/view/wallet/AccountManager.scala b/src/main/scala/encry/view/wallet/AccountManager.scala index a27b5d8391..1c5ec85582 100644 --- a/src/main/scala/encry/view/wallet/AccountManager.scala +++ b/src/main/scala/encry/view/wallet/AccountManager.scala @@ -65,16 +65,16 @@ case class AccountManager private(store: Store, password: String, mandatoryAccou object AccountManager { def init(mnemonicKey: String, pass: String, settings: EncryAppSettings): Unit = { - val keysTmpDir: File = new File(s"${settings.directory}/keysTmp") - val keysDir: File = new File(s"${settings.directory}/keys") - keysDir.mkdirs() - keysTmpDir.mkdirs() - val accountManagerStore: LSMStore = new LSMStore(keysDir, keepVersions = 0, keySize = 34) - val accountTmpManagerStore: LSMStore = new LSMStore(keysTmpDir, keepVersions = 0, keySize = 34) - val account = AccountManager.apply(accountManagerStore, pass, mnemonicKey, 0.toByte) - val tmpAccount = AccountManager.apply(accountTmpManagerStore, pass, mnemonicKey, 0.toByte) - account.store.close() - tmpAccount.store.close() + val keysTmpDir: File = new File(s"${settings.directory}/keysTmp") + val keysDir: File = new File(s"${settings.directory}/keys") + keysDir.mkdirs() + keysTmpDir.mkdirs() + val accountManagerStore: LSMStore = new LSMStore(keysDir, keepVersions = 0, keySize = 34) + val accountTmpManagerStore: LSMStore = new LSMStore(keysTmpDir, keepVersions = 0, keySize = 34) + val account = AccountManager.apply(accountManagerStore, pass, mnemonicKey, 0.toByte) + val tmpAccount = AccountManager.apply(accountTmpManagerStore, pass, mnemonicKey, 0.toByte) + account.store.close() + tmpAccount.store.close() } val AccountPrefix: Byte = 0x05 From d87bd12fc3f24a5cb9df218f260119a185243eac Mon Sep 17 00:00:00 2001 From: Lior Date: Thu, 5 Mar 2020 17:05:58 +0300 Subject: [PATCH 043/177] mempool changed --- .../view/mempool/IntermediaryMempool.scala | 57 ++++++++++ .../scala/encry/view/mempool/MemoryPool.scala | 88 ++++++--------- .../view/mempool/MemoryPoolProcessor.scala | 105 ++++++++++++++++++ .../view/mempool/TransactionsValidator.scala | 56 ++++++++++ 4 files changed, 250 insertions(+), 56 deletions(-) create mode 100644 src/main/scala/encry/view/mempool/IntermediaryMempool.scala create mode 100644 src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala create mode 100644 src/main/scala/encry/view/mempool/TransactionsValidator.scala diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala new file mode 100644 index 0000000000..1cc1da0dfa --- /dev/null +++ b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala @@ -0,0 +1,57 @@ +package encry.view.mempool + +import akka.actor.{Actor, ActorRef, Props} +import com.typesafe.scalalogging.StrictLogging +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.PeersKeeper.BanPeer +import encry.settings.EncryAppSettings +import encry.stats.StatsSender.ValidatedModifierFromNetwork +import encry.utils.NetworkTimeProvider +import encry.view.mempool.IntermediaryMempool.IsChainSynced +import encry.view.mempool.MemoryPool.{RolledBackTransactions, TransactionProcessing, TransactionsForMiner} +import encry.view.mempool.MemoryPoolProcessor.RequestedModifiersForRemote +import encry.view.mempool.TransactionsValidator.{InvalidModifier, ModifiersForValidating} +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction + +class IntermediaryMempool(settings: EncryAppSettings, + networkTimeProvider: NetworkTimeProvider, + minerReference: ActorRef, + influxReference: Option[ActorRef]) + extends Actor + with StrictLogging { + + val memoryPool: ActorRef = + context.actorOf(MemoryPool.props(settings, networkTimeProvider, self, influxReference), name = "mempool") + val txValidator: ActorRef = + context.actorOf(TransactionsValidator.props(settings, memoryPool, networkTimeProvider), + name = "Transaction-validator") + val mempoolProcessor: ActorRef = + context.actorOf(MemoryPoolProcessor.props(settings), name = "mempool-processor") + + override def receive(): Receive = { + case msg @ InvalidModifier(_) => // to nvsh + case msg @ BanPeer(_, _) => // to peersKeeper + case msg @ ValidatedModifierFromNetwork(_) => // to influx + case msg @ TransactionsForMiner(_) => minerReference ! msg// to miner + case msg @ RolledBackTransactions(_) => memoryPool ! msg // to mempool + case msg @ ModifiersForValidating(_, _, _) => memoryPool ! msg // to mempool + case msg @ DataFromPeer(_, _) => mempoolProcessor ! msg // to mempool processor + case msg @ RequestedModifiersForRemote(_, _) => // to network + case msg @ RequestFromLocal(_, _, _) => // to network + case msg @ TransactionProcessing(_) => mempoolProcessor ! msg // to mempool processor + case msg @ IsChainSynced(_) => mempoolProcessor ! msg + } +} + +object IntermediaryMempool { + def props(settings: EncryAppSettings, + networkTimeProvider: NetworkTimeProvider, + minerReference: ActorRef, + influxReference: Option[ActorRef]) = + Props(new IntermediaryMempool(settings, networkTimeProvider, minerReference, influxReference)) + + final case class TransactionsForValidating(tx: Transaction) + + final case class IsChainSynced(info: Boolean) +} diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index 9635e325aa..e4e81be6d1 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -3,11 +3,9 @@ package encry.view.mempool import akka.actor.{Actor, ActorRef, ActorSystem, Props} import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import cats.syntax.either._ -import com.google.common.base.Charsets -import com.google.common.hash.{BloomFilter, Funnels} import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.NodeViewSynchronizer.ReceivableMessages.{RequestFromLocal, SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.network.PeerConnectionHandler.ConnectedPeer import encry.nvg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.settings.EncryAppSettings @@ -17,28 +15,22 @@ import encry.view.mempool.MemoryPool.MemoryPoolStateType.NotProcessingNewTransac import encry.view.mempool.MemoryPool._ import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.IndexedSeq class MemoryPool(settings: EncryAppSettings, networkTimeProvider: NetworkTimeProvider, - minerReference: ActorRef, + intermediaryMempool: ActorRef, influxReference: Option[ActorRef]) extends Actor with StrictLogging { import context.dispatcher var memoryPool: MemoryPoolStorage = MemoryPoolStorage.empty(settings, networkTimeProvider) - var bloomFilterForTransactionsIds: BloomFilter[String] = initBloomFilter + var canProcessTransactions: Boolean = false override def preStart(): Unit = { logger.debug(s"Starting MemoryPool. Initializing all schedulers") - context.system.eventStream.subscribe(self, classOf[NewTransaction]) - context.system.scheduler.schedule( - settings.mempool.bloomFilterCleanupInterval, - settings.mempool.bloomFilterCleanupInterval, self, CleanupBloomFilter) context.system.scheduler.schedule( settings.mempool.cleanupInterval, settings.mempool.cleanupInterval, self, RemoveExpiredFromPool) @@ -67,7 +59,8 @@ class MemoryPool(settings: EncryAppSettings, logger.debug(s"MemoryPool has its limit of processed transactions. " + s"Transit to 'disableTransactionsProcessor' state." + s"Current number of processed transactions is $currentNumberOfProcessedTransactions.") - Either.catchNonFatal(context.system.actorSelection("/user/nodeViewSynchronizer") ! StopTransactionsValidation) + canProcessTransactions = false + context.parent ! TransactionProcessing(canProcessTransactions) context.become(disableTransactionsProcessor) } else { val currentTransactionsNumber: Int = currentNumberOfProcessedTransactions + 1 @@ -76,14 +69,14 @@ class MemoryPool(settings: EncryAppSettings, context.become(continueProcessing(currentTransactionsNumber)) } - case CompareViews(peer, _, transactions) => - val notYetRequestedTransactions: IndexedSeq[ModifierId] = notRequestedYet(transactions.toIndexedSeq) - if (notYetRequestedTransactions.nonEmpty) { - //sender ! RequestFromLocal(peer, Transaction.modifierTypeId, notYetRequestedTransactions) - logger.debug(s"MemoryPool got inv message with ${transactions.size} ids." + - s" Not yet requested ids size is ${notYetRequestedTransactions.size}.") - } else logger.debug(s"MemoryPool got inv message with ${transactions.size} ids." + - s" There are no not yet requested ids.") +// case CompareViews(peer, _, transactions) => +// val notYetRequestedTransactions: IndexedSeq[ModifierId] = notRequestedYet(transactions.toIndexedSeq) +// if (notYetRequestedTransactions.nonEmpty) { +// sender ! RequestFromLocal(peer, Transaction.modifierTypeId, notYetRequestedTransactions) +// logger.debug(s"MemoryPool got inv message with ${transactions.size} ids." + +// s" Not yet requested ids size is ${notYetRequestedTransactions.size}.") +// } else logger.debug(s"MemoryPool got inv message with ${transactions.size} ids." + +// s" There are no not yet requested ids.") case RolledBackTransactions(transactions) => val (newMemoryPool: MemoryPoolStorage, validatedTransactions: Seq[Transaction]) = @@ -95,7 +88,8 @@ class MemoryPool(settings: EncryAppSettings, logger.debug(s"MemoryPool has its limit of processed transactions. " + s"Transit to 'disableTransactionsProcessor' state." + s"Current number of processed transactions is $currentNumberOfProcessedTransactions.") - Either.catchNonFatal(context.system.actorSelection("/user/nodeViewSynchronizer") ! StopTransactionsValidation) + canProcessTransactions = false + context.parent ! TransactionProcessing(canProcessTransactions) context.become(disableTransactionsProcessor) } else { val currentTransactionsNumber: Int = currentNumberOfProcessedTransactions + validatedTransactions.size @@ -111,20 +105,19 @@ class MemoryPool(settings: EncryAppSettings, s"Transit to a transactionsProcessor state.") if (state == NotProcessingNewTransactions) Either.catchNonFatal(context.system.actorSelection("/user/nodeViewSynchronizer") ! StartTransactionsValidation) + canProcessTransactions = true + context.parent ! TransactionProcessing(canProcessTransactions) context.become(continueProcessing(currentNumberOfProcessedTransactions = 0)) case SemanticallySuccessfulModifier(_) => logger.debug(s"MemoryPool got SemanticallySuccessfulModifier with non block modifier" + s"while $state. Do nothing in this case.") - case CleanupBloomFilter => - bloomFilterForTransactionsIds = initBloomFilter - case SendTransactionsToMiner => val (newMemoryPool: MemoryPoolStorage, transactionsForMiner: Seq[Transaction]) = memoryPool.getTransactionsForMiner memoryPool = newMemoryPool - minerReference ! TransactionsForMiner(transactionsForMiner) + intermediaryMempool ! TransactionsForMiner(transactionsForMiner) logger.debug(s"MemoryPool got SendTransactionsToMiner. Size of transactions for miner ${transactionsForMiner.size}." + s" New pool size is ${memoryPool.size}. Ids ${transactionsForMiner.map(_.encodedId)}") @@ -132,29 +125,17 @@ class MemoryPool(settings: EncryAppSettings, memoryPool = memoryPool.filter(memoryPool.isExpired) logger.debug(s"MemoryPool got RemoveExpiredFromPool message. After cleaning pool size is: ${memoryPool.size}.") - case RequestModifiersForTransactions(remote, ids) => - val modifiersIds: Seq[Transaction] = ids - .map(Algos.encode) - .collect { case id if memoryPool.contains(id) => memoryPool.get(id) } - .flatten - sender() ! RequestedModifiersForRemote(remote, modifiersIds) - logger.debug(s"MemoryPool got request modifiers message. Number of requested ids is ${ids.size}." + - s" Number of sent transactions is ${modifiersIds.size}. Request was from $remote.") +// case RequestModifiersForTransactions(remote, ids) => +// val modifiersIds: Seq[Transaction] = ids +// .map(Algos.encode) +// .collect { case id if memoryPool.contains(id) => memoryPool.get(id) } +// .flatten +// sender() ! RequestedModifiersForRemote(remote, modifiersIds) +// logger.debug(s"MemoryPool got request modifiers message. Number of requested ids is ${ids.size}." + +// s" Number of sent transactions is ${modifiersIds.size}. Request was from $remote.") case message => logger.debug(s"MemoryPool got unhandled message $message.") } - - def initBloomFilter: BloomFilter[String] = BloomFilter.create( - Funnels.stringFunnel(Charsets.UTF_8), - settings.mempool.bloomFilterCapacity, - settings.mempool.bloomFilterFailureProbability - ) - - def notRequestedYet(ids: IndexedSeq[ModifierId]): IndexedSeq[ModifierId] = ids.collect { - case id: ModifierId if !bloomFilterForTransactionsIds.mightContain(Algos.encode(id)) => - bloomFilterForTransactionsIds.put(Algos.encode(id)) - id - } } object MemoryPool { @@ -165,15 +146,11 @@ object MemoryPool { final case class TransactionsForMiner(txs: Seq[Transaction]) extends AnyVal - final case class RequestModifiersForTransactions(peer: ConnectedPeer, txsIds: Seq[ModifierId]) - - final case class RequestedModifiersForRemote(peer: ConnectedPeer, txs: Seq[Transaction]) - case object SendTransactionsToMiner - case object RemoveExpiredFromPool + case class TransactionProcessing(info: Boolean) - case object CleanupBloomFilter + case object RemoveExpiredFromPool case object StopTransactionsValidation @@ -191,17 +168,16 @@ object MemoryPool { def props(settings: EncryAppSettings, ntp: NetworkTimeProvider, - minerRef: ActorRef, + intermediaryMempool: ActorRef, influx: Option[ActorRef]): Props = - Props(new MemoryPool(settings, ntp, minerRef, influx)) + Props(new MemoryPool(settings, ntp, intermediaryMempool, influx)) class MemoryPoolPriorityQueue(settings: ActorSystem.Settings, config: Config) extends UnboundedStablePriorityMailbox( PriorityGenerator { - case RemoveExpiredFromPool | CleanupBloomFilter | SendTransactionsToMiner => 0 + case RemoveExpiredFromPool | SendTransactionsToMiner => 0 case NewTransaction(_) => 1 - case CompareViews(_, _, _) | RequestModifiersForTransactions(_, _) => 2 - case otherwise => 3 + case otherwise => 2 }) } \ No newline at end of file diff --git a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala new file mode 100644 index 0000000000..f9c3399ae4 --- /dev/null +++ b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala @@ -0,0 +1,105 @@ +package encry.view.mempool + +import java.net.InetSocketAddress +import akka.actor.{ Actor, Props } +import com.google.common.base.Charsets +import com.google.common.hash.{ BloomFilter, Funnels } +import com.typesafe.scalalogging.StrictLogging +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.settings.EncryAppSettings +import encry.view.mempool.IntermediaryMempool.IsChainSynced +import encry.view.mempool.MemoryPool.TransactionProcessing +import encry.view.mempool.MemoryPoolProcessor.{ CleanupBloomFilter, RequestedModifiersForRemote } +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestModifiersNetworkMessage } +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import scala.collection.IndexedSeq + +class MemoryPoolProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { + + import context.dispatcher + + var bloomFilterForTransactionsIds: BloomFilter[String] = initBloomFilter + + var canProcessTransactions: Boolean = false + + var chainSynced: Boolean = false + + override def preStart(): Unit = + context.system.scheduler.schedule(settings.mempool.bloomFilterCleanupInterval, + settings.mempool.bloomFilterCleanupInterval, + self, + CleanupBloomFilter) + + override def receive: Receive = { + + case TransactionProcessing(info) => canProcessTransactions = info + + case IsChainSynced(info) => chainSynced = info + + case CleanupBloomFilter => + bloomFilterForTransactionsIds = initBloomFilter + + case DataFromPeer(message, remote) => + message match { + + case RequestModifiersNetworkMessage((_, requestedIds)) => + val modifiersIds: Seq[Transaction] = requestedIds + .map(Algos.encode) + .collect { case id if memoryPool.contains(id) => memoryPool.get(id) } + .flatten + context.parent ! RequestedModifiersForRemote(remote, modifiersIds) + logger.debug( + s"MemoryPool got request modifiers message. Number of requested ids is ${requestedIds.size}." + + s" Number of sent transactions is ${modifiersIds.size}. Request was from $remote." + ) + + case InvNetworkMessage((_, txs)) => + val notYetRequestedTransactions: IndexedSeq[ModifierId] = notRequestedYet(txs.toIndexedSeq) + if (notYetRequestedTransactions.nonEmpty) { + sender ! RequestFromLocal(Some(remote), Transaction.modifierTypeId, notYetRequestedTransactions.toList) + logger.debug( + s"MemoryPool got inv message with ${txs.size} ids." + + s" Not yet requested ids size is ${notYetRequestedTransactions.size}." + ) + } else + logger.debug( + s"MemoryPool got inv message with ${txs.size} ids." + + s" There are no not yet requested ids." + ) + + case InvNetworkMessage(invData) => + logger.debug( + s"Get inv with tx: ${invData._2.map(Algos.encode).mkString(",")}, but " + + s"chainSynced is $chainSynced and canProcessTransactions is $canProcessTransactions." + ) + + case _ => logger.debug(s"MemoryPoolProcessor got invalid type of DataFromPeer message!") + } + + } + def initBloomFilter: BloomFilter[String] = BloomFilter.create( + Funnels.stringFunnel(Charsets.UTF_8), + settings.mempool.bloomFilterCapacity, + settings.mempool.bloomFilterFailureProbability + ) + + def notRequestedYet(ids: IndexedSeq[ModifierId]): IndexedSeq[ModifierId] = ids.collect { + case id: ModifierId if !bloomFilterForTransactionsIds.mightContain(Algos.encode(id)) => + bloomFilterForTransactionsIds.put(Algos.encode(id)) + id + } + +} + +object MemoryPoolProcessor { + + def props(settings: EncryAppSettings) = Props(new MemoryPoolProcessor(settings)) + + case object CleanupBloomFilter + + final case class RequestedModifiersForRemote(peer: InetSocketAddress, txs: Seq[Transaction]) + +} diff --git a/src/main/scala/encry/view/mempool/TransactionsValidator.scala b/src/main/scala/encry/view/mempool/TransactionsValidator.scala new file mode 100644 index 0000000000..4268635170 --- /dev/null +++ b/src/main/scala/encry/view/mempool/TransactionsValidator.scala @@ -0,0 +1,56 @@ +package encry.view.mempool + +import TransactionProto.TransactionProtoMessage +import akka.actor.{Actor, ActorRef, Props} +import com.typesafe.scalalogging.StrictLogging +import encry.network.BlackList.BanReason.{CorruptedSerializedBytes, SyntacticallyInvalidTransaction} +import encry.network.PeerConnectionHandler.ConnectedPeer +import encry.network.PeersKeeper.BanPeer +import encry.settings.EncryAppSettings +import encry.utils.NetworkTimeProvider +import encry.view.mempool.MemoryPool.NewTransaction +import encry.view.mempool.TransactionsValidator.{InvalidModifier, ModifiersForValidating} +import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionProtoSerializer} +import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} +import scala.util.{Failure, Success, Try} + +class TransactionsValidator(settings: EncryAppSettings, + memPool: ActorRef, + networkTimeProvider: NetworkTimeProvider) + extends Actor + with StrictLogging { + + override def receive(): Receive = { + case ModifiersForValidating(remote, typeId, filteredModifiers) => + typeId match { + case Transaction.modifierTypeId => + filteredModifiers.foreach { + case (id, bytes) => + Try(TransactionProtoSerializer.fromProto(TransactionProtoMessage.parseFrom(bytes))).flatten match { + case Success(tx) if tx.semanticValidity.isSuccess => memPool ! NewTransaction(tx) + case Success(tx) => + logger.info(s"Transaction with id: ${tx.encodedId} invalid cause of: ${tx.semanticValidity}.") + context.parent ! BanPeer(remote.socketAddress, SyntacticallyInvalidTransaction) + context.parent ! InvalidModifier(id) + case Failure(ex) => + context.parent ! BanPeer(remote.socketAddress, CorruptedSerializedBytes) + context.parent ! InvalidModifier(id) + logger.info(s"Received modifier from $remote can't be parsed cause of: ${ex.getMessage}.") + } + } + } + } + +} + +object TransactionsValidator { + + final case class ModifiersForValidating(remote: ConnectedPeer, + typeId: ModifierTypeId, + modifiers: Map[ModifierId, Array[Byte]]) + + final case class InvalidModifier(ids: ModifierId) extends AnyVal + + def props(settings: EncryAppSettings, memPool: ActorRef, ntp: NetworkTimeProvider): Props = + Props(new TransactionsValidator(settings, memPool, ntp)) +} From d8b344d8fa253ef5923983b06ee73951cb57920d Mon Sep 17 00:00:00 2001 From: Lior Date: Thu, 5 Mar 2020 17:13:08 +0300 Subject: [PATCH 044/177] code cleanup --- .../encry/network/NodeViewSynchronizer.scala | 35 ++----------------- .../view/mempool/IntermediaryMempool.scala | 2 +- .../scala/encry/view/mempool/MemoryPool.scala | 4 +-- .../view/mempool/MemoryPoolProcessor.scala | 17 +++++---- 4 files changed, 16 insertions(+), 42 deletions(-) diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index 0875248abc..dc569c7efa 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -1,45 +1,16 @@ package encry.network -import HeaderProto.HeaderProtoMessage import java.net.InetSocketAddress - -import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props} +import akka.actor.{ActorSystem, PoisonPill} import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} -import akka.util.Timeout import com.typesafe.config.Config -import com.typesafe.scalalogging.StrictLogging -import encry.consensus.HistoryConsensus._ -import encry.local.miner.Miner.{ClIMiner, DisableMining, StartMining} -import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} -import encry.network.NodeViewSynchronizer.ReceivableMessages._ -import encry.network.PeerConnectionHandler.ConnectedPeer -import encry.network.PeersKeeper._ -import encry.network.PrioritiesCalculator.AccumulatedPeersStatistic -import encry.settings.EncryAppSettings -import encry.utils.CoreTaggedTypes.VersionTag -import encry.utils.Utils._ -import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} -import encry.view.NodeViewErrors.ModifierApplyError +import encry.nvg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.view.history.History import encry.view.mempool.MemoryPool._ import encry.view.state.UtxoState -import org.encryfoundation.common.modifiers.{NodeViewModifier, PersistentNodeViewModifier} -import org.encryfoundation.common.modifiers.history._ -import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionProtoSerializer} -import org.encryfoundation.common.network.BasicMessagesRepo._ -import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} -import scala.concurrent.duration._ -import encry.network.ModifiersToNetworkUtils._ -import encry.nvg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} -import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} -import encry.view.fast.sync.SnapshotHolder -import encry.view.fast.sync.SnapshotHolder.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks, UpdateSnapshot} - -import scala.util.Try - //class NodeViewSynchronizer(influxRef: Option[ActorRef], // nodeViewHolderRef: ActorRef, // settings: EncryAppSettings, diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala index 1cc1da0dfa..e798dbd897 100644 --- a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala +++ b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala @@ -27,7 +27,7 @@ class IntermediaryMempool(settings: EncryAppSettings, context.actorOf(TransactionsValidator.props(settings, memoryPool, networkTimeProvider), name = "Transaction-validator") val mempoolProcessor: ActorRef = - context.actorOf(MemoryPoolProcessor.props(settings), name = "mempool-processor") + context.actorOf(MemoryPoolProcessor.props(settings, networkTimeProvider), name = "mempool-processor") override def receive(): Receive = { case msg @ InvalidModifier(_) => // to nvsh diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index e4e81be6d1..695ded5134 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -5,8 +5,6 @@ import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import cats.syntax.either._ import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.network.NodeViewSynchronizer.ReceivableMessages.{RequestFromLocal, SemanticallySuccessfulModifier, SuccessfulTransaction} -import encry.network.PeerConnectionHandler.ConnectedPeer import encry.nvg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider @@ -128,7 +126,7 @@ class MemoryPool(settings: EncryAppSettings, // case RequestModifiersForTransactions(remote, ids) => // val modifiersIds: Seq[Transaction] = ids // .map(Algos.encode) -// .collect { case id if memoryPool.contains(id) => memoryPool.get(id) } +// .collect { case if memorascyPool.contains(id) => memoryPool.get(id) } // .flatten // sender() ! RequestedModifiersForRemote(remote, modifiersIds) // logger.debug(s"MemoryPool got request modifiers message. Number of requested ids is ${ids.size}." + diff --git a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala index f9c3399ae4..0c919244b5 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala +++ b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala @@ -1,23 +1,26 @@ package encry.view.mempool import java.net.InetSocketAddress -import akka.actor.{ Actor, Props } + +import akka.actor.{Actor, Props} import com.google.common.base.Charsets -import com.google.common.hash.{ BloomFilter, Funnels } +import com.google.common.hash.{BloomFilter, Funnels} import com.typesafe.scalalogging.StrictLogging import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.settings.EncryAppSettings +import encry.utils.NetworkTimeProvider import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.TransactionProcessing -import encry.view.mempool.MemoryPoolProcessor.{ CleanupBloomFilter, RequestedModifiersForRemote } +import encry.view.mempool.MemoryPoolProcessor.{CleanupBloomFilter, RequestedModifiersForRemote} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestModifiersNetworkMessage } +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestModifiersNetworkMessage} import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId + import scala.collection.IndexedSeq -class MemoryPoolProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { +class MemoryPoolProcessor(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends Actor with StrictLogging { import context.dispatcher @@ -27,6 +30,8 @@ class MemoryPoolProcessor(settings: EncryAppSettings) extends Actor with StrictL var chainSynced: Boolean = false + var memoryPool: MemoryPoolStorage = MemoryPoolStorage.empty(settings, ntp) + override def preStart(): Unit = context.system.scheduler.schedule(settings.mempool.bloomFilterCleanupInterval, settings.mempool.bloomFilterCleanupInterval, @@ -96,7 +101,7 @@ class MemoryPoolProcessor(settings: EncryAppSettings) extends Actor with StrictL object MemoryPoolProcessor { - def props(settings: EncryAppSettings) = Props(new MemoryPoolProcessor(settings)) + def props(settings: EncryAppSettings, ntp: NetworkTimeProvider) = Props(new MemoryPoolProcessor(settings, ntp)) case object CleanupBloomFilter From 8148c2d306afeaedc5a826cbc27c7be57fb8e5cd Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 5 Mar 2020 17:23:57 +0300 Subject: [PATCH 045/177] actor snapshot holder moved to the nvg --- .../benches/SnapshotAssemblerBench.scala | 2 +- .../encry/network/NodeViewSynchronizer.scala | 33 +++- .../scala/encry/nvg/IntermediaryNVH.scala | 7 +- src/main/scala/encry/nvg/NodeViewHolder.scala | 12 +- .../SnapshotProcessorActor.scala} | 181 +++++++++--------- .../sync/SnapshotDownloadController.scala | 41 ++-- ...SnapshotDownloadControllerStorageAPI.scala | 2 +- .../view/fast/sync/SnapshotProcessor.scala | 42 ++-- .../sync/SnapshotProcessorStorageAPI.scala | 3 +- .../encry/view/state/avlTree/AvlTree.scala | 4 +- 10 files changed, 186 insertions(+), 141 deletions(-) rename src/main/scala/encry/{view/fast/sync/SnapshotHolder.scala => nvg/SnapshotProcessorActor.scala} (70%) diff --git a/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala b/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala index fa63fc3e4e..5d9294009a 100644 --- a/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala +++ b/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala @@ -7,12 +7,12 @@ import benches.SnapshotAssemblerBench.SnapshotAssemblerBenchState import encry.view.state.avlTree.utils.implicits.Instances._ import benches.StateBenches.{StateBenchState, benchSettings} import benches.Utils.{getRandomTempDir, utxoFromBoxHolder} +import encry.nvg.SnapshotProcessorActor import encry.settings.Settings import encry.storage.{RootNodesStorage, VersionalStorage} import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} import encry.utils.FileHelper -import encry.view.fast.sync.SnapshotHolder import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import org.encryfoundation.common.utils.TaggedTypes.Height diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index dc569c7efa..e3d4b4bbbb 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -1,16 +1,43 @@ package encry.network +import HeaderProto.HeaderProtoMessage import java.net.InetSocketAddress -import akka.actor.{ActorSystem, PoisonPill} + +import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props} import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} +import akka.util.Timeout import com.typesafe.config.Config -import encry.nvg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} +import com.typesafe.scalalogging.StrictLogging +import encry.consensus.HistoryConsensus._ +import encry.local.miner.Miner.{ClIMiner, DisableMining, StartMining} +import encry.network.DeliveryManager.FullBlockChainIsSynced +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} +import encry.network.NodeViewSynchronizer.ReceivableMessages._ +import encry.network.PeerConnectionHandler.ConnectedPeer +import encry.network.PeersKeeper._ +import encry.network.PrioritiesCalculator.AccumulatedPeersStatistic +import encry.settings.EncryAppSettings +import encry.utils.CoreTaggedTypes.VersionTag +import encry.utils.Utils._ +import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} +import encry.view.NodeViewErrors.ModifierApplyError import encry.view.history.History import encry.view.mempool.MemoryPool._ import encry.view.state.UtxoState -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.modifiers.{NodeViewModifier, PersistentNodeViewModifier} +import org.encryfoundation.common.modifiers.history._ +import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionProtoSerializer} +import org.encryfoundation.common.network.BasicMessagesRepo._ +import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} +import scala.concurrent.duration._ +import encry.network.ModifiersToNetworkUtils._ +import encry.nvg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} +import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} + +import scala.util.Try + //class NodeViewSynchronizer(influxRef: Option[ActorRef], // nodeViewHolderRef: ActorRef, // settings: EncryAppSettings, diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index aa23f0af5d..f088689cc5 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -20,15 +20,10 @@ import encry.nvg.NodeViewHolder.{ SyntacticallyFailedModification, UpdateHistoryReader } +import encry.nvg.SnapshotProcessorActor.{ FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks } import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider -import encry.view.fast.sync.SnapshotHolder.{ - FastSyncDone, - HeaderChainIsSynced, - RequiredManifestHeightAndId, - TreeChunks -} import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions import org.encryfoundation.common.network.BasicMessagesRepo.{ diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index ff11c3536b..d29b40c6c0 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -24,14 +24,22 @@ import encry.nvg.NodeViewHolder.{ UpdateHistoryReader, UpdateInformation } +import encry.nvg.SnapshotProcessorActor.{ + FastSyncDone, + FastSyncFinished, + HeaderChainIsSynced, + RemoveRedundantManifestIds, + RequiredManifestHeightAndId, + SnapshotChunk, + TreeChunks +} +import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ManifestId import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.NodeViewErrors.ModifierApplyError import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId -import encry.view.fast.sync.SnapshotHolder._ import encry.view.history.storage.HistoryStorage import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } import encry.view.mempool.MemoryPool.RolledBackTransactions diff --git a/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala b/src/main/scala/encry/nvg/SnapshotProcessorActor.scala similarity index 70% rename from src/main/scala/encry/view/fast/sync/SnapshotHolder.scala rename to src/main/scala/encry/nvg/SnapshotProcessorActor.scala index c922650269..c19d543bc6 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala +++ b/src/main/scala/encry/nvg/SnapshotProcessorActor.scala @@ -1,20 +1,28 @@ -package encry.view.fast.sync +package encry.nvg import SnapshotChunkProto.SnapshotChunkMessage import SnapshotManifestProto.SnapshotManifestProtoMessage import akka.actor.{ Actor, ActorRef, Cancellable, Props } +import cats.syntax.either._ import cats.syntax.option._ import com.google.protobuf.ByteString import com.typesafe.scalalogging.StrictLogging +import encry.network.BlackList.BanReason.{ + InvalidChunkMessage, + InvalidResponseManifestMessage, + InvalidStateAfterFastSync +} +import encry.network.Broadcast import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } import encry.network.NodeViewSynchronizer.ReceivableMessages.ChangedHistory -import encry.network.PeersKeeper.SendToNetwork -import encry.network.{ Broadcast, PeerConnectionHandler } +import encry.network.PeersKeeper.{ BanPeer, SendToNetwork } import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.nvg.SnapshotProcessorActor.SnapshotManifest.{ ChunkId, ManifestId } +import encry.nvg.SnapshotProcessorActor._ import encry.settings.EncryAppSettings import encry.storage.VersionalStorage.{ StorageKey, StorageValue } -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.{ ChunkId, ManifestId } -import encry.view.fast.sync.SnapshotHolder._ +import encry.view.fast.sync.FastSyncExceptions.{ ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage } +import encry.view.fast.sync.{ RequestsPerPeriodProcessor, SnapshotDownloadController, SnapshotProcessor } import encry.view.history.History import encry.view.state.UtxoState import encry.view.state.avlTree.{ Node, NodeSerilalizer } @@ -26,11 +34,12 @@ import supertagged.TaggedType import scala.util.Try -class SnapshotHolder(settings: EncryAppSettings, - networkController: ActorRef, - nodeViewHolder: ActorRef, - nodeViewSynchronizer: ActorRef) - extends Actor +class SnapshotProcessorActor( + settings: EncryAppSettings, + networkController: ActorRef, + nodeViewHolder: ActorRef, + nodeViewSynchronizer: ActorRef +) extends Actor with StrictLogging { import context.dispatcher @@ -93,43 +102,43 @@ class SnapshotHolder(settings: EncryAppSettings, logger.info( s"Got new manifest message ${Algos.encode(manifest.manifestId.toByteArray)} while processing chunks." ) -// case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => -// (for { -// controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) -// (controller, chunk) = controllerAndChunk -// validChunk <- snapshotProcessor.validateChunkId(chunk) -// processor = snapshotProcessor.updateCache(validChunk) -// newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { -// case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] -// case t => t.asLeft[SnapshotProcessor] -// } -// } yield (newProcessor, controller)) match { -// case Left(err: UnexpectedChunkMessage) => -// logger.info(s"Error has occurred ${err.error} with peer $remote") -// case Left(error) => -// logger.info(s"Error has occurred: $error") -// nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) -// restartFastSync(history) -// case Right((processor, controller)) -// if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => -// nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) -// restartFastSync(history) -// case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => -// processor.assembleUTXOState() match { -// case Right(state) => -// logger.info(s"Tree is valid on Snapshot holder!") -// processor.wallet.foreach { wallet: EncryWallet => -// (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] -// } -// case _ => -// nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) -// restartFastSync(history).asLeft[Unit] -// } -// case Right((processor, controller)) => -// snapshotDownloadController = controller -// snapshotProcessor = processor -// if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks -// } + case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => + (for { + controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) + (controller, chunk) = controllerAndChunk + validChunk <- snapshotProcessor.validateChunkId(chunk) + processor = snapshotProcessor.updateCache(validChunk) + newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { + case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] + case t => t.asLeft[SnapshotProcessor] + } + } yield (newProcessor, controller)) match { + case Left(err: UnexpectedChunkMessage) => + logger.info(s"Error has occurred ${err.error} with peer $remote") + case Left(error) => + logger.info(s"Error has occurred: $error") + nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) + restartFastSync(history) + case Right((processor, controller)) + if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => + nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) + restartFastSync(history) + case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => + processor.assembleUTXOState() match { + case Right(state) => + logger.info(s"Tree is valid on Snapshot holder!") + processor.wallet.foreach { wallet: EncryWallet => + (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] + } + case _ => + nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) + restartFastSync(history).asLeft[Unit] + } + case Right((processor, controller)) => + snapshotDownloadController = controller + snapshotProcessor = processor + if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks + } case ResponseChunkMessage(_) => logger.info(s"Received chunk from unexpected peer ${remote}") @@ -149,9 +158,9 @@ class SnapshotHolder(settings: EncryAppSettings, case Right(controllerAndIds) => snapshotDownloadController = controllerAndIds._1 controllerAndIds._2.foreach { msg => - snapshotDownloadController.cp.foreach { peer: PeerConnectionHandler.ConnectedPeer => - peer.handlerRef ! msg - } +// snapshotDownloadController.cp.foreach { peer: PeerConnectionHandler.ConnectedPeer => +// peer.handlerRef ! msg +// } } context.become(fastSyncMod(history, timer).orElse(commonMessages)) } @@ -173,7 +182,7 @@ class SnapshotHolder(settings: EncryAppSettings, snapshotDownloadController.awaitedChunks.map { id => RequestChunkMessage(id.data) }.foreach { msg => - snapshotDownloadController.cp.foreach(peer => peer.handlerRef ! msg) + //snapshotDownloadController.cp.foreach(peer => peer.handlerRef ! msg) } context.become(fastSyncMod(history, timer).orElse(commonMessages)) @@ -210,39 +219,39 @@ class SnapshotHolder(settings: EncryAppSettings, context.become(awaitManifestMod(newScheduler.some, history).orElse(commonMessages)) case DataFromPeer(message, remote) => -// message match { -// case ResponseManifestMessage(manifest) => -// val isValidManifest: Boolean = -// snapshotDownloadController.checkManifestValidity(manifest.manifestId.toByteArray, history) -// val canBeProcessed: Boolean = snapshotDownloadController.canNewManifestBeProcessed -// if (isValidManifest && canBeProcessed) { -// (for { -// controller <- snapshotDownloadController.processManifest(manifest, remote, history) -// processor <- snapshotProcessor.initializeApplicableChunksCache( -// history, -// snapshotDownloadController.requiredManifestHeight -// ) -// } yield (controller, processor)) match { -// case Left(error) => -// nodeViewSynchronizer ! BanPeer(remote, InvalidResponseManifestMessage(error.error)) -// case Right((controller, processor)) => -// logger.debug(s"Request manifest message successfully processed.") -// responseManifestTimeout.foreach(_.cancel()) -// snapshotDownloadController = controller -// snapshotProcessor = processor -// self ! RequestNextChunks -// logger.debug("Manifest processed successfully.") -// context.become(fastSyncMod(history, none)) -// } -// } else if (!isValidManifest) { -// logger.info(s"Got manifest with invalid id ${Algos.encode(manifest.manifestId.toByteArray)}") -// nodeViewSynchronizer ! BanPeer( -// remote, -// InvalidResponseManifestMessage(s"Invalid manifest id ${Algos.encode(manifest.manifestId.toByteArray)}") -// ) -// } else logger.info(s"Doesn't need to process new manifest.") -// case _ => -// } + message match { + case ResponseManifestMessage(manifest) => + val isValidManifest: Boolean = + snapshotDownloadController.checkManifestValidity(manifest.manifestId.toByteArray, history) + val canBeProcessed: Boolean = snapshotDownloadController.canNewManifestBeProcessed + if (isValidManifest && canBeProcessed) { + (for { + controller <- snapshotDownloadController.processManifest(manifest, remote, history) + processor <- snapshotProcessor.initializeApplicableChunksCache( + history, + snapshotDownloadController.requiredManifestHeight + ) + } yield (controller, processor)) match { + case Left(error) => + nodeViewSynchronizer ! BanPeer(remote, InvalidResponseManifestMessage(error.error)) + case Right((controller, processor)) => + logger.debug(s"Request manifest message successfully processed.") + responseManifestTimeout.foreach(_.cancel()) + snapshotDownloadController = controller + snapshotProcessor = processor + self ! RequestNextChunks + logger.debug("Manifest processed successfully.") + context.become(fastSyncMod(history, none)) + } + } else if (!isValidManifest) { + logger.info(s"Got manifest with invalid id ${Algos.encode(manifest.manifestId.toByteArray)}") + nodeViewSynchronizer ! BanPeer( + remote, + InvalidResponseManifestMessage(s"Invalid manifest id ${Algos.encode(manifest.manifestId.toByteArray)}") + ) + } else logger.info(s"Doesn't need to process new manifest.") + case _ => + } case msg @ RequiredManifestHeightAndId(_, _) => self ! msg @@ -316,7 +325,7 @@ class SnapshotHolder(settings: EncryAppSettings, context.system.scheduler.scheduleOnce(settings.snapshotSettings.responseTimeout)(self ! CheckDelivery).some } -object SnapshotHolder { +object SnapshotProcessorActor { case object RemoveRedundantManifestIds @@ -383,7 +392,7 @@ object SnapshotHolder { networkController: ActorRef, nodeViewHolderRef: ActorRef, nodeViewSynchronizer: ActorRef): Props = Props( - new SnapshotHolder(settings, networkController, nodeViewHolderRef, nodeViewSynchronizer) + new SnapshotProcessorActor(settings, networkController, nodeViewHolderRef, nodeViewSynchronizer) ) } diff --git a/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala b/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala index a9895bcfc2..588752bda5 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala @@ -1,46 +1,43 @@ package encry.view.fast.sync import java.io.File +import java.net.InetSocketAddress import SnapshotChunkProto.SnapshotChunkMessage import SnapshotManifestProto.SnapshotManifestProtoMessage import cats.syntax.either._ import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging -import encry.network.PeerConnectionHandler.ConnectedPeer +import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ChunkId +import encry.nvg.SnapshotProcessorActor.{ SnapshotChunk, SnapshotChunkSerializer, SnapshotManifestSerializer } import encry.settings.EncryAppSettings import encry.storage.levelDb.versionalLevelDB.LevelDbFactory import encry.view.fast.sync.FastSyncExceptions._ -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ChunkId -import encry.view.fast.sync.SnapshotHolder.{SnapshotChunk, SnapshotChunkSerializer, SnapshotManifestSerializer} -import encry.view.fast.sync.FastSyncExceptions._ -import encry.view.fast.sync.SnapshotHolder.{ SnapshotChunk, SnapshotChunkSerializer, SnapshotManifestSerializer } import encry.view.history.History import io.iohk.iodb.ByteArrayWrapper -import org.encryfoundation.common.network.BasicMessagesRepo.{ NetworkMessage, RequestChunkMessage } -import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.network.BasicMessagesRepo.RequestChunkMessage import org.encryfoundation.common.utils.Algos -import org.iq80.leveldb.{DB, Options} +import org.iq80.leveldb.{ DB, Options } -final case class SnapshotDownloadController(requiredManifestId: Array[Byte], - awaitedChunks: Set[ByteArrayWrapper], - settings: EncryAppSettings, - cp: Option[ConnectedPeer], - requiredManifestHeight: Int, - storage: DB, - batchesSize: Int, - nextGroupForRequestNumber: Int) - extends SnapshotDownloadControllerStorageAPI +final case class SnapshotDownloadController( + requiredManifestId: Array[Byte], + awaitedChunks: Set[ByteArrayWrapper], + settings: EncryAppSettings, + cp: Option[InetSocketAddress], + requiredManifestHeight: Int, + storage: DB, + batchesSize: Int, + nextGroupForRequestNumber: Int +) extends SnapshotDownloadControllerStorageAPI with StrictLogging with AutoCloseable { def processManifest( manifestProto: SnapshotManifestProtoMessage, - remote: ConnectedPeer, + remote: InetSocketAddress, history: History ): Either[SnapshotDownloadControllerException, SnapshotDownloadController] = { - logger.info(s"Got new manifest from ${remote.socketAddress}.") + logger.info(s"Got new manifest from $remote.") Either.fromTry(SnapshotManifestSerializer.fromProto(manifestProto)) match { case Left(error) => logger.info(s"Manifest was parsed with error ${error.getCause}.") @@ -74,9 +71,9 @@ final case class SnapshotDownloadController(requiredManifestId: Array[Byte], def processRequestedChunk( chunkMessage: SnapshotChunkMessage, - remote: ConnectedPeer + remote: InetSocketAddress ): Either[ChunkValidationError, (SnapshotDownloadController, SnapshotChunk)] = { - logger.debug(s"Got new chunk from ${remote.socketAddress}.") + logger.debug(s"Got new chunk from $remote.") Either.fromTry(SnapshotChunkSerializer.fromProto(chunkMessage)) match { case Left(error) => logger.info(s"Chunk was parsed with error ${error.getCause}.") @@ -123,7 +120,7 @@ final case class SnapshotDownloadController(requiredManifestId: Array[Byte], def canNewManifestBeProcessed: Boolean = cp.isEmpty - def canChunkBeProcessed(remote: ConnectedPeer): Boolean = cp.exists(_.socketAddress == remote.socketAddress) + def canChunkBeProcessed(remote: InetSocketAddress): Boolean = cp.contains(remote) def reInitFastSync: SnapshotDownloadController = try { diff --git a/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala b/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala index b01c7ae46c..b0ec686518 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala @@ -1,8 +1,8 @@ package encry.view.fast.sync import com.typesafe.scalalogging.StrictLogging +import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ChunkId import encry.settings.EncryAppSettings -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ChunkId import org.encryfoundation.common.utils.Algos import org.iq80.leveldb.DB diff --git a/src/main/scala/encry/view/fast/sync/SnapshotProcessor.scala b/src/main/scala/encry/view/fast/sync/SnapshotProcessor.scala index 3675c8dbce..07d25cc879 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotProcessor.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotProcessor.scala @@ -1,42 +1,50 @@ package encry.view.fast.sync import java.io.File + import akka.actor.ActorRef import cats.syntax.either._ import cats.syntax.option._ import com.google.common.primitives.Ints import com.typesafe.scalalogging.StrictLogging +import encry.nvg.SnapshotProcessorActor.{ + SnapshotChunk, + SnapshotChunkSerializer, + SnapshotManifest, + SnapshotManifestSerializer +} +import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ManifestId import encry.settings.EncryAppSettings -import encry.storage.{RootNodesStorage, VersionalStorage} -import encry.storage.VersionalStorage.{StorageKey, StorageType, StorageValue, StorageVersion} +import encry.storage.{ RootNodesStorage, VersionalStorage } +import encry.storage.VersionalStorage.{ StorageKey, StorageType, StorageValue, StorageVersion } import encry.storage.iodb.versionalIODB.IODBWrapper -import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} +import encry.storage.levelDb.versionalLevelDB.{ LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion } import encry.view.fast.sync.FastSyncExceptions._ -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ManifestId -import encry.view.fast.sync.SnapshotHolder.{SnapshotChunk, SnapshotChunkSerializer, SnapshotManifest, SnapshotManifestSerializer} import encry.view.history.History import encry.view.state.UtxoState import encry.view.state.avlTree._ import encry.view.state.avlTree.utils.implicits.Instances._ import encry.view.wallet.EncryWallet -import io.iohk.iodb.{ByteArrayWrapper, LSMStore} +import io.iohk.iodb.{ ByteArrayWrapper, LSMStore } import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.state.StateModifierSerializer import org.encryfoundation.common.modifiers.state.box.EncryBaseBox import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{Height, ModifierId} -import org.iq80.leveldb.{DB, Options} +import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } +import org.iq80.leveldb.{ DB, Options } import scorex.utils.Random -import scala.collection.immutable.{HashMap, HashSet} + +import scala.collection.immutable.{ HashMap, HashSet } import scala.language.postfixOps -import scala.util.{Failure, Success} +import scala.util.{ Failure, Success } -final case class SnapshotProcessor(settings: EncryAppSettings, - storage: VersionalStorage, - applicableChunks: HashSet[ByteArrayWrapper], - chunksCache: HashMap[ByteArrayWrapper, SnapshotChunk], - wallet: Option[EncryWallet]) - extends StrictLogging +final case class SnapshotProcessor( + settings: EncryAppSettings, + storage: VersionalStorage, + applicableChunks: HashSet[ByteArrayWrapper], + chunksCache: HashMap[ByteArrayWrapper, SnapshotChunk], + wallet: Option[EncryWallet] +) extends StrictLogging with SnapshotProcessorStorageAPI with AutoCloseable { @@ -181,7 +189,7 @@ final case class SnapshotProcessor(settings: EncryAppSettings, rootNode <- getRootNode height <- getHeight //todo: remove RootNodesStorage.emptyRootStorage - avlTree = new AvlTree[StorageKey, StorageValue](rootNode, storage, RootNodesStorage.emptyRootStorage) + avlTree = new AvlTree[StorageKey, StorageValue](rootNode, storage, RootNodesStorage.emptyRootStorage) } yield UtxoState(avlTree, height, settings.constants, influxRef) private def getHeight: Either[EmptyHeightKey, Height] = diff --git a/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala b/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala index 5ebbe5250f..c0f868cbb3 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala @@ -3,10 +3,11 @@ package encry.view.fast.sync import SnapshotChunkProto.SnapshotChunkMessage import SnapshotManifestProto.SnapshotManifestProtoMessage import com.typesafe.scalalogging.StrictLogging +import encry.nvg.SnapshotProcessorActor.{ SnapshotManifest, SnapshotManifestSerializer } import encry.storage.VersionalStorage import encry.storage.VersionalStorage.{ StorageKey, StorageValue } -import encry.view.fast.sync.SnapshotHolder.{ SnapshotManifest, SnapshotManifestSerializer } import org.encryfoundation.common.utils.Algos + import scala.util.Try trait SnapshotProcessorStorageAPI extends StrictLogging { diff --git a/src/main/scala/encry/view/state/avlTree/AvlTree.scala b/src/main/scala/encry/view/state/avlTree/AvlTree.scala index 20c039bf6c..983cd6317a 100644 --- a/src/main/scala/encry/view/state/avlTree/AvlTree.scala +++ b/src/main/scala/encry/view/state/avlTree/AvlTree.scala @@ -4,10 +4,10 @@ import cats.syntax.order._ import cats.{Monoid, Order} import com.google.common.primitives.Ints import com.typesafe.scalalogging.StrictLogging +import encry.nvg.SnapshotProcessorActor.SnapshotChunk +import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ChunkId import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} import encry.storage.{RootNodesStorage, VersionalStorage} -import encry.view.fast.sync.SnapshotHolder.SnapshotChunk -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ChunkId import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree.Direction import encry.view.state.avlTree.AvlTree.Directions.{EMPTY, LEFT, RIGHT} From 548191bbd26fbe72e8310742c55d3e9bed54b431 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 5 Mar 2020 18:10:32 +0300 Subject: [PATCH 046/177] add processing getPeers message on pk --- .../scala/encry/network/NetworkRouter.scala | 6 +- src/main/scala/encry/network/PK.scala | 52 ++++- ...hotDownloadControllerStorageAPITests.scala | 90 ++++---- .../sync/SnapshotDownloadControllerTest.scala | 212 +++++++++--------- 4 files changed, 205 insertions(+), 155 deletions(-) diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 8c19347344..8d95e93c67 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -34,8 +34,8 @@ class NetworkRouter(settings: NetworkSettings, IO(Tcp) ! Bind(self, settings.bindAddress, options = KeepAlive(true) :: Nil, pullMode = false) - val peersKeeper = context.system.actorOf(PK.props(settings, blackListSettings), "peersKeeper") - val deliveryManager = context.system.actorOf(DM.props(settings), "deliveryManager") + val peersKeeper = context.actorOf(PK.props(settings, blackListSettings), "peersKeeper") + val deliveryManager = context.actorOf(DM.props(settings), "deliveryManager") val externalSocketAddress: Option[InetSocketAddress] = settings.declaredAddress override def receive: Receive = bindingLogic orElse businessLogic orElse peersLogic orElse { @@ -67,7 +67,7 @@ class NetworkRouter(settings: NetworkSettings, case msg: ModifierFromNetwork => handlerForMods ! msg case msg: OtherNodeSyncingStatus => peersKeeper ! msg case msg: MessageToNetwork => - context.system.actorOf(MessageBuilder.props(msg, peersKeeper, deliveryManager), s"messageBuilder${Random.nextInt()}") + context.actorOf(MessageBuilder.props(msg, peersKeeper, deliveryManager), s"messageBuilder${Random.nextInt()}") } def peersLogic: Receive = { diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 3c892e5b0f..6f0067b68e 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -4,13 +4,18 @@ import java.net.{InetAddress, InetSocketAddress} import akka.actor.{Actor, Props} import com.typesafe.scalalogging.StrictLogging +import encry.network.BlackList.BanReason.SentPeersMessageWithoutRequest import encry.network.BlackList.{BanReason, BanTime, BanType} +import encry.network.ConnectedPeersCollection.PeerInfo import encry.network.MessageBuilder.{GetPeerByPredicate, GetPeerInfo, GetPeers} +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} +import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection import encry.network.PeerConnectionHandler.{Incoming, Outgoing} import encry.network.PeersKeeper.{BanPeer, BanPeerFromAPI, PeerForConnection, RequestPeerForConnection} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} import encry.settings.{BlackListSettings, NetworkSettings} +import org.encryfoundation.common.network.BasicMessagesRepo.{GetPeersNetworkMessage, PeersNetworkMessage} import scala.util.{Random, Try} @@ -33,7 +38,14 @@ class PK(networkSettings: NetworkSettings, var peersForConnection: Map[InetSocketAddress, Int] = networkSettings.knownPeers .collect { case peer: InetSocketAddress if !isSelf(peer) => peer -> 0 }.toMap - override def receive: Receive = banPeersLogic orElse { + override def preStart(): Unit = { + context.parent ! RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), self) + } + + override def receive: Receive = banPeersLogic orElse networkMessagesProcessingLogic orElse { case RequestPeerForConnection if connectedPeers.size < networkSettings.maxConnections => def mapReason(address: InetAddress, r: BanReason, t: BanTime, bt: BanType): (InetAddress, BanReason) = address -> r logger.info(s"Got request for new connection. Current number of connections is: ${connectedPeers.size}, " + @@ -58,6 +70,8 @@ class PK(networkSettings: NetworkSettings, logger.info(s"Adding new peer: $peer to awaitingHandshakeConnections." + s" Current is: ${awaitingHandshakeConnections.mkString(",")}") } + case OtherNodeSyncingStatus(remote, comparison, _) => + connectedPeers = connectedPeers.updateHistoryComparisonResult(Map(remote -> comparison)) case NewConnection(remote, remoteConnection) if connectedPeers.size < networkSettings.maxConnections && !isSelf(remote) => logger.info(s"Peers keeper got request for verifying the connection with remote: $remote. " + s"Remote InetSocketAddress is: $remote. Remote InetAddress is ${remote.getAddress}. " + @@ -118,6 +132,40 @@ class PK(networkSettings: NetworkSettings, } } + def networkMessagesProcessingLogic: Receive = { + case DataFromPeer(message, remote) => message match { + case PeersNetworkMessage(peers) if !connectWithOnlyKnownPeers => + logger.info(s"Got peers message from $remote with peers ${peers.mkString(",")}") + peers + .filterNot { p => + blackList.contains(p.getAddress) || connectedPeers.contains(p) || isSelf(p) || peersForConnection.contains(p) + }.foreach { p => + logger.info(s"Found new peer: $p. Adding it to the available peers collection.") + peersForConnection = peersForConnection.updated(p, 0) + } + logger.info(s"New available peers collection after processing peers from $remote is: ${peersForConnection.keys.mkString(",")}.") + + case PeersNetworkMessage(_) => + logger.info(s"Got PeersNetworkMessage from $remote, but connectWithOnlyKnownPeers: $connectWithOnlyKnownPeers, " + + s"so ignore this message and ban this peer.") + self ! BanPeer(remote, SentPeersMessageWithoutRequest) + + case GetPeersNetworkMessage => + def findPeersForRemote(add: InetSocketAddress, info: PeerInfo): Boolean = + Try { + if (remote.getAddress.isSiteLocalAddress) true + else add.getAddress.isSiteLocalAddress && add != remote + }.getOrElse(false) + + val peers: Seq[InetSocketAddress] = connectedPeers.collect(findPeersForRemote, getPeersForRemote) + logger.info(s"Got request for local known peers. Sending to: $remote peers: ${peers.mkString(",")}.") + logger.info(s"Remote is side local: ${remote} : ${Try(remote.getAddress.isSiteLocalAddress)}") + connectedPeers.getAll.find(_._1 == remote).foreach { + case (_, info) => info.connectedPeer.handlerRef ! PeersNetworkMessage(peers) + } + } + } + def banPeersLogic: Receive = { case BanPeer(peer, reason) => logger.info(s"Banning peer: ${peer} for $reason.") @@ -129,6 +177,8 @@ class PK(networkSettings: NetworkSettings, blackList = blackList.banPeer(reason, peer.getAddress) } + def getPeersForRemote(add: InetSocketAddress, info: PeerInfo): InetSocketAddress = add + def isSelf(address: InetSocketAddress): Boolean = Try(address == networkSettings.bindAddress || networkSettings.declaredAddress.contains(address) || InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) || diff --git a/src/test/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPITests.scala b/src/test/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPITests.scala index d07e2029f3..b1bc8ad5eb 100644 --- a/src/test/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPITests.scala +++ b/src/test/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPITests.scala @@ -1,45 +1,45 @@ -package encry.view.fast.sync - -import encry.settings.EncryAppSettings -import encry.storage.levelDb.versionalLevelDB.LevelDbFactory -import encry.utils.FileHelper -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ChunkId -import org.iq80.leveldb.{ DB, Options } -import org.scalatest.{ Matchers, WordSpecLike } -import scorex.utils.Random - -class SnapshotDownloadControllerStorageAPITests extends WordSpecLike with Matchers { - - val settingsR: EncryAppSettings = EncryAppSettings.read() - - def init: SnapshotDownloadControllerStorageAPI = new SnapshotDownloadControllerStorageAPI { - override val storage: DB = LevelDbFactory.factory.open(FileHelper.getRandomTempDir, new Options) - override val settings: EncryAppSettings = settingsR - } - - "Inside SnapshotDownloadControllerStorageAPI class" should { - "insert many should insert all ids correctly / split for groups with correct size" in { - val api: SnapshotDownloadControllerStorageAPI = init - val randomIds: List[ChunkId] = (1 to 20001).map(_ => Random.randomBytes()).toList.map(ChunkId @@ _) - val groups = randomIds.grouped(settingsR.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod).toList - val insertionsResult = api.insertMany(groups) - insertionsResult.isRight shouldBe true - } - "get next for request should return batch if such exists / remove returned batch" in { - val api: SnapshotDownloadControllerStorageAPI = init - val randomIds: List[ChunkId] = (1 to 5000).map(_ => Random.randomBytes()).toList.map(ChunkId @@ _) - val groups = randomIds.grouped(settingsR.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod).toList - val _ = api.insertMany(groups) - val groupsL = randomIds.grouped(settingsR.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod).toList - (0 until groupsL.size).foreach { r => - val res = api.getNextForRequest(r) - api.getNextForRequest(r).isLeft shouldBe true - res.isRight shouldBe true - res.right.get.nonEmpty shouldBe true - res.right.get.head.sameElements(groupsL(r).head) shouldBe true - res.right.get.forall(j => groupsL(r).exists(_.sameElements(j))) shouldBe true - groupsL(r).forall(j => res.right.get.exists(_.sameElements(j))) shouldBe true - } - } - } -} +//package encry.view.fast.sync +// +//import encry.settings.EncryAppSettings +//import encry.storage.levelDb.versionalLevelDB.LevelDbFactory +//import encry.utils.FileHelper +//import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.ChunkId +//import org.iq80.leveldb.{ DB, Options } +//import org.scalatest.{ Matchers, WordSpecLike } +//import scorex.utils.Random +// +//class SnapshotDownloadControllerStorageAPITests extends WordSpecLike with Matchers { +// +// val settingsR: EncryAppSettings = EncryAppSettings.read() +// +// def init: SnapshotDownloadControllerStorageAPI = new SnapshotDownloadControllerStorageAPI { +// override val storage: DB = LevelDbFactory.factory.open(FileHelper.getRandomTempDir, new Options) +// override val settings: EncryAppSettings = settingsR +// } +// +// "Inside SnapshotDownloadControllerStorageAPI class" should { +// "insert many should insert all ids correctly / split for groups with correct size" in { +// val api: SnapshotDownloadControllerStorageAPI = init +// val randomIds: List[ChunkId] = (1 to 20001).map(_ => Random.randomBytes()).toList.map(ChunkId @@ _) +// val groups = randomIds.grouped(settingsR.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod).toList +// val insertionsResult = api.insertMany(groups) +// insertionsResult.isRight shouldBe true +// } +// "get next for request should return batch if such exists / remove returned batch" in { +// val api: SnapshotDownloadControllerStorageAPI = init +// val randomIds: List[ChunkId] = (1 to 5000).map(_ => Random.randomBytes()).toList.map(ChunkId @@ _) +// val groups = randomIds.grouped(settingsR.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod).toList +// val _ = api.insertMany(groups) +// val groupsL = randomIds.grouped(settingsR.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod).toList +// (0 until groupsL.size).foreach { r => +// val res = api.getNextForRequest(r) +// api.getNextForRequest(r).isLeft shouldBe true +// res.isRight shouldBe true +// res.right.get.nonEmpty shouldBe true +// res.right.get.head.sameElements(groupsL(r).head) shouldBe true +// res.right.get.forall(j => groupsL(r).exists(_.sameElements(j))) shouldBe true +// groupsL(r).forall(j => res.right.get.exists(_.sameElements(j))) shouldBe true +// } +// } +// } +//} diff --git a/src/test/scala/encry/view/fast/sync/SnapshotDownloadControllerTest.scala b/src/test/scala/encry/view/fast/sync/SnapshotDownloadControllerTest.scala index af09dbb389..4be8c49554 100644 --- a/src/test/scala/encry/view/fast/sync/SnapshotDownloadControllerTest.scala +++ b/src/test/scala/encry/view/fast/sync/SnapshotDownloadControllerTest.scala @@ -1,106 +1,106 @@ -package encry.view.fast.sync - -import java.net.InetSocketAddress - -import akka.actor.ActorSystem -import akka.testkit.TestProbe -import encry.modifiers.InstanceFactory -import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} -import encry.settings.{EncryAppSettings, TestNetSettings} -import encry.utils.FileHelper -import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.{ChunkId, ManifestId} -import encry.view.fast.sync.SnapshotHolder.{SnapshotManifest, SnapshotManifestSerializer} -import org.encryfoundation.common.network.BasicMessagesRepo.Handshake -import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} -import scorex.utils.Random - -class SnapshotDownloadControllerTest - extends WordSpecLike - with Matchers - with InstanceFactory - with OneInstancePerTest - with TestNetSettings { - implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") - - "Snapshot download controller" should { - "process new manifest message correctly" in { - val settingsWithRandomDir = EncryAppSettings - .read() - .copy( - directory = FileHelper.getRandomTempDir.toString - ) - val snapshotDownloadController = SnapshotDownloadController.empty(settingsWithRandomDir) - val history = generateDummyHistory(settings) - val randomChunks = (0 to 20001).map(_ => ChunkId @@ Random.randomBytes()).toList - val randomManifest = SnapshotManifestSerializer.toProto( - SnapshotManifest( - ManifestId @@ Random.randomBytes(), - randomChunks - ) - ) - val address = new InetSocketAddress("0.0.0.0", 9000) - val peer: ConnectedPeer = ConnectedPeer( - address, - TestProbe().ref, - Incoming, - Handshake(protocolToBytes(settings.network.appVersion), "0.0.0.0", Some(address), System.currentTimeMillis()) - ) - val newController = snapshotDownloadController.processManifest( - randomManifest, - peer, - history - ) - - val requiredBatchesSize = - (randomChunks.size / settingsWithRandomDir.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod) + 1 - - newController.isRight shouldBe true - newController.right.get.batchesSize shouldBe requiredBatchesSize - } - "provide correct getNextBatchAndRemoveItFromController function" in { - val settingsWithRandomDir = EncryAppSettings - .read() - .copy( - directory = FileHelper.getRandomTempDir.toString - ) - val snapshotDownloadController = SnapshotDownloadController.empty(settingsWithRandomDir) - val history = generateDummyHistory(settings) - val randomChunks = (1 to 20000).map(_ => ChunkId @@ Random.randomBytes()).toList - val randomManifest = SnapshotManifestSerializer.toProto( - SnapshotManifest( - ManifestId @@ Random.randomBytes(), - randomChunks - ) - ) - val address = new InetSocketAddress("0.0.0.0", 9000) - val peer: ConnectedPeer = ConnectedPeer( - address, - TestProbe().ref, - Incoming, - Handshake(protocolToBytes(settings.network.appVersion), "0.0.0.0", Some(address), System.currentTimeMillis()) - ) - val newController = snapshotDownloadController.processManifest( - randomManifest, - peer, - history - ) - val requiredBatchesSize = randomChunks.size / settingsWithRandomDir.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod - - val nextController = newController.right.get.getNextBatchAndRemoveItFromController - nextController.isRight shouldBe true - nextController.right.get._1.nextGroupForRequestNumber shouldBe 1 - nextController.right.get._1.batchesSize shouldBe requiredBatchesSize - 1 - - val nextController1 = (nextController.right.get._1.nextGroupForRequestNumber until requiredBatchesSize) - .foldLeft(nextController.right.get._1) { - case (controllerN, _) => - controllerN.getNextBatchAndRemoveItFromController.right.get._1 - } - - nextController1.batchesSize shouldBe 0 - - nextController1.getNextBatchAndRemoveItFromController.isLeft shouldBe true - - } - } -} +//package encry.view.fast.sync +// +//import java.net.InetSocketAddress +// +//import akka.actor.ActorSystem +//import akka.testkit.TestProbe +//import encry.modifiers.InstanceFactory +//import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} +//import encry.settings.{EncryAppSettings, TestNetSettings} +//import encry.utils.FileHelper +//import encry.view.fast.sync.SnapshotHolder.SnapshotManifest.{ChunkId, ManifestId} +//import encry.view.fast.sync.SnapshotHolder.{SnapshotManifest, SnapshotManifestSerializer} +//import org.encryfoundation.common.network.BasicMessagesRepo.Handshake +//import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} +//import scorex.utils.Random +// +//class SnapshotDownloadControllerTest +// extends WordSpecLike +// with Matchers +// with InstanceFactory +// with OneInstancePerTest +// with TestNetSettings { +// implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") +// +// "Snapshot download controller" should { +// "process new manifest message correctly" in { +// val settingsWithRandomDir = EncryAppSettings +// .read() +// .copy( +// directory = FileHelper.getRandomTempDir.toString +// ) +// val snapshotDownloadController = SnapshotDownloadController.empty(settingsWithRandomDir) +// val history = generateDummyHistory(settings) +// val randomChunks = (0 to 20001).map(_ => ChunkId @@ Random.randomBytes()).toList +// val randomManifest = SnapshotManifestSerializer.toProto( +// SnapshotManifest( +// ManifestId @@ Random.randomBytes(), +// randomChunks +// ) +// ) +// val address = new InetSocketAddress("0.0.0.0", 9000) +// val peer: ConnectedPeer = ConnectedPeer( +// address, +// TestProbe().ref, +// Incoming, +// Handshake(protocolToBytes(settings.network.appVersion), "0.0.0.0", Some(address), System.currentTimeMillis()) +// ) +// val newController = snapshotDownloadController.processManifest( +// randomManifest, +// peer, +// history +// ) +// +// val requiredBatchesSize = +// (randomChunks.size / settingsWithRandomDir.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod) + 1 +// +// newController.isRight shouldBe true +// newController.right.get.batchesSize shouldBe requiredBatchesSize +// } +// "provide correct getNextBatchAndRemoveItFromController function" in { +// val settingsWithRandomDir = EncryAppSettings +// .read() +// .copy( +// directory = FileHelper.getRandomTempDir.toString +// ) +// val snapshotDownloadController = SnapshotDownloadController.empty(settingsWithRandomDir) +// val history = generateDummyHistory(settings) +// val randomChunks = (1 to 20000).map(_ => ChunkId @@ Random.randomBytes()).toList +// val randomManifest = SnapshotManifestSerializer.toProto( +// SnapshotManifest( +// ManifestId @@ Random.randomBytes(), +// randomChunks +// ) +// ) +// val address = new InetSocketAddress("0.0.0.0", 9000) +// val peer: ConnectedPeer = ConnectedPeer( +// address, +// TestProbe().ref, +// Incoming, +// Handshake(protocolToBytes(settings.network.appVersion), "0.0.0.0", Some(address), System.currentTimeMillis()) +// ) +// val newController = snapshotDownloadController.processManifest( +// randomManifest, +// peer, +// history +// ) +// val requiredBatchesSize = randomChunks.size / settingsWithRandomDir.snapshotSettings.chunksNumberPerRequestWhileFastSyncMod +// +// val nextController = newController.right.get.getNextBatchAndRemoveItFromController +// nextController.isRight shouldBe true +// nextController.right.get._1.nextGroupForRequestNumber shouldBe 1 +// nextController.right.get._1.batchesSize shouldBe requiredBatchesSize - 1 +// +// val nextController1 = (nextController.right.get._1.nextGroupForRequestNumber until requiredBatchesSize) +// .foldLeft(nextController.right.get._1) { +// case (controllerN, _) => +// controllerN.getNextBatchAndRemoveItFromController.right.get._1 +// } +// +// nextController1.batchesSize shouldBe 0 +// +// nextController1.getNextBatchAndRemoveItFromController.isLeft shouldBe true +// +// } +// } +//} From 87c0d195d5c0b89dff3b8c656ccff7a70ec9bf02 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 10:07:52 +0300 Subject: [PATCH 047/177] SyncInfo error was fixed --- src/main/scala/encry/nvg/NodeViewHolder.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index d29b40c6c0..ba4c4c8c84 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -70,6 +70,8 @@ class NodeViewHolder( var nodeView: NodeView = restoreState().getOrElse(genesisState) + context.parent ! UpdateHistoryReader(HistoryReader(nodeView.history)) + var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] context.parent ! BlockAndHeaderInfo(nodeView.history.getBestHeader, nodeView.history.getBestBlock) From b2f4f8c5b9c69a75fe64f420e06a23ec1b766b0d Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 10:38:58 +0300 Subject: [PATCH 048/177] local commit --- .../benches/SnapshotAssemblerBench.scala | 2 +- .../scala/encry/nvg/IntermediaryNVH.scala | 31 +++- src/main/scala/encry/nvg/NodeViewHolder.scala | 7 +- .../scala/encry/nvg/SnapshotDownloader.scala | 140 ++++++++++++++++++ .../encry/nvg/SnapshotIntermediary.scala | 12 ++ ...sorActor.scala => SnapshotProcessor.scala} | 88 ++++------- .../view/fast/sync/FastSyncExceptions.scala | 2 +- .../sync/RequestsPerPeriodProcessor.scala | 2 +- .../sync/SnapshotDownloadController.scala | 4 +- ...SnapshotDownloadControllerStorageAPI.scala | 2 +- ...otProcessor.scala => SnapshotHolder.scala} | 50 +++---- .../sync/SnapshotProcessorStorageAPI.scala | 2 +- .../encry/view/state/avlTree/AvlTree.scala | 4 +- 13 files changed, 250 insertions(+), 96 deletions(-) create mode 100644 src/main/scala/encry/nvg/SnapshotDownloader.scala create mode 100644 src/main/scala/encry/nvg/SnapshotIntermediary.scala rename src/main/scala/encry/nvg/{SnapshotProcessorActor.scala => SnapshotProcessor.scala} (83%) rename src/main/scala/encry/view/fast/sync/{SnapshotProcessor.scala => SnapshotHolder.scala} (91%) diff --git a/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala b/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala index 5d9294009a..bcc7d55023 100644 --- a/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala +++ b/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala @@ -7,7 +7,7 @@ import benches.SnapshotAssemblerBench.SnapshotAssemblerBenchState import encry.view.state.avlTree.utils.implicits.Instances._ import benches.StateBenches.{StateBenchState, benchSettings} import benches.Utils.{getRandomTempDir, utxoFromBoxHolder} -import encry.nvg.SnapshotProcessorActor +import encry.nvg.SnapshotProcessor import encry.settings.Settings import encry.storage.{RootNodesStorage, VersionalStorage} import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index f088689cc5..92a7054b5a 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -2,6 +2,7 @@ package encry.nvg import akka.actor.{ Actor, ActorRef, Props } import akka.routing.BalancingPool +import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.local.miner.Miner.{ DisableMining, StartMining } @@ -20,7 +21,7 @@ import encry.nvg.NodeViewHolder.{ SyntacticallyFailedModification, UpdateHistoryReader } -import encry.nvg.SnapshotProcessorActor.{ FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks } +import encry.nvg.SnapshotProcessor.{ FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks } import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider @@ -28,7 +29,11 @@ import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, + RequestChunkMessage, + RequestManifestMessage, RequestModifiersNetworkMessage, + ResponseChunkMessage, + ResponseManifestMessage, SyncInfoNetworkMessage } import org.encryfoundation.common.utils.Algos @@ -62,6 +67,22 @@ class IntermediaryNVH( .props(ModifiersValidator.props(nodeViewHolder, settings)), name = "Modifiers-validator-router" ) + val snapshotProcessor: Option[ActorRef] = + if (settings.constants.SnapshotCreationHeight <= settings.constants.MaxRollbackDepth || + (!settings.snapshotSettings.enableFastSynchronization && !settings.snapshotSettings.enableSnapshotCreation)) + none[ActorRef] + else { + intermediaryNetwork ! RegisterMessagesHandler( + Seq( + RequestManifestMessage.NetworkMessageTypeID -> "RequestManifest", + ResponseManifestMessage.NetworkMessageTypeID -> "ResponseManifestMessage", + RequestChunkMessage.NetworkMessageTypeID -> "RequestChunkMessage", + ResponseChunkMessage.NetworkMessageTypeID -> "ResponseChunkMessage" + ), + self + ) + context.actorOf(SnapshotProcessor.props(settings, nodeViewHolder)).some + } var historyReader: HistoryReader = HistoryReader.empty @@ -69,7 +90,13 @@ class IntermediaryNVH( case ModifierFromNetwork(remote, typeId, modifierId, modifierBytes) => logger.info(s"Got modifier ${Algos.encode(modifierId)} of type $typeId from $remote for validation.") modifiersValidatorRouter ! ModifierForValidation(historyReader, modifierId, typeId, modifierBytes, remote) - case msg @ DataFromPeer(_, _) => networkMessagesProcessor ! msg + case msg @ DataFromPeer(_: SyncInfoNetworkMessage, _) => networkMessagesProcessor ! msg + case msg @ DataFromPeer(_: InvNetworkMessage, _) => networkMessagesProcessor ! msg + case msg @ DataFromPeer(_: RequestModifiersNetworkMessage, _) => networkMessagesProcessor ! msg + case msg @ DataFromPeer(_: RequestManifestMessage, _) => snapshotProcessor.foreach(_ ! msg) + case msg @ DataFromPeer(_: ResponseManifestMessage, _) => snapshotProcessor.foreach(_ ! msg) + case msg @ DataFromPeer(_: RequestChunkMessage, _) => snapshotProcessor.foreach(_ ! msg) + case msg @ DataFromPeer(_: ResponseChunkMessage, _) => snapshotProcessor.foreach(_ ! msg) case msg @ UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! msg diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index ba4c4c8c84..f62007ec6e 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -24,7 +24,7 @@ import encry.nvg.NodeViewHolder.{ UpdateHistoryReader, UpdateInformation } -import encry.nvg.SnapshotProcessorActor.{ +import encry.nvg.SnapshotProcessor.{ FastSyncDone, FastSyncFinished, HeaderChainIsSynced, @@ -33,7 +33,7 @@ import encry.nvg.SnapshotProcessorActor.{ SnapshotChunk, TreeChunks } -import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ManifestId +import encry.nvg.SnapshotProcessor.SnapshotManifest.ManifestId import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag @@ -70,10 +70,9 @@ class NodeViewHolder( var nodeView: NodeView = restoreState().getOrElse(genesisState) - context.parent ! UpdateHistoryReader(HistoryReader(nodeView.history)) - var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] + context.parent ! UpdateHistoryReader(HistoryReader(nodeView.history)) context.parent ! BlockAndHeaderInfo(nodeView.history.getBestHeader, nodeView.history.getBestBlock) context.system.scheduler.schedule(1.seconds, 10.seconds) { diff --git a/src/main/scala/encry/nvg/SnapshotDownloader.scala b/src/main/scala/encry/nvg/SnapshotDownloader.scala new file mode 100644 index 0000000000..408d1473b9 --- /dev/null +++ b/src/main/scala/encry/nvg/SnapshotDownloader.scala @@ -0,0 +1,140 @@ +package encry.nvg + +import akka.actor.{Actor, Cancellable} +import cats.syntax.option._ +import cats.syntax.either._ +import com.typesafe.scalalogging.StrictLogging +import encry.network.BlackList.BanReason.{InvalidChunkMessage, InvalidStateAfterFastSync} +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.PeersKeeper.BanPeer +import encry.nvg.SnapshotProcessor.{CheckDelivery, FastSyncDone, FastSyncFinished, RequestNextChunks, RequiredManifestHeightAndId} +import encry.settings.EncryAppSettings +import encry.view.fast.sync.FastSyncExceptions.{ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage} +import encry.view.fast.sync.{RequestsPerPeriodProcessor, SnapshotDownloadController, SnapshotHolder} +import encry.view.history.{History, HistoryReader} +import encry.view.wallet.EncryWallet +import org.encryfoundation.common.network.BasicMessagesRepo.{RequestChunkMessage, ResponseChunkMessage, ResponseManifestMessage} +import org.encryfoundation.common.utils.Algos + +class SnapshotDownloader(settings: EncryAppSettings) extends Actor with StrictLogging { + + var snapshotHolder: SnapshotHolder = + SnapshotHolder.initialize( + settings, + if (settings.snapshotSettings.enableFastSynchronization) settings.storage.state + else settings.storage.snapshotHolder + ) + var snapshotDownloadController: SnapshotDownloadController = SnapshotDownloadController.empty(settings) + var historyReader: HistoryReader = HistoryReader.empty + + override def receive: Receive = ??? + + def fastSyncMod( + history: History, + responseTimeout: Option[Cancellable] + ): Receive = { + case DataFromPeer(message, remote) => + logger.debug(s"Snapshot holder got from ${remote} message ${message.NetworkMessageTypeID}.") + message match { + case ResponseManifestMessage(manifest) => + logger.info( + s"Got new manifest message ${Algos.encode(manifest.manifestId.toByteArray)} while processing chunks." + ) + case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => + (for { + controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) + (controller, chunk) = controllerAndChunk + validChunk <- snapshotHolder.validateChunkId(chunk) + processor = snapshotHolder.updateCache(validChunk) + newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { + case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] + case t => t.asLeft[SnapshotHolder] + } + } yield (newProcessor, controller)) match { + case Left(err: UnexpectedChunkMessage) => + logger.info(s"Error has occurred ${err.error} with peer $remote") + case Left(error) => + logger.info(s"Error has occurred: $error") + nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) + restartFastSync(history) + case Right((processor, controller)) + if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => + nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) + restartFastSync(history) + case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => + processor.assembleUTXOState() match { + case Right(state) => + logger.info(s"Tree is valid on Snapshot holder!") + processor.wallet.foreach { wallet: EncryWallet => + (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] + } + case _ => + nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) + restartFastSync(history).asLeft[Unit] + } + case Right((processor, controller)) => + snapshotDownloadController = controller + snapshotHolder = processor + if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks + } + + case ResponseChunkMessage(_) => + logger.info(s"Received chunk from unexpected peer ${remote}") + + case _ => + } + + case RequestNextChunks => + responseTimeout.foreach(_.cancel()) + (for { + controllerAndIds <- snapshotDownloadController.getNextBatchAndRemoveItFromController + _ = logger.info(s"Current notYetRequested batches is ${snapshotDownloadController.batchesSize}.") + } yield controllerAndIds) match { + case Left(err) => + logger.info(s"Error has occurred: ${err.error}") + throw new Exception(s"Error has occurred: ${err.error}") + case Right(controllerAndIds) => + snapshotDownloadController = controllerAndIds._1 + controllerAndIds._2.foreach { msg => + // snapshotDownloadController.cp.foreach { peer: PeerConnectionHandler.ConnectedPeer => + // peer.handlerRef ! msg + // } + } + context.become(fastSyncMod(history, timer).orElse(commonMessages)) + } + + case RequiredManifestHeightAndId(height, manifestId) => + logger.info( + s"Snapshot holder while header sync got message RequiredManifestHeight with height $height." + + s"New required manifest id is ${Algos.encode(manifestId)}." + ) + snapshotDownloadController = snapshotDownloadController.copy( + requiredManifestHeight = height, + requiredManifestId = manifestId + ) + restartFastSync(history) + self ! BroadcastManifestRequestMessage + context.become(awaitManifestMod(none, history).orElse(commonMessages)) + + case CheckDelivery => + snapshotDownloadController.awaitedChunks.map { id => + RequestChunkMessage(id.data) + }.foreach { msg => + //snapshotDownloadController.cp.foreach(peer => peer.handlerRef ! msg) + } + context.become(fastSyncMod(history, timer).orElse(commonMessages)) + + case FastSyncDone => + if (settings.snapshotSettings.enableSnapshotCreation) { + logger.info(s"Snapshot holder context.become to snapshot processing") + snapshotHolder = SnapshotHolder.recreateAfterFastSyncIsDone(settings) + snapshotDownloadController.storage.close() + context.system.scheduler + .scheduleOnce(settings.snapshotSettings.updateRequestsPerTime)(self ! DropProcessedCount) + context.become(workMod(history).orElse(commonMessages)) + } else { + logger.info(s"Stop processing snapshots") + context.stop(self) + } + } +} diff --git a/src/main/scala/encry/nvg/SnapshotIntermediary.scala b/src/main/scala/encry/nvg/SnapshotIntermediary.scala new file mode 100644 index 0000000000..9e08c81b36 --- /dev/null +++ b/src/main/scala/encry/nvg/SnapshotIntermediary.scala @@ -0,0 +1,12 @@ +package encry.nvg + +import akka.actor.Actor + +class SnapshotIntermediary extends Actor { + + + + override def receive: Receive = { + case _ => + } +} diff --git a/src/main/scala/encry/nvg/SnapshotProcessorActor.scala b/src/main/scala/encry/nvg/SnapshotProcessor.scala similarity index 83% rename from src/main/scala/encry/nvg/SnapshotProcessorActor.scala rename to src/main/scala/encry/nvg/SnapshotProcessor.scala index c19d543bc6..bc52858361 100644 --- a/src/main/scala/encry/nvg/SnapshotProcessorActor.scala +++ b/src/main/scala/encry/nvg/SnapshotProcessor.scala @@ -2,30 +2,26 @@ package encry.nvg import SnapshotChunkProto.SnapshotChunkMessage import SnapshotManifestProto.SnapshotManifestProtoMessage -import akka.actor.{ Actor, ActorRef, Cancellable, Props } +import akka.actor.{Actor, ActorRef, Cancellable, Props} import cats.syntax.either._ import cats.syntax.option._ import com.google.protobuf.ByteString import com.typesafe.scalalogging.StrictLogging -import encry.network.BlackList.BanReason.{ - InvalidChunkMessage, - InvalidResponseManifestMessage, - InvalidStateAfterFastSync -} +import encry.network.BlackList.BanReason.{InvalidChunkMessage, InvalidResponseManifestMessage, InvalidStateAfterFastSync} import encry.network.Broadcast -import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NodeViewSynchronizer.ReceivableMessages.ChangedHistory -import encry.network.PeersKeeper.{ BanPeer, SendToNetwork } +import encry.network.PeersKeeper.{BanPeer, SendToNetwork} import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier -import encry.nvg.SnapshotProcessorActor.SnapshotManifest.{ ChunkId, ManifestId } -import encry.nvg.SnapshotProcessorActor._ +import encry.nvg.SnapshotProcessor.SnapshotManifest.{ChunkId, ManifestId} +import encry.nvg.SnapshotProcessor._ import encry.settings.EncryAppSettings -import encry.storage.VersionalStorage.{ StorageKey, StorageValue } -import encry.view.fast.sync.FastSyncExceptions.{ ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage } -import encry.view.fast.sync.{ RequestsPerPeriodProcessor, SnapshotDownloadController, SnapshotProcessor } -import encry.view.history.History +import encry.storage.VersionalStorage.{StorageKey, StorageValue} +import encry.view.fast.sync.FastSyncExceptions.{ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage} +import encry.view.fast.sync.{RequestsPerPeriodProcessor, SnapshotDownloadController, SnapshotHolder} +import encry.view.history.{History, HistoryReader} import encry.view.state.UtxoState -import encry.view.state.avlTree.{ Node, NodeSerilalizer } +import encry.view.state.avlTree.{Node, NodeSerilalizer} import encry.view.wallet.EncryWallet import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.network.BasicMessagesRepo._ @@ -34,11 +30,9 @@ import supertagged.TaggedType import scala.util.Try -class SnapshotProcessorActor( +class SnapshotProcessor( settings: EncryAppSettings, - networkController: ActorRef, - nodeViewHolder: ActorRef, - nodeViewSynchronizer: ActorRef + nodeViewHolder: ActorRef ) extends Actor with StrictLogging { @@ -46,33 +40,17 @@ class SnapshotProcessorActor( //todo 1. Add connection agreement (case while peer reconnects with other handler.ref) - var snapshotProcessor: SnapshotProcessor = - SnapshotProcessor.initialize( + var snapshotHolder: SnapshotHolder = + SnapshotHolder.initialize( settings, if (settings.snapshotSettings.enableFastSynchronization) settings.storage.state else settings.storage.snapshotHolder ) var snapshotDownloadController: SnapshotDownloadController = SnapshotDownloadController.empty(settings) var requestsProcessor: RequestsPerPeriodProcessor = RequestsPerPeriodProcessor.empty(settings) + var historyReader: HistoryReader = HistoryReader.empty - override def preStart(): Unit = - if (settings.constants.SnapshotCreationHeight <= settings.constants.MaxRollbackDepth || - (!settings.snapshotSettings.enableFastSynchronization && !settings.snapshotSettings.enableSnapshotCreation)) { - logger.info(s"Stop self(~_~)SnapshotHolder(~_~)") - context.stop(self) - } else { - context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) - logger.info(s"SnapshotHolder started.") - networkController ! RegisterMessagesHandler( - Seq( - RequestManifestMessage.NetworkMessageTypeID -> "RequestManifest", - ResponseManifestMessage.NetworkMessageTypeID -> "ResponseManifestMessage", - RequestChunkMessage.NetworkMessageTypeID -> "RequestChunkMessage", - ResponseChunkMessage.NetworkMessageTypeID -> "ResponseChunkMessage" - ), - self - ) - } + context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) override def receive: Receive = awaitingHistory @@ -227,7 +205,7 @@ class SnapshotProcessorActor( if (isValidManifest && canBeProcessed) { (for { controller <- snapshotDownloadController.processManifest(manifest, remote, history) - processor <- snapshotProcessor.initializeApplicableChunksCache( + processor <- snapshotHolder.initializeApplicableChunksCache( history, snapshotDownloadController.requiredManifestHeight ) @@ -238,7 +216,7 @@ class SnapshotProcessorActor( logger.debug(s"Request manifest message successfully processed.") responseManifestTimeout.foreach(_.cancel()) snapshotDownloadController = controller - snapshotProcessor = processor + snapshotHolder = processor self ! RequestNextChunks logger.debug("Manifest processed successfully.") context.become(fastSyncMod(history, none)) @@ -262,9 +240,9 @@ class SnapshotProcessorActor( def workMod(history: History): Receive = { case TreeChunks(chunks, id) => - val manifestIds: Seq[Array[Byte]] = snapshotProcessor.potentialManifestsIds + val manifestIds: Seq[Array[Byte]] = snapshotHolder.potentialManifestsIds if (!manifestIds.exists(_.sameElements(id))) { - snapshotProcessor.createNewSnapshot(ManifestId @@ id, manifestIds, chunks) + snapshotHolder.createNewSnapshot(ManifestId @@ id, manifestIds, chunks) } else logger.info(s"Doesn't need to create snapshot") case SemanticallySuccessfulModifier(block: Block) if history.isFullChainSynced => @@ -272,10 +250,10 @@ class SnapshotProcessorActor( val condition: Int = (block.header.height - settings.constants.MaxRollbackDepth) % settings.constants.SnapshotCreationHeight logger.info(s"condition = $condition") - if (condition == 0) snapshotProcessor.processNewBlock(block, history) match { + if (condition == 0) snapshotHolder.processNewBlock(block, history) match { case Left(_) => case Right(newProcessor) => - snapshotProcessor = newProcessor + snapshotHolder = newProcessor requestsProcessor = RequestsPerPeriodProcessor.empty(settings) nodeViewHolder ! RemoveRedundantManifestIds } @@ -283,8 +261,8 @@ class SnapshotProcessorActor( case DataFromPeer(message, remote) => message match { case RequestManifestMessage(requiredManifestId) - if requestsProcessor.canBeProcessed(snapshotProcessor, requiredManifestId) => - snapshotProcessor.actualManifest.foreach { m => + if requestsProcessor.canBeProcessed(snapshotHolder, requiredManifestId) => + snapshotHolder.actualManifest.foreach { m => logger.info(s"Sent to remote actual manifest with id ${Algos.encode(requiredManifestId)}") //remote.handlerRef ! ResponseManifestMessage(SnapshotManifestSerializer.toProto(m)) } @@ -294,7 +272,7 @@ class SnapshotProcessorActor( //if requestsProcessor.canProcessRequest(remote) => logger.debug(s"Got RequestChunkMessage. Current handledRequests ${requestsProcessor.handledRequests}.") - val chunkFromDB: Option[SnapshotChunkMessage] = snapshotProcessor.getChunkById(chunkId) + val chunkFromDB: Option[SnapshotChunkMessage] = snapshotHolder.getChunkById(chunkId) chunkFromDB.foreach { chunk => logger.debug(s"Sent to $remote chunk $chunk.") val networkMessage: NetworkMessage = ResponseChunkMessage(chunk) @@ -318,14 +296,14 @@ class SnapshotProcessorActor( def restartFastSync(history: History): Unit = { logger.info(s"Restart fast sync!") snapshotDownloadController = snapshotDownloadController.reInitFastSync - snapshotProcessor = snapshotProcessor.reInitStorage + snapshotHolder = snapshotHolder.reInitStorage } def timer: Option[Cancellable] = context.system.scheduler.scheduleOnce(settings.snapshotSettings.responseTimeout)(self ! CheckDelivery).some } -object SnapshotProcessorActor { +object SnapshotProcessor { case object RemoveRedundantManifestIds @@ -388,11 +366,9 @@ object SnapshotProcessorActor { ) } - def props(settings: EncryAppSettings, - networkController: ActorRef, - nodeViewHolderRef: ActorRef, - nodeViewSynchronizer: ActorRef): Props = Props( - new SnapshotProcessorActor(settings, networkController, nodeViewHolderRef, nodeViewSynchronizer) - ) + def props(settings: EncryAppSettings, nodeViewHolderRef: ActorRef): Props = + Props( + new SnapshotProcessor(settings, nodeViewHolderRef) + ) } diff --git a/src/main/scala/encry/view/fast/sync/FastSyncExceptions.scala b/src/main/scala/encry/view/fast/sync/FastSyncExceptions.scala index 2753064729..fd7d628477 100644 --- a/src/main/scala/encry/view/fast/sync/FastSyncExceptions.scala +++ b/src/main/scala/encry/view/fast/sync/FastSyncExceptions.scala @@ -24,7 +24,7 @@ object FastSyncExceptions { sealed trait SnapshotDownloadControllerException extends FastSyncException final case class InvalidManifestBytes(error: String) extends SnapshotDownloadControllerException - final case class ApplicableChunkIsAbsent(error: String, processor: SnapshotProcessor) extends FastSyncException + final case class ApplicableChunkIsAbsent(error: String, processor: SnapshotHolder) extends FastSyncException final case class BestHeaderAtHeightIsAbsent(error: String) extends FastSyncException final case class InitializeHeightAndRootKeysException(error: String) extends FastSyncException final case class ChunksIdsToDownloadException(error: String) extends FastSyncException diff --git a/src/main/scala/encry/view/fast/sync/RequestsPerPeriodProcessor.scala b/src/main/scala/encry/view/fast/sync/RequestsPerPeriodProcessor.scala index df3d6f393c..0ddb3dc64c 100644 --- a/src/main/scala/encry/view/fast/sync/RequestsPerPeriodProcessor.scala +++ b/src/main/scala/encry/view/fast/sync/RequestsPerPeriodProcessor.scala @@ -7,7 +7,7 @@ import org.encryfoundation.common.utils.Algos final case class RequestsPerPeriodProcessor(handledRequests: Int, settings: EncryAppSettings) extends StrictLogging { - def canBeProcessed(processor: SnapshotProcessor, manifestId: Array[Byte]): Boolean = { + def canBeProcessed(processor: SnapshotHolder, manifestId: Array[Byte]): Boolean = { val actualManifestID: Option[Array[Byte]] = processor.actualManifestId logger.info(s"Requested id ${Algos.encode(manifestId)}, current manifest id ${actualManifestID.map(Algos.encode)}.") actualManifestID.exists(_.sameElements(manifestId)) diff --git a/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala b/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala index 588752bda5..d2635d4f21 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala @@ -8,8 +8,8 @@ import SnapshotManifestProto.SnapshotManifestProtoMessage import cats.syntax.either._ import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ChunkId -import encry.nvg.SnapshotProcessorActor.{ SnapshotChunk, SnapshotChunkSerializer, SnapshotManifestSerializer } +import encry.nvg.SnapshotProcessor.SnapshotManifest.ChunkId +import encry.nvg.SnapshotProcessor.{ SnapshotChunk, SnapshotChunkSerializer, SnapshotManifestSerializer } import encry.settings.EncryAppSettings import encry.storage.levelDb.versionalLevelDB.LevelDbFactory import encry.view.fast.sync.FastSyncExceptions._ diff --git a/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala b/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala index b0ec686518..79d3b4d1df 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala @@ -1,7 +1,7 @@ package encry.view.fast.sync import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ChunkId +import encry.nvg.SnapshotProcessor.SnapshotManifest.ChunkId import encry.settings.EncryAppSettings import org.encryfoundation.common.utils.Algos import org.iq80.leveldb.DB diff --git a/src/main/scala/encry/view/fast/sync/SnapshotProcessor.scala b/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala similarity index 91% rename from src/main/scala/encry/view/fast/sync/SnapshotProcessor.scala rename to src/main/scala/encry/view/fast/sync/SnapshotHolder.scala index 07d25cc879..ef047908a7 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotProcessor.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala @@ -7,13 +7,13 @@ import cats.syntax.either._ import cats.syntax.option._ import com.google.common.primitives.Ints import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessorActor.{ +import encry.nvg.SnapshotProcessor.{ SnapshotChunk, SnapshotChunkSerializer, SnapshotManifest, SnapshotManifestSerializer } -import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ManifestId +import encry.nvg.SnapshotProcessor.SnapshotManifest.ManifestId import encry.settings.EncryAppSettings import encry.storage.{ RootNodesStorage, VersionalStorage } import encry.storage.VersionalStorage.{ StorageKey, StorageType, StorageValue, StorageVersion } @@ -38,7 +38,7 @@ import scala.collection.immutable.{ HashMap, HashSet } import scala.language.postfixOps import scala.util.{ Failure, Success } -final case class SnapshotProcessor( +final case class SnapshotHolder( settings: EncryAppSettings, storage: VersionalStorage, applicableChunks: HashSet[ByteArrayWrapper], @@ -48,24 +48,24 @@ final case class SnapshotProcessor( with SnapshotProcessorStorageAPI with AutoCloseable { - def updateCache(chunk: SnapshotChunk): SnapshotProcessor = + def updateCache(chunk: SnapshotChunk): SnapshotHolder = this.copy(chunksCache = chunksCache.updated(ByteArrayWrapper(chunk.id), chunk)) - def initializeApplicableChunksCache(history: History, height: Int): Either[FastSyncException, SnapshotProcessor] = + def initializeApplicableChunksCache(history: History, height: Int): Either[FastSyncException, SnapshotHolder] = for { stateRoot <- Either.fromOption( history.getBestHeaderAtHeight(height).map(_.stateRoot), BestHeaderAtHeightIsAbsent(s"There is no best header at required height $height") ) - processor: SnapshotProcessor = this.copy(applicableChunks = HashSet(ByteArrayWrapper(stateRoot))) + processor: SnapshotHolder = this.copy(applicableChunks = HashSet(ByteArrayWrapper(stateRoot))) resultedProcessor <- processor.initializeHeightAndRootKeys(stateRoot, height) match { case Left(error) => - InitializeHeightAndRootKeysException(error.getMessage).asLeft[SnapshotProcessor] + InitializeHeightAndRootKeysException(error.getMessage).asLeft[SnapshotHolder] case Right(newProcessor) => newProcessor.asRight[FastSyncException] } } yield resultedProcessor - private def initializeHeightAndRootKeys(rootNodeId: Array[Byte], height: Int): Either[Throwable, SnapshotProcessor] = + private def initializeHeightAndRootKeys(rootNodeId: Array[Byte], height: Int): Either[Throwable, SnapshotHolder] = Either.catchNonFatal { storage.insert( StorageVersion @@ Random.randomBytes(), @@ -75,7 +75,7 @@ final case class SnapshotProcessor( this } - def reInitStorage: SnapshotProcessor = + def reInitStorage: SnapshotHolder = try { storage.close() wallet.foreach(_.close()) @@ -84,7 +84,7 @@ final case class SnapshotProcessor( import org.apache.commons.io.FileUtils FileUtils.deleteDirectory(stateDir) FileUtils.deleteDirectory(walletDir) - SnapshotProcessor.initialize(settings, settings.storage.state) + SnapshotHolder.initialize(settings, settings.storage.state) } catch { case err: Throwable => throw new Exception(s"Exception ${err.getMessage} has occurred while restarting fast sync process") @@ -98,7 +98,7 @@ final case class SnapshotProcessor( case emptyNode: EmptyNode[StorageKey, StorageValue] => List.empty[Node[StorageKey, StorageValue]] } - def applyChunk(chunk: SnapshotChunk): Either[ChunkApplyError, SnapshotProcessor] = { + def applyChunk(chunk: SnapshotChunk): Either[ChunkApplyError, SnapshotHolder] = { val kSerializer: Serializer[StorageKey] = implicitly[Serializer[StorageKey]] val vSerializer: Serializer[StorageValue] = implicitly[Serializer[StorageValue]] val nodes: List[Node[StorageKey, StorageValue]] = flatten(chunk.node) @@ -154,7 +154,7 @@ final case class SnapshotProcessor( val newApplicableChunk = (applicableChunks -- toStorage.map(node => ByteArrayWrapper(node.hash))) ++ toApplicable.map(node => ByteArrayWrapper(node.hash)) this.copy(applicableChunks = newApplicableChunk).asRight[ChunkApplyError] - case Left(exception) => ChunkApplyError(exception.getMessage).asLeft[SnapshotProcessor] + case Left(exception) => ChunkApplyError(exception.getMessage).asLeft[SnapshotHolder] } } @@ -165,7 +165,7 @@ final case class SnapshotProcessor( s"Node hash:(${Algos.encode(chunk.node.hash)}) doesn't equal to chunk id:(${Algos.encode(chunk.id)})" ).asLeft[SnapshotChunk] - private def getNextApplicableChunk: Either[FastSyncException, (SnapshotChunk, SnapshotProcessor)] = + private def getNextApplicableChunk: Either[FastSyncException, (SnapshotChunk, SnapshotHolder)] = for { idAndChunk <- Either.fromOption(chunksCache.find { case (id, _) => applicableChunks.contains(id) }, ApplicableChunkIsAbsent("There are no applicable chunks in cache", this)) @@ -176,7 +176,7 @@ final case class SnapshotProcessor( (chunk, this.copy(chunksCache = newChunksCache)) } - def processNextApplicableChunk(snapshotProcessor: SnapshotProcessor): Either[FastSyncException, SnapshotProcessor] = + def processNextApplicableChunk(snapshotProcessor: SnapshotHolder): Either[FastSyncException, SnapshotHolder] = for { chunkAndProcessor <- snapshotProcessor.getNextApplicableChunk (chunk, processor) = chunkAndProcessor @@ -214,7 +214,7 @@ final case class SnapshotProcessor( node <- getNode(rootNodeId) } yield node - def processNewBlock(block: Block, history: History): Either[ProcessNewBlockError, SnapshotProcessor] = { + def processNewBlock(block: Block, history: History): Either[ProcessNewBlockError, SnapshotHolder] = { logger.info( s"Start updating actual manifest to new one at height " + s"${block.header.height} with block id ${block.encodedId}." @@ -226,7 +226,7 @@ final case class SnapshotProcessor( id: ManifestId, manifestIds: Seq[Array[Byte]], newChunks: List[SnapshotChunk] - ): Either[ProcessNewSnapshotError, SnapshotProcessor] = { + ): Either[ProcessNewSnapshotError, SnapshotHolder] = { //todo add only exists chunks val manifest: SnapshotManifest = SnapshotManifest(id, newChunks.map(_.id)) val snapshotToDB: List[(StorageKey, StorageValue)] = newChunks.map { elem => @@ -242,12 +242,12 @@ final case class SnapshotProcessor( val toApply: List[(StorageKey, StorageValue)] = manifestToDB :: updateList :: snapshotToDB logger.info(s"A new snapshot created successfully. Insertion started.") Either.catchNonFatal(storage.insert(StorageVersion @@ Random.randomBytes(), toApply, List.empty)) match { - case Left(value) => ProcessNewSnapshotError(value.getMessage).asLeft[SnapshotProcessor] + case Left(value) => ProcessNewSnapshotError(value.getMessage).asLeft[SnapshotHolder] case Right(_) => this.asRight[ProcessNewSnapshotError] } } - private def updateActualSnapshot(history: History, height: Int): Either[ProcessNewBlockError, SnapshotProcessor] = + private def updateActualSnapshot(history: History, height: Int): Either[ProcessNewBlockError, SnapshotHolder] = for { bestManifestId <- Either.fromOption( history.getBestHeaderAtHeight(height).map(header => Algos.hash(header.stateRoot ++ header.id)), @@ -269,7 +269,7 @@ final case class SnapshotProcessor( Either.catchNonFatal( storage.insert(StorageVersion @@ Random.randomBytes(), List(toApply), toDelete.map(StorageKey @@ _)) ) match { - case Left(error) => ProcessNewBlockError(error.getMessage).asLeft[SnapshotProcessor] + case Left(error) => ProcessNewBlockError(error.getMessage).asLeft[SnapshotHolder] case Right(_) => this.asRight[ProcessNewBlockError] } } @@ -278,15 +278,15 @@ final case class SnapshotProcessor( override def close(): Unit = storage.close() } -object SnapshotProcessor extends StrictLogging { +object SnapshotHolder extends StrictLogging { - def initialize(settings: EncryAppSettings, storageType: StorageType): SnapshotProcessor = + def initialize(settings: EncryAppSettings, storageType: StorageType): SnapshotHolder = if (settings.snapshotSettings.enableFastSynchronization) create(settings, new File(s"${settings.directory}/state"), storageType) else create(settings, getDirProcessSnapshots(settings), storageType) - def recreateAfterFastSyncIsDone(settings: EncryAppSettings): SnapshotProcessor = { + def recreateAfterFastSyncIsDone(settings: EncryAppSettings): SnapshotHolder = { val snapshotStorage = getDirProcessSnapshots(settings) snapshotStorage.mkdirs() val storage: VersionalStorage = @@ -299,12 +299,12 @@ object SnapshotProcessor extends StrictLogging { val levelDBInit: DB = LevelDbFactory.factory.open(snapshotStorage, new Options) VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settings.levelDB, keySize = 32)) } - new SnapshotProcessor(settings, storage, HashSet.empty, HashMap.empty, none[EncryWallet]) + new SnapshotHolder(settings, storage, HashSet.empty, HashMap.empty, none[EncryWallet]) } def getDirProcessSnapshots(settings: EncryAppSettings): File = new File(s"${settings.directory}/snapshots") - def create(settings: EncryAppSettings, snapshotsDir: File, storageType: StorageType): SnapshotProcessor = { + def create(settings: EncryAppSettings, snapshotsDir: File, storageType: StorageType): SnapshotHolder = { snapshotsDir.mkdirs() val storage: VersionalStorage = storageType match { @@ -328,6 +328,6 @@ object SnapshotProcessor extends StrictLogging { .some else none[EncryWallet] - new SnapshotProcessor(settings, storage, HashSet.empty, HashMap.empty, wallet) + new SnapshotHolder(settings, storage, HashSet.empty, HashMap.empty, wallet) } } diff --git a/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala b/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala index c0f868cbb3..f52505f78c 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala @@ -3,7 +3,7 @@ package encry.view.fast.sync import SnapshotChunkProto.SnapshotChunkMessage import SnapshotManifestProto.SnapshotManifestProtoMessage import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessorActor.{ SnapshotManifest, SnapshotManifestSerializer } +import encry.nvg.SnapshotProcessor.{ SnapshotManifest, SnapshotManifestSerializer } import encry.storage.VersionalStorage import encry.storage.VersionalStorage.{ StorageKey, StorageValue } import org.encryfoundation.common.utils.Algos diff --git a/src/main/scala/encry/view/state/avlTree/AvlTree.scala b/src/main/scala/encry/view/state/avlTree/AvlTree.scala index 983cd6317a..9b97e83d45 100644 --- a/src/main/scala/encry/view/state/avlTree/AvlTree.scala +++ b/src/main/scala/encry/view/state/avlTree/AvlTree.scala @@ -4,8 +4,8 @@ import cats.syntax.order._ import cats.{Monoid, Order} import com.google.common.primitives.Ints import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessorActor.SnapshotChunk -import encry.nvg.SnapshotProcessorActor.SnapshotManifest.ChunkId +import encry.nvg.SnapshotProcessor.SnapshotChunk +import encry.nvg.SnapshotProcessor.SnapshotManifest.ChunkId import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} import encry.storage.{RootNodesStorage, VersionalStorage} import encry.view.state.UtxoState From abb6008c2fff3c84d9c429b48cf6e220c35619db Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 11:03:51 +0300 Subject: [PATCH 049/177] add processing getPeers message on pk --- src/main/scala/encry/network/DM.scala | 6 ++- .../scala/encry/network/MessageBuilder.scala | 44 ++++++++++++------- src/main/scala/encry/network/Messages.scala | 1 + .../scala/encry/network/NetworkRouter.scala | 5 ++- src/main/scala/encry/network/PK.scala | 7 ++- 5 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 965e62d027..2b5e86a237 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -4,7 +4,7 @@ import java.net.InetSocketAddress import akka.actor.{Actor, Props} import com.typesafe.scalalogging.StrictLogging -import encry.network.DM.{AwaitingRequest, RequestSent} +import encry.network.DM.{AwaitingRequest, IsRequested, RequestSent} import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.RegisterMessagesHandler import encry.network.NetworkRouter.ModifierFromNetwork @@ -47,6 +47,9 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging } else logger.info(s"Peer $source sent spam mod of type $modTypeId and id ${Algos.encode(modId)}") case SemanticallySuccessfulModifier(mod) => receivedModifier -= toKey(mod.id) case SemanticallyFailedModification(mod, _) => receivedModifier -= toKey(mod.id) + case IsRequested(modId) => + logger.info(s"Going to check if ${Algos.encode(modId)} has been requested. Res: ${receivedModifier.contains(toKey(modId))}") + sender() ! receivedModifier.contains(toKey(modId)) } def toKey(id: ModifierId): ModifierIdAsKey = new mutable.WrappedArray.ofByte(id) @@ -57,6 +60,5 @@ object DM { case class AwaitingRequest(peer: InetSocketAddress, modTypeId: ModifierTypeId, modId: ModifierId, attempts: Int) case class RequestSent(peer: InetSocketAddress, modTypeId: ModifierTypeId, modId: ModifierId) case class IsRequested(modifierId: ModifierId) - def props(networkSettings: NetworkSettings): Props = Props(new DM(networkSettings)) } diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 171ac31988..3ee27506b9 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -11,16 +11,16 @@ import encry.network.ConnectedPeersCollection.PeerInfo import encry.network.DM.{IsRequested, RequestSent} import encry.network.MessageBuilder.{GetPeerInfo, GetPeerWithEqualHistory, GetPeerWithOlderHistory, GetPeers} import encry.network.Messages.MessageToNetwork -import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} +import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendPeers, SendSyncInfo} import encry.network.PeerConnectionHandler.ConnectedPeer -import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, PeersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} import org.encryfoundation.common.utils.Algos +import scala.concurrent.Await import scala.concurrent.duration._ import scala.util.Try -case class MessageBuilder(msg: MessageToNetwork, - peersKeeper: ActorRef, +case class MessageBuilder(peersKeeper: ActorRef, deliveryManager: ActorRef) extends Actor with StrictLogging { import context.dispatcher @@ -31,13 +31,21 @@ case class MessageBuilder(msg: MessageToNetwork, case RequestFromLocal(Some(peer), modTypeId, modsIds) => Try { (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].foreach { peer => + logger.info(s"Going to req mods from ${peer.socketAddress}") modsIds.foreach { modId => - for { - isRequested <- (deliveryManager ? IsRequested(modId)).mapTo[Boolean] - } yield if (!isRequested) { - peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) - deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId) - } else logger.debug(s"Duplicate request for modifier of type ${modTypeId} and id: ${Algos.encode(modId)}") + val res123 = (deliveryManager ? IsRequested(modId)) + //Await.result(res123, 2 minutes) + logger.info(s"res: ${res123.getClass}") + res123.mapTo[Boolean].foreach { res => + if (res) logger.info(s"Another res: ${res}") + } + (deliveryManager ? IsRequested(modId)).mapTo[Boolean].foreach { isRequested => + logger.info(s"isRequested: ${isRequested}") + if (!isRequested) { + peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) + deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId) + } else logger.debug(s"Duplicate request for modifier of type ${modTypeId} and id: ${Algos.encode(modId)}") + } } } } @@ -48,7 +56,7 @@ case class MessageBuilder(msg: MessageToNetwork, modsIds.foreach { modId => for { isRequested <- (deliveryManager ? IsRequested(modId)).mapTo[Boolean] - } yield if (!isRequested) { + } yield if (isRequested) { peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId) } else logger.debug(s"Duplicate request for modifier of type ${modTypeId} and id: ${Algos.encode(modId)}") @@ -58,7 +66,7 @@ case class MessageBuilder(msg: MessageToNetwork, context.stop(self) case SendSyncInfo(syncInfo) => (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => - peers.foreach(_.handlerRef ! syncInfo) + peers.foreach(_.handlerRef ! SyncInfoNetworkMessage(syncInfo)) } context.stop(self) case ResponseFromLocal(peer, modTypeId, modsIds) => @@ -73,6 +81,13 @@ case class MessageBuilder(msg: MessageToNetwork, peers.foreach(_.handlerRef ! InvNetworkMessage(modTypeId -> List(modInfo))) } context.stop(self) + case SendPeers(peers, remote) => + Try { + (peersKeeper ? GetPeerInfo(remote)).mapTo[ConnectedPeer].map { peer => + peer.handlerRef ! PeersNetworkMessage(peers) + } + } + } } @@ -102,7 +117,6 @@ object MessageBuilder { override def predicate: PeerInfo => Boolean = (info: PeerInfo) => info.historyComparisonResult == Younger } - def props(msg: MessageToNetwork, - peersKeeper: ActorRef, - deliveryManager: ActorRef): Props = Props(new MessageBuilder(msg, peersKeeper, deliveryManager)) + def props(peersKeeper: ActorRef, + deliveryManager: ActorRef): Props = Props(new MessageBuilder(peersKeeper, deliveryManager)) } \ No newline at end of file diff --git a/src/main/scala/encry/network/Messages.scala b/src/main/scala/encry/network/Messages.scala index 10f881290e..a66b748415 100644 --- a/src/main/scala/encry/network/Messages.scala +++ b/src/main/scala/encry/network/Messages.scala @@ -17,5 +17,6 @@ object Messages { modifiers: Map[ModifierId, Array[Byte]]) extends MessageToNetwork final case class BroadcastModifier(modifierTypeId: ModifierTypeId, modifierId: ModifierId) extends MessageToNetwork + final case class SendPeers(peers: List[InetSocketAddress], to: InetSocketAddress) extends MessageToNetwork } } diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 8d95e93c67..6d2b77b758 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -67,7 +67,10 @@ class NetworkRouter(settings: NetworkSettings, case msg: ModifierFromNetwork => handlerForMods ! msg case msg: OtherNodeSyncingStatus => peersKeeper ! msg case msg: MessageToNetwork => - context.actorOf(MessageBuilder.props(msg, peersKeeper, deliveryManager), s"messageBuilder${Random.nextInt()}") + context.actorOf( + MessageBuilder.props(peersKeeper, deliveryManager), + s"messageBuilder${Random.nextInt()}" + ) ! msg } def peersLogic: Receive = { diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 6f0067b68e..8a87c7cb19 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -8,6 +8,7 @@ import encry.network.BlackList.BanReason.SentPeersMessageWithoutRequest import encry.network.BlackList.{BanReason, BanTime, BanType} import encry.network.ConnectedPeersCollection.PeerInfo import encry.network.MessageBuilder.{GetPeerByPredicate, GetPeerInfo, GetPeers} +import encry.network.Messages.MessageToNetwork.SendPeers import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection @@ -157,12 +158,10 @@ class PK(networkSettings: NetworkSettings, else add.getAddress.isSiteLocalAddress && add != remote }.getOrElse(false) - val peers: Seq[InetSocketAddress] = connectedPeers.collect(findPeersForRemote, getPeersForRemote) + val peers: List[InetSocketAddress] = connectedPeers.collect(findPeersForRemote, getPeersForRemote).toList logger.info(s"Got request for local known peers. Sending to: $remote peers: ${peers.mkString(",")}.") logger.info(s"Remote is side local: ${remote} : ${Try(remote.getAddress.isSiteLocalAddress)}") - connectedPeers.getAll.find(_._1 == remote).foreach { - case (_, info) => info.connectedPeer.handlerRef ! PeersNetworkMessage(peers) - } + context.parent ! SendPeers(peers, remote) } } From 9ec8d6e59d20c1f7e120fe9db121d73faf292e77 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 13:35:14 +0300 Subject: [PATCH 050/177] simple pipeline of mods downloading --- src/main/scala/encry/network/DM.scala | 26 ++++++++++--- .../scala/encry/network/MessageBuilder.scala | 39 ++++++------------- .../scala/encry/network/NetworkRouter.scala | 2 + 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 2b5e86a237..3a938ce0bc 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -3,10 +3,11 @@ package encry.network import java.net.InetSocketAddress import akka.actor.{Actor, Props} +import akka.pattern._ import com.typesafe.scalalogging.StrictLogging -import encry.network.DM.{AwaitingRequest, IsRequested, RequestSent} +import encry.network.DM.{AwaitingRequest, IsRequested, RequestSent, RequestStatus} import encry.network.Messages.MessageToNetwork.RequestFromLocal -import encry.network.NetworkController.ReceivableMessages.RegisterMessagesHandler +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NetworkRouter.ModifierFromNetwork import encry.settings.NetworkSettings import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage @@ -16,6 +17,7 @@ import cats.syntax.option._ import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier} import scala.collection.mutable +import scala.concurrent.Future case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging { @@ -30,6 +32,14 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging context.parent ! RegisterMessagesHandler(Seq(ModifiersNetworkMessage.NetworkMessageTypeID -> "ModifiersNetworkMessage"), self) override def receive: Receive = { + case DataFromPeer(ModifiersNetworkMessage(data), source) => + data._2.foreach { case (id, bytes) => + if (expectedModifiers.contains(toKey(id))) { + context.parent ! ModifierFromNetwork(source, data._1, id, bytes) + expectedModifiers -= toKey(id) + receivedModifier += toKey(id) + } else logger.info("Receive spam!") + } case RequestSent(peer, modTypeId, modId) => expectedModifiers += toKey(modId) context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! @@ -47,9 +57,12 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging } else logger.info(s"Peer $source sent spam mod of type $modTypeId and id ${Algos.encode(modId)}") case SemanticallySuccessfulModifier(mod) => receivedModifier -= toKey(mod.id) case SemanticallyFailedModification(mod, _) => receivedModifier -= toKey(mod.id) - case IsRequested(modId) => - logger.info(s"Going to check if ${Algos.encode(modId)} has been requested. Res: ${receivedModifier.contains(toKey(modId))}") - sender() ! receivedModifier.contains(toKey(modId)) + case IsRequested(modIds) => + //logger.info(s"Going to check if ${Algos.encode(modId)} has been requested. Res: ${receivedModifier.contains(toKey(modId))}") + sender ! RequestStatus( + modIds.filter(id => receivedModifier.contains(toKey(id)) || expectedModifiers.contains(toKey(id))), + modIds.filter(id => !receivedModifier.contains(toKey(id)) && !expectedModifiers.contains(toKey(id))) + ) } def toKey(id: ModifierId): ModifierIdAsKey = new mutable.WrappedArray.ofByte(id) @@ -59,6 +72,7 @@ object DM { case class AwaitingRequest(peer: InetSocketAddress, modTypeId: ModifierTypeId, modId: ModifierId, attempts: Int) case class RequestSent(peer: InetSocketAddress, modTypeId: ModifierTypeId, modId: ModifierId) - case class IsRequested(modifierId: ModifierId) + case class IsRequested(modifiersId: List[ModifierId]) + case class RequestStatus(requested: List[ModifierId], notRequested: List[ModifierId]) def props(networkSettings: NetworkSettings): Props = Props(new DM(networkSettings)) } diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 3ee27506b9..7740ff4b92 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -8,15 +8,13 @@ import akka.util.Timeout import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.{Equal, Older, Younger} import encry.network.ConnectedPeersCollection.PeerInfo -import encry.network.DM.{IsRequested, RequestSent} -import encry.network.MessageBuilder.{GetPeerInfo, GetPeerWithEqualHistory, GetPeerWithOlderHistory, GetPeers} -import encry.network.Messages.MessageToNetwork +import encry.network.DM.{IsRequested, RequestSent, RequestStatus} +import encry.network.MessageBuilder.{GetPeerInfo, GetPeerWithEqualHistory, GetPeerWithOlderHistory, GetPeers, MsgSent} import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendPeers, SendSyncInfo} import encry.network.PeerConnectionHandler.ConnectedPeer import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, PeersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} import org.encryfoundation.common.utils.Algos -import scala.concurrent.Await import scala.concurrent.duration._ import scala.util.Try @@ -30,36 +28,22 @@ case class MessageBuilder(peersKeeper: ActorRef, override def receive: Receive = { case RequestFromLocal(Some(peer), modTypeId, modsIds) => Try { - (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].foreach { peer => + (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].map { peer => logger.info(s"Going to req mods from ${peer.socketAddress}") - modsIds.foreach { modId => - val res123 = (deliveryManager ? IsRequested(modId)) - //Await.result(res123, 2 minutes) - logger.info(s"res: ${res123.getClass}") - res123.mapTo[Boolean].foreach { res => - if (res) logger.info(s"Another res: ${res}") - } - (deliveryManager ? IsRequested(modId)).mapTo[Boolean].foreach { isRequested => - logger.info(s"isRequested: ${isRequested}") - if (!isRequested) { - peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) - deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId) - } else logger.debug(s"Duplicate request for modifier of type ${modTypeId} and id: ${Algos.encode(modId)}") - } + (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => + peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> status.notRequested) + modsIds.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) + context.parent ! MsgSent(RequestModifiersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } } } - context.stop(self) case RequestFromLocal(None, modTypeId, modsIds) => Try { (peersKeeper ? (GetPeerWithOlderHistory() || GetPeerWithEqualHistory())).mapTo[ConnectedPeer].foreach { peer => - modsIds.foreach { modId => - for { - isRequested <- (deliveryManager ? IsRequested(modId)).mapTo[Boolean] - } yield if (isRequested) { - peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> modsIds) - deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId) - } else logger.debug(s"Duplicate request for modifier of type ${modTypeId} and id: ${Algos.encode(modId)}") + (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => + peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> status.notRequested) + modsIds.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) + context.parent ! MsgSent(RequestModifiersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } } } @@ -94,6 +78,7 @@ case class MessageBuilder(peersKeeper: ActorRef, object MessageBuilder { case object GetPeers + case class MsgSent(msgType: Byte, receiver: InetSocketAddress) case class GetPeerInfo(peerIp: InetSocketAddress) trait GetPeerByPredicate { diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 6d2b77b758..a379da3e75 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -9,6 +9,7 @@ import akka.io.Tcp.SO.KeepAlive import com.typesafe.scalalogging.StrictLogging import encry.network.BlackList.BanReason.InvalidNetworkMessage import encry.network.Messages.MessageToNetwork +import encry.network.MessageBuilder.{GetPeerInfo, GetPeerWithEqualHistory, GetPeerWithOlderHistory, GetPeers, MsgSent} import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus @@ -43,6 +44,7 @@ class NetworkRouter(settings: NetworkSettings, logger.info(s"Registering handlers for ${types.mkString(",")}.") val ids = types.map(_._1) messagesHandlers += (ids -> handler) + case msg: MsgSent => context.stop(sender()) case CommandFailed(cmd: Tcp.Command) => logger.info(s"Failed to execute: $cmd.") case RegisterForModsHandling => handlerForMods = sender() case msg => logger.warn(s"NetworkController: got something strange $msg.") From 72220c3fb74992b84a1f717a1fc2d67cfd5618c0 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 13:39:44 +0300 Subject: [PATCH 051/177] snapshot processor frame --- src/main/scala/encry/network/Messages.scala | 1 + src/main/scala/encry/nvg/NodeViewHolder.scala | 15 - .../scala/encry/nvg/SnapshotDownloader.scala | 285 +++++++++------ .../scala/encry/nvg/SnapshotProcessor.scala | 336 +++++++++--------- .../encry/view/history/HistoryReader.scala | 13 + 5 files changed, 357 insertions(+), 293 deletions(-) diff --git a/src/main/scala/encry/network/Messages.scala b/src/main/scala/encry/network/Messages.scala index a66b748415..a9303937eb 100644 --- a/src/main/scala/encry/network/Messages.scala +++ b/src/main/scala/encry/network/Messages.scala @@ -18,5 +18,6 @@ object Messages { final case class BroadcastModifier(modifierTypeId: ModifierTypeId, modifierId: ModifierId) extends MessageToNetwork final case class SendPeers(peers: List[InetSocketAddress], to: InetSocketAddress) extends MessageToNetwork + final case class BroadcastManifestRequest(manifestId: Array[Byte]) extends MessageToNetwork } } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index f62007ec6e..a7c6fd5a01 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -339,21 +339,6 @@ class NodeViewHolder( case _: Header => true case _: Payload => false }, success = true) - if (historyBeforeStUpdate.fastSyncInProgress.fastSyncVal && - modifier.modifierTypeId == Payload.modifierTypeId && - historyBeforeStUpdate.getBestBlockHeight >= historyBeforeStUpdate.lastAvailableManifestHeight) { - logger.info( - s"getBestBlockHeight ${historyBeforeStUpdate.getBestBlockHeight}. " + - s"heightOfLastAvailablePayloadForRequest ${historyBeforeStUpdate.lastAvailableManifestHeight}" - ) - historyBeforeStUpdate.getBestHeaderAtHeight(historyBeforeStUpdate.lastAvailableManifestHeight).foreach { - h: Header => - context.parent ! RequiredManifestHeightAndId( - historyBeforeStUpdate.lastAvailableManifestHeight, - Algos.hash(h.stateRoot ++ h.id) - ) - } - } logger.info( s"Going to apply modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} to the state. " + s"Progress info is: $progressInfo." diff --git a/src/main/scala/encry/nvg/SnapshotDownloader.scala b/src/main/scala/encry/nvg/SnapshotDownloader.scala index 408d1473b9..902bba67c3 100644 --- a/src/main/scala/encry/nvg/SnapshotDownloader.scala +++ b/src/main/scala/encry/nvg/SnapshotDownloader.scala @@ -1,23 +1,32 @@ package encry.nvg -import akka.actor.{Actor, Cancellable} -import cats.syntax.option._ +import akka.actor.{ Actor, Cancellable } import cats.syntax.either._ +import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging -import encry.network.BlackList.BanReason.{InvalidChunkMessage, InvalidStateAfterFastSync} +import encry.network.BlackList.BanReason.{ InvalidChunkMessage, InvalidStateAfterFastSync } +import encry.network.Messages.MessageToNetwork.BroadcastManifestRequest import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.PeersKeeper.BanPeer -import encry.nvg.SnapshotProcessor.{CheckDelivery, FastSyncDone, FastSyncFinished, RequestNextChunks, RequiredManifestHeightAndId} +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.nvg.SnapshotProcessor._ import encry.settings.EncryAppSettings -import encry.view.fast.sync.FastSyncExceptions.{ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage} -import encry.view.fast.sync.{RequestsPerPeriodProcessor, SnapshotDownloadController, SnapshotHolder} -import encry.view.history.{History, HistoryReader} +import encry.view.fast.sync.FastSyncExceptions.{ ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage } +import encry.view.fast.sync.{ SnapshotDownloadController, SnapshotHolder } +import encry.view.history.{ History, HistoryReader } import encry.view.wallet.EncryWallet -import org.encryfoundation.common.network.BasicMessagesRepo.{RequestChunkMessage, ResponseChunkMessage, ResponseManifestMessage} +import org.encryfoundation.common.modifiers.history.{ Header, Payload } +import org.encryfoundation.common.network.BasicMessagesRepo.{ + RequestChunkMessage, + ResponseChunkMessage, + ResponseManifestMessage +} import org.encryfoundation.common.utils.Algos class SnapshotDownloader(settings: EncryAppSettings) extends Actor with StrictLogging { + import context.dispatcher + var snapshotHolder: SnapshotHolder = SnapshotHolder.initialize( settings, @@ -25,116 +34,172 @@ class SnapshotDownloader(settings: EncryAppSettings) extends Actor with StrictLo else settings.storage.snapshotHolder ) var snapshotDownloadController: SnapshotDownloadController = SnapshotDownloadController.empty(settings) - var historyReader: HistoryReader = HistoryReader.empty + var historyReader: HistoryReader = HistoryReader.empty + var requiredManifestHeight: Int = 0 override def receive: Receive = ??? - - def fastSyncMod( - history: History, - responseTimeout: Option[Cancellable] - ): Receive = { - case DataFromPeer(message, remote) => - logger.debug(s"Snapshot holder got from ${remote} message ${message.NetworkMessageTypeID}.") - message match { - case ResponseManifestMessage(manifest) => - logger.info( - s"Got new manifest message ${Algos.encode(manifest.manifestId.toByteArray)} while processing chunks." - ) - case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => - (for { - controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) - (controller, chunk) = controllerAndChunk - validChunk <- snapshotHolder.validateChunkId(chunk) - processor = snapshotHolder.updateCache(validChunk) - newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { - case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] - case t => t.asLeft[SnapshotHolder] - } - } yield (newProcessor, controller)) match { - case Left(err: UnexpectedChunkMessage) => - logger.info(s"Error has occurred ${err.error} with peer $remote") - case Left(error) => - logger.info(s"Error has occurred: $error") - nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) - restartFastSync(history) - case Right((processor, controller)) - if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => - nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) - restartFastSync(history) - case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => - processor.assembleUTXOState() match { - case Right(state) => - logger.info(s"Tree is valid on Snapshot holder!") - processor.wallet.foreach { wallet: EncryWallet => - (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] - } - case _ => - nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) - restartFastSync(history).asLeft[Unit] - } - case Right((processor, controller)) => - snapshotDownloadController = controller - snapshotHolder = processor - if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks + def receive1: Receive = { + case SemanticallySuccessfulModifier(modifier) if modifier.modifierTypeId == Payload.modifierTypeId => + val bestBlockHeight: Int = historyReader.getBestBlockHeight + if (historyReader.isFastSyncInProcess && bestBlockHeight >= requiredManifestHeight) { + logger.info( + s"Snapshot downloader got new block. Current best block height is: $bestBlockHeight. " + + s"Height of last available payload for request is: $requiredManifestHeight." + ) + historyReader + .getBestHeaderAtHeight(requiredManifestHeight) + .map { h: Header => + RequiredManifestHeightAndId( + requiredManifestHeight, + Algos.hash(h.stateRoot ++ h.id) + ) } - - case ResponseChunkMessage(_) => - logger.info(s"Received chunk from unexpected peer ${remote}") - - case _ => - } - - case RequestNextChunks => - responseTimeout.foreach(_.cancel()) - (for { - controllerAndIds <- snapshotDownloadController.getNextBatchAndRemoveItFromController - _ = logger.info(s"Current notYetRequested batches is ${snapshotDownloadController.batchesSize}.") - } yield controllerAndIds) match { - case Left(err) => - logger.info(s"Error has occurred: ${err.error}") - throw new Exception(s"Error has occurred: ${err.error}") - case Right(controllerAndIds) => - snapshotDownloadController = controllerAndIds._1 - controllerAndIds._2.foreach { msg => - // snapshotDownloadController.cp.foreach { peer: PeerConnectionHandler.ConnectedPeer => - // peer.handlerRef ! msg - // } + .foreach { manifestToId: RequiredManifestHeightAndId => + logger.info( + s"Manifest height is: ${manifestToId.height}. " + + s"Manifest id is: ${Algos.encode(manifestToId.manifestId)}" + ) + snapshotDownloadController = snapshotDownloadController.copy( + requiredManifestHeight = manifestToId.height, + requiredManifestId = manifestToId.manifestId + ) } - context.become(fastSyncMod(history, timer).orElse(commonMessages)) + restartFastSync() + self ! BroadcastManifestRequestMessage } - - case RequiredManifestHeightAndId(height, manifestId) => + case SemanticallySuccessfulModifier(_) => + case BroadcastManifestRequestMessage => logger.info( - s"Snapshot holder while header sync got message RequiredManifestHeight with height $height." + - s"New required manifest id is ${Algos.encode(manifestId)}." - ) - snapshotDownloadController = snapshotDownloadController.copy( - requiredManifestHeight = height, - requiredManifestId = manifestId + s"Snapshot downloader got BroadcastManifestRequestMessage message. " + + s"Required manifest id is: ${Algos.encode(snapshotDownloadController.requiredManifestId)}." ) - restartFastSync(history) - self ! BroadcastManifestRequestMessage - context.become(awaitManifestMod(none, history).orElse(commonMessages)) + context.parent ! BroadcastManifestRequest(snapshotDownloadController.requiredManifestId) - case CheckDelivery => - snapshotDownloadController.awaitedChunks.map { id => - RequestChunkMessage(id.data) - }.foreach { msg => - //snapshotDownloadController.cp.foreach(peer => peer.handlerRef ! msg) - } - context.become(fastSyncMod(history, timer).orElse(commonMessages)) + val newScheduler: Cancellable = + context.system.scheduler.scheduleOnce(settings.snapshotSettings.manifestReAskTimeout) { + logger.info(s"Trigger scheduler for re-request manifest") + self ! BroadcastManifestRequestMessage + } + logger.info(s"Start awaiting manifest network message.") +// context.become(awaitingManifestFromNetwork(newScheduler).orElse(commonMessages)) + } - case FastSyncDone => - if (settings.snapshotSettings.enableSnapshotCreation) { - logger.info(s"Snapshot holder context.become to snapshot processing") - snapshotHolder = SnapshotHolder.recreateAfterFastSyncIsDone(settings) - snapshotDownloadController.storage.close() - context.system.scheduler - .scheduleOnce(settings.snapshotSettings.updateRequestsPerTime)(self ! DropProcessedCount) - context.become(workMod(history).orElse(commonMessages)) - } else { - logger.info(s"Stop processing snapshots") - context.stop(self) - } + def awaitingManifestFromNetwork(scheduler: Cancellable): Receive = { + case _ => + } + +// def fastSyncMod( +// history: History, +// responseTimeout: Option[Cancellable] +// ): Receive = { +// case DataFromPeer(message, remote) => +// logger.debug(s"Snapshot holder got from ${remote} message ${message.NetworkMessageTypeID}.") +// message match { +// case ResponseManifestMessage(manifest) => +// logger.info( +// s"Got new manifest message ${Algos.encode(manifest.manifestId.toByteArray)} while processing chunks." +// ) +// case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => +// (for { +// controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) +// (controller, chunk) = controllerAndChunk +// validChunk <- snapshotHolder.validateChunkId(chunk) +// processor = snapshotHolder.updateCache(validChunk) +// newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { +// case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] +// case t => t.asLeft[SnapshotHolder] +// } +// } yield (newProcessor, controller)) match { +// case Left(err: UnexpectedChunkMessage) => +// logger.info(s"Error has occurred ${err.error} with peer $remote") +// case Left(error) => +// logger.info(s"Error has occurred: $error") +// nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) +// restartFastSync(history) +// case Right((processor, controller)) +// if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => +// nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) +// restartFastSync(history) +// case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => +// processor.assembleUTXOState() match { +// case Right(state) => +// logger.info(s"Tree is valid on Snapshot holder!") +// processor.wallet.foreach { wallet: EncryWallet => +// (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] +// } +// case _ => +// nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) +// restartFastSync(history).asLeft[Unit] +// } +// case Right((processor, controller)) => +// snapshotDownloadController = controller +// snapshotHolder = processor +// if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks +// } +// +// case ResponseChunkMessage(_) => +// logger.info(s"Received chunk from unexpected peer ${remote}") +// +// case _ => +// } +// +// case RequestNextChunks => +// responseTimeout.foreach(_.cancel()) +// (for { +// controllerAndIds <- snapshotDownloadController.getNextBatchAndRemoveItFromController +// _ = logger.info(s"Current notYetRequested batches is ${snapshotDownloadController.batchesSize}.") +// } yield controllerAndIds) match { +// case Left(err) => +// logger.info(s"Error has occurred: ${err.error}") +// throw new Exception(s"Error has occurred: ${err.error}") +// case Right(controllerAndIds) => +// snapshotDownloadController = controllerAndIds._1 +// controllerAndIds._2.foreach { msg => +// // snapshotDownloadController.cp.foreach { peer: PeerConnectionHandler.ConnectedPeer => +// // peer.handlerRef ! msg +// // } +// } +// context.become(fastSyncMod(history, timer).orElse(commonMessages)) +// } +// +// case RequiredManifestHeightAndId(height, manifestId) => +// logger.info( +// s"Snapshot holder while header sync got message RequiredManifestHeight with height $height." + +// s"New required manifest id is ${Algos.encode(manifestId)}." +// ) +// snapshotDownloadController = snapshotDownloadController.copy( +// requiredManifestHeight = height, +// requiredManifestId = manifestId +// ) +// restartFastSync(history) +// self ! BroadcastManifestRequestMessage +// context.become(awaitManifestMod(none, history).orElse(commonMessages)) +// +// case CheckDelivery => +// snapshotDownloadController.awaitedChunks.map { id => +// RequestChunkMessage(id.data) +// }.foreach { msg => +// //snapshotDownloadController.cp.foreach(peer => peer.handlerRef ! msg) +// } +// context.become(fastSyncMod(history, timer).orElse(commonMessages)) +// +// case FastSyncDone => +// if (settings.snapshotSettings.enableSnapshotCreation) { +// logger.info(s"Snapshot holder context.become to snapshot processing") +// snapshotHolder = SnapshotHolder.recreateAfterFastSyncIsDone(settings) +// snapshotDownloadController.storage.close() +// context.system.scheduler +// .scheduleOnce(settings.snapshotSettings.updateRequestsPerTime)(self ! DropProcessedCount) +// context.become(workMod(history).orElse(commonMessages)) +// } else { +// logger.info(s"Stop processing snapshots") +// context.stop(self) +// } +// } + + def restartFastSync(): Unit = { + logger.info(s"Restart fast sync!") + snapshotDownloadController = snapshotDownloadController.reInitFastSync + snapshotHolder = snapshotHolder.reInitStorage } } diff --git a/src/main/scala/encry/nvg/SnapshotProcessor.scala b/src/main/scala/encry/nvg/SnapshotProcessor.scala index bc52858361..ed002d7f03 100644 --- a/src/main/scala/encry/nvg/SnapshotProcessor.scala +++ b/src/main/scala/encry/nvg/SnapshotProcessor.scala @@ -59,7 +59,7 @@ class SnapshotProcessor( if (settings.snapshotSettings.enableFastSynchronization && !history.isBestBlockDefined && !settings.node.offlineGeneration) { logger.info(s"Start in fast sync regime") - context.become(fastSyncMod(history, none).orElse(commonMessages)) +// context.become(fastSyncMod(history, none).orElse(commonMessages)) } else { logger.info(s"Start in snapshot processing regime") context.system.scheduler @@ -69,174 +69,174 @@ class SnapshotProcessor( case nonsense => logger.info(s"Snapshot holder got $nonsense while history awaiting") } - def fastSyncMod( - history: History, - responseTimeout: Option[Cancellable] - ): Receive = { - case DataFromPeer(message, remote) => - logger.debug(s"Snapshot holder got from ${remote} message ${message.NetworkMessageTypeID}.") - message match { - case ResponseManifestMessage(manifest) => - logger.info( - s"Got new manifest message ${Algos.encode(manifest.manifestId.toByteArray)} while processing chunks." - ) - case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => - (for { - controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) - (controller, chunk) = controllerAndChunk - validChunk <- snapshotProcessor.validateChunkId(chunk) - processor = snapshotProcessor.updateCache(validChunk) - newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { - case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] - case t => t.asLeft[SnapshotProcessor] - } - } yield (newProcessor, controller)) match { - case Left(err: UnexpectedChunkMessage) => - logger.info(s"Error has occurred ${err.error} with peer $remote") - case Left(error) => - logger.info(s"Error has occurred: $error") - nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) - restartFastSync(history) - case Right((processor, controller)) - if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => - nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) - restartFastSync(history) - case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => - processor.assembleUTXOState() match { - case Right(state) => - logger.info(s"Tree is valid on Snapshot holder!") - processor.wallet.foreach { wallet: EncryWallet => - (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] - } - case _ => - nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) - restartFastSync(history).asLeft[Unit] - } - case Right((processor, controller)) => - snapshotDownloadController = controller - snapshotProcessor = processor - if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks - } - - case ResponseChunkMessage(_) => - logger.info(s"Received chunk from unexpected peer ${remote}") - - case _ => - } - - case RequestNextChunks => - responseTimeout.foreach(_.cancel()) - (for { - controllerAndIds <- snapshotDownloadController.getNextBatchAndRemoveItFromController - _ = logger.info(s"Current notYetRequested batches is ${snapshotDownloadController.batchesSize}.") - } yield controllerAndIds) match { - case Left(err) => - logger.info(s"Error has occurred: ${err.error}") - throw new Exception(s"Error has occurred: ${err.error}") - case Right(controllerAndIds) => - snapshotDownloadController = controllerAndIds._1 - controllerAndIds._2.foreach { msg => -// snapshotDownloadController.cp.foreach { peer: PeerConnectionHandler.ConnectedPeer => -// peer.handlerRef ! msg +// def fastSyncMod( +// history: History, +// responseTimeout: Option[Cancellable] +// ): Receive = { +// case DataFromPeer(message, remote) => +// logger.debug(s"Snapshot holder got from ${remote} message ${message.NetworkMessageTypeID}.") +// message match { +// case ResponseManifestMessage(manifest) => +// logger.info( +// s"Got new manifest message ${Algos.encode(manifest.manifestId.toByteArray)} while processing chunks." +// ) +// case ResponseChunkMessage(chunk) if snapshotDownloadController.canChunkBeProcessed(remote) => +// (for { +// controllerAndChunk <- snapshotDownloadController.processRequestedChunk(chunk, remote) +// (controller, chunk) = controllerAndChunk +// validChunk <- snapshotProcessor.validateChunkId(chunk) +// processor = snapshotProcessor.updateCache(validChunk) +// newProcessor <- processor.processNextApplicableChunk(processor).leftFlatMap { +// case e: ApplicableChunkIsAbsent => e.processor.asRight[FastSyncException] +// case t => t.asLeft[SnapshotProcessor] +// } +// } yield (newProcessor, controller)) match { +// case Left(err: UnexpectedChunkMessage) => +// logger.info(s"Error has occurred ${err.error} with peer $remote") +// case Left(error) => +// logger.info(s"Error has occurred: $error") +// nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage(error.error)) +// restartFastSync(history) +// case Right((processor, controller)) +// if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty && processor.chunksCache.nonEmpty => +// nodeViewSynchronizer ! BanPeer(remote, InvalidChunkMessage("For request is empty, buffer is nonEmpty")) +// restartFastSync(history) +// case Right((processor, controller)) if controller.awaitedChunks.isEmpty && controller.isBatchesSizeEmpty => +// processor.assembleUTXOState() match { +// case Right(state) => +// logger.info(s"Tree is valid on Snapshot holder!") +// processor.wallet.foreach { wallet: EncryWallet => +// (nodeViewHolder ! FastSyncFinished(state, wallet)).asRight[FastSyncException] +// } +// case _ => +// nodeViewSynchronizer ! BanPeer(remote, InvalidStateAfterFastSync("State after fast sync is invalid")) +// restartFastSync(history).asLeft[Unit] +// } +// case Right((processor, controller)) => +// snapshotDownloadController = controller +// snapshotProcessor = processor +// if (snapshotDownloadController.awaitedChunks.isEmpty) self ! RequestNextChunks +// } +// +// case ResponseChunkMessage(_) => +// logger.info(s"Received chunk from unexpected peer ${remote}") +// +// case _ => +// } +// +// case RequestNextChunks => +// responseTimeout.foreach(_.cancel()) +// (for { +// controllerAndIds <- snapshotDownloadController.getNextBatchAndRemoveItFromController +// _ = logger.info(s"Current notYetRequested batches is ${snapshotDownloadController.batchesSize}.") +// } yield controllerAndIds) match { +// case Left(err) => +// logger.info(s"Error has occurred: ${err.error}") +// throw new Exception(s"Error has occurred: ${err.error}") +// case Right(controllerAndIds) => +// snapshotDownloadController = controllerAndIds._1 +// controllerAndIds._2.foreach { msg => +//// snapshotDownloadController.cp.foreach { peer: PeerConnectionHandler.ConnectedPeer => +//// peer.handlerRef ! msg +//// } +// } +// context.become(fastSyncMod(history, timer).orElse(commonMessages)) +// } +// +// case RequiredManifestHeightAndId(height, manifestId) => +//// logger.info( +//// s"Snapshot holder while header sync got message RequiredManifestHeight with height $height." + +//// s"New required manifest id is ${Algos.encode(manifestId)}." +//// ) +//// snapshotDownloadController = snapshotDownloadController.copy( +//// requiredManifestHeight = height, +//// requiredManifestId = manifestId +//// ) +// restartFastSync(history) +// self ! BroadcastManifestRequestMessage +//// context.become(awaitManifestMod(none, history).orElse(commonMessages)) +// +// case CheckDelivery => +// snapshotDownloadController.awaitedChunks.map { id => +// RequestChunkMessage(id.data) +// }.foreach { msg => +// //snapshotDownloadController.cp.foreach(peer => peer.handlerRef ! msg) +// } +// context.become(fastSyncMod(history, timer).orElse(commonMessages)) +// +// case FastSyncDone => +// if (settings.snapshotSettings.enableSnapshotCreation) { +// logger.info(s"Snapshot holder context.become to snapshot processing") +// snapshotProcessor = SnapshotProcessor.recreateAfterFastSyncIsDone(settings) +// snapshotDownloadController.storage.close() +// context.system.scheduler +// .scheduleOnce(settings.snapshotSettings.updateRequestsPerTime)(self ! DropProcessedCount) +// context.become(workMod(history).orElse(commonMessages)) +// } else { +// logger.info(s"Stop processing snapshots") +// context.stop(self) +// } +// } +// +// def awaitManifestMod( +// responseManifestTimeout: Option[Cancellable], +// history: History +// ): Receive = { +// case BroadcastManifestRequestMessage => +//// logger.info( +//// s"Snapshot holder got HeaderChainIsSynced. Broadcasts request for new manifest with id " + +//// s"${Algos.encode(snapshotDownloadController.requiredManifestId)}" +//// ) +//// nodeViewSynchronizer ! SendToNetwork(RequestManifestMessage(snapshotDownloadController.requiredManifestId), +//// Broadcast) +//// val newScheduler = context.system.scheduler.scheduleOnce(settings.snapshotSettings.manifestReAskTimeout) { +//// logger.info(s"Trigger scheduler for re-request manifest") +//// self ! BroadcastManifestRequestMessage +//// } +//// logger.info(s"Start awaiting manifest network message.") +//// context.become(awaitManifestMod(newScheduler.some, history).orElse(commonMessages)) +// +// case DataFromPeer(message, remote) => +// message match { +// case ResponseManifestMessage(manifest) => +// val isValidManifest: Boolean = +// snapshotDownloadController.checkManifestValidity(manifest.manifestId.toByteArray, history) +// val canBeProcessed: Boolean = snapshotDownloadController.canNewManifestBeProcessed +// if (isValidManifest && canBeProcessed) { +// (for { +// controller <- snapshotDownloadController.processManifest(manifest, remote, history) +// processor <- snapshotHolder.initializeApplicableChunksCache( +// history, +// snapshotDownloadController.requiredManifestHeight +// ) +// } yield (controller, processor)) match { +// case Left(error) => +// nodeViewSynchronizer ! BanPeer(remote, InvalidResponseManifestMessage(error.error)) +// case Right((controller, processor)) => +// logger.debug(s"Request manifest message successfully processed.") +// responseManifestTimeout.foreach(_.cancel()) +// snapshotDownloadController = controller +// snapshotHolder = processor +// self ! RequestNextChunks +// logger.debug("Manifest processed successfully.") +// context.become(fastSyncMod(history, none)) // } - } - context.become(fastSyncMod(history, timer).orElse(commonMessages)) - } - - case RequiredManifestHeightAndId(height, manifestId) => - logger.info( - s"Snapshot holder while header sync got message RequiredManifestHeight with height $height." + - s"New required manifest id is ${Algos.encode(manifestId)}." - ) - snapshotDownloadController = snapshotDownloadController.copy( - requiredManifestHeight = height, - requiredManifestId = manifestId - ) - restartFastSync(history) - self ! BroadcastManifestRequestMessage - context.become(awaitManifestMod(none, history).orElse(commonMessages)) - - case CheckDelivery => - snapshotDownloadController.awaitedChunks.map { id => - RequestChunkMessage(id.data) - }.foreach { msg => - //snapshotDownloadController.cp.foreach(peer => peer.handlerRef ! msg) - } - context.become(fastSyncMod(history, timer).orElse(commonMessages)) - - case FastSyncDone => - if (settings.snapshotSettings.enableSnapshotCreation) { - logger.info(s"Snapshot holder context.become to snapshot processing") - snapshotProcessor = SnapshotProcessor.recreateAfterFastSyncIsDone(settings) - snapshotDownloadController.storage.close() - context.system.scheduler - .scheduleOnce(settings.snapshotSettings.updateRequestsPerTime)(self ! DropProcessedCount) - context.become(workMod(history).orElse(commonMessages)) - } else { - logger.info(s"Stop processing snapshots") - context.stop(self) - } - } - - def awaitManifestMod( - responseManifestTimeout: Option[Cancellable], - history: History - ): Receive = { - case BroadcastManifestRequestMessage => - logger.info( - s"Snapshot holder got HeaderChainIsSynced. Broadcasts request for new manifest with id " + - s"${Algos.encode(snapshotDownloadController.requiredManifestId)}" - ) - nodeViewSynchronizer ! SendToNetwork(RequestManifestMessage(snapshotDownloadController.requiredManifestId), - Broadcast) - val newScheduler = context.system.scheduler.scheduleOnce(settings.snapshotSettings.manifestReAskTimeout) { - logger.info(s"Trigger scheduler for re-request manifest") - self ! BroadcastManifestRequestMessage - } - logger.info(s"Start awaiting manifest network message.") - context.become(awaitManifestMod(newScheduler.some, history).orElse(commonMessages)) - - case DataFromPeer(message, remote) => - message match { - case ResponseManifestMessage(manifest) => - val isValidManifest: Boolean = - snapshotDownloadController.checkManifestValidity(manifest.manifestId.toByteArray, history) - val canBeProcessed: Boolean = snapshotDownloadController.canNewManifestBeProcessed - if (isValidManifest && canBeProcessed) { - (for { - controller <- snapshotDownloadController.processManifest(manifest, remote, history) - processor <- snapshotHolder.initializeApplicableChunksCache( - history, - snapshotDownloadController.requiredManifestHeight - ) - } yield (controller, processor)) match { - case Left(error) => - nodeViewSynchronizer ! BanPeer(remote, InvalidResponseManifestMessage(error.error)) - case Right((controller, processor)) => - logger.debug(s"Request manifest message successfully processed.") - responseManifestTimeout.foreach(_.cancel()) - snapshotDownloadController = controller - snapshotHolder = processor - self ! RequestNextChunks - logger.debug("Manifest processed successfully.") - context.become(fastSyncMod(history, none)) - } - } else if (!isValidManifest) { - logger.info(s"Got manifest with invalid id ${Algos.encode(manifest.manifestId.toByteArray)}") - nodeViewSynchronizer ! BanPeer( - remote, - InvalidResponseManifestMessage(s"Invalid manifest id ${Algos.encode(manifest.manifestId.toByteArray)}") - ) - } else logger.info(s"Doesn't need to process new manifest.") - case _ => - } - - case msg @ RequiredManifestHeightAndId(_, _) => - self ! msg - responseManifestTimeout.foreach(_.cancel()) - logger.info(s"Got RequiredManifestHeightAndId while awaitManifestMod") - context.become(fastSyncMod(history, none)) - } +// } else if (!isValidManifest) { +// logger.info(s"Got manifest with invalid id ${Algos.encode(manifest.manifestId.toByteArray)}") +// nodeViewSynchronizer ! BanPeer( +// remote, +// InvalidResponseManifestMessage(s"Invalid manifest id ${Algos.encode(manifest.manifestId.toByteArray)}") +// ) +// } else logger.info(s"Doesn't need to process new manifest.") +// case _ => +// } +// +// case msg @ RequiredManifestHeightAndId(_, _) => +// self ! msg +// responseManifestTimeout.foreach(_.cancel()) +// logger.info(s"Got RequiredManifestHeightAndId while awaitManifestMod") +// context.become(fastSyncMod(history, none)) +// } def workMod(history: History): Receive = { case TreeChunks(chunks, id) => diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index e4f652d869..8e3deb193e 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -1,6 +1,7 @@ package encry.view.history import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Older } +import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -10,6 +11,10 @@ trait HistoryReader { def getBestHeaderHeight: Int + def getBestBlockHeight: Int + + def getBestHeaderAtHeight(h: Int): Option[Header] + def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] def compare(si: SyncInfo): HistoryComparisonResult @@ -25,28 +30,36 @@ trait HistoryReader { def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] def syncInfo: SyncInfo + + def isFastSyncInProcess: Boolean } object HistoryReader { def empty: HistoryReader = new HistoryReader { def isModifierDefined(id: ModifierId): Boolean = false def getBestHeaderHeight: Int = -1 + def getBestBlockHeight: Int = -1 + def getBestHeaderAtHeight(h: Int): Option[Header] = None def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty var isFullChainSynced: Boolean = false def compare(si: SyncInfo): HistoryComparisonResult = Older def modifierBytesById(id: ModifierId): Option[Array[Byte]] = None def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty def syncInfo: SyncInfo = SyncInfo(Seq.empty) + def isFastSyncInProcess: Boolean = false } def apply(history: History): HistoryReader = new HistoryReader { def isModifierDefined(id: ModifierId): Boolean = history.isModifierDefined(id) def getBestHeaderHeight: Int = history.getBestHeaderHeight + def getBestBlockHeight: Int = history.getBestBlockHeight + def getBestHeaderAtHeight(h: Int): Option[Header] = history.getBestHeaderAtHeight(h) def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = history.continuationIds(info, size) def compare(si: SyncInfo): HistoryComparisonResult = history.compare(si) var isFullChainSynced: Boolean = history.isFullChainSynced def modifierBytesById(id: ModifierId): Option[Array[Byte]] = history.modifierBytesById(id) def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = history.payloadsIdsToDownload(howMany, HashSet.empty) def syncInfo: SyncInfo = history.syncInfo + def isFastSyncInProcess: Boolean = history.fastSyncInProgress.fastSyncVal } } From fcb9d81d5835399f474e3abcfcca5a36de95b28c Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 13:40:06 +0300 Subject: [PATCH 052/177] add removing app.mod from expected modifier --- src/main/scala/encry/network/NetworkRouter.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index a379da3e75..d1b5968234 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -17,6 +17,7 @@ import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionVerified, NewConnection, OutgoingConnectionFailed} import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConnection, RequestPeerForConnection} +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} @@ -66,6 +67,7 @@ class NetworkRouter(settings: NetworkSettings, case MessageFromNetwork(message, Some(remote)) => peersKeeper ! BanPeer(remote.socketAddress, InvalidNetworkMessage(message.messageName)) logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") + case msg: SemanticallySuccessfulModifier => deliveryManager ! msg case msg: ModifierFromNetwork => handlerForMods ! msg case msg: OtherNodeSyncingStatus => peersKeeper ! msg case msg: MessageToNetwork => From 309a96b3eb0d9f89a9349dd414bf3283962f0ea2 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 13:46:35 +0300 Subject: [PATCH 053/177] fixed scala match error --- src/main/scala/encry/nvg/NetworkMessagesProcessor.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 17d755f257..8415387d83 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -62,6 +62,7 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St block.encodedId -> toProto(block.header), block.payload.encodedId -> toProto(block.payload) ) + case _ => } case DataFromPeer(message, remote) => message match { From e1caeca743bead87921d30c02897b8aa33098100 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 13:58:23 +0300 Subject: [PATCH 054/177] add logging --- src/main/scala/encry/network/DM.scala | 7 +++++-- src/main/scala/encry/network/MessageBuilder.scala | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 3a938ce0bc..889a14f7b0 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -45,8 +45,11 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! AwaitingRequest(peer, modTypeId, modId, 1) ) - case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts <= networkSettings.maxDeliveryChecks => - if (expectedModifiers.contains(toKey(modId))) context.parent ! RequestFromLocal(peer.some, modTypeId, List(modId)) + case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts <= networkSettings.maxDeliveryChecks && expectedModifiers.contains(toKey(modId))=> + context.parent ! RequestFromLocal(peer.some, modTypeId, List(modId)) + context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! + AwaitingRequest(peer, modTypeId, modId, attempts + 1) + ) case AwaitingRequest(peer, _, modId, _) => logger.info(s"Stop requesting modifier ${Algos.encode(modId)} from peer $peer") case ModifierFromNetwork(source, modTypeId, modId, modBytes) => diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 7740ff4b92..60b5f315a9 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -33,6 +33,7 @@ case class MessageBuilder(peersKeeper: ActorRef, (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> status.notRequested) modsIds.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) + logger.info(s"Requested or received: ${status.requested.length}. Not request or not received: ${status.notRequested.length}") context.parent ! MsgSent(RequestModifiersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } } From 5b68110f0fe79a36cda8508c8c30e776895bb807 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 14:11:36 +0300 Subject: [PATCH 055/177] logging improved --- src/main/scala/encry/nvg/NodeViewHolder.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index a7c6fd5a01..bfed745d26 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -77,14 +77,16 @@ class NodeViewHolder( context.system.scheduler.schedule(1.seconds, 10.seconds) { logger.info( - s"History best header id is: ${nodeView.history.getBestHeaderId.map(Algos.encode)}.\n " + + s"\n History best header id is: ${nodeView.history.getBestHeaderId.map(Algos.encode)}.\n " + s"History best header height is: ${nodeView.history.getBestHeaderHeight}.\n " + s"History best block id is: ${nodeView.history.getBestBlockId.map(Algos.encode)}.\n " + s"History best block height is: ${nodeView.history.getBestBlockHeight}.\n " + s"History best block header is: ${nodeView.history.getHeaderOfBestBlock.map(_.encodedId)}.\n " + s"State height is: ${nodeView.state.height}.\n " + - s"Cache size is: ${ModifiersCache.size}.\n " + - s"Cache elements are: ${ModifiersCache.cache.keys.toList.map(key => Algos.encode(key.toArray)).mkString(",")}." + s"Cache size is: ${ModifiersCache.size}.\n " + ) + logger.info( + s"Cache elements are: ${ModifiersCache.cache.keys.toList.map(key => Algos.encode(key.toArray)).mkString(",")}." ) } From 956127be1a078bf194760bb17b8d3a7c78abf455 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 14:11:47 +0300 Subject: [PATCH 056/177] send info about request to delivery manager --- src/main/scala/encry/network/MessageBuilder.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 60b5f315a9..ea5439805c 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -32,7 +32,7 @@ case class MessageBuilder(peersKeeper: ActorRef, logger.info(s"Going to req mods from ${peer.socketAddress}") (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> status.notRequested) - modsIds.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) + status.notRequested.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) logger.info(s"Requested or received: ${status.requested.length}. Not request or not received: ${status.notRequested.length}") context.parent ! MsgSent(RequestModifiersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } From c2c7e68342a82898dae27b0ed7824ff9d4545cbd Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 14:18:27 +0300 Subject: [PATCH 057/177] provide stopping msgBuilder after msg sending --- src/main/scala/encry/network/MessageBuilder.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index ea5439805c..941cb03de1 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -52,24 +52,27 @@ case class MessageBuilder(peersKeeper: ActorRef, case SendSyncInfo(syncInfo) => (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => peers.foreach(_.handlerRef ! SyncInfoNetworkMessage(syncInfo)) + context.parent ! MsgSent(SyncInfoNetworkMessage.NetworkMessageTypeID, peers.head.socketAddress) } - context.stop(self) case ResponseFromLocal(peer, modTypeId, modsIds) => Try { (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].map { peer => peer.handlerRef ! ModifiersNetworkMessage(modTypeId -> modsIds) + context.parent ! MsgSent(ModifiersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } } context.stop(self) case BroadcastModifier(modTypeId, modInfo) => (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => peers.foreach(_.handlerRef ! InvNetworkMessage(modTypeId -> List(modInfo))) + context.parent ! MsgSent(InvNetworkMessage.NetworkMessageTypeID, peers.head.socketAddress) } context.stop(self) case SendPeers(peers, remote) => Try { (peersKeeper ? GetPeerInfo(remote)).mapTo[ConnectedPeer].map { peer => peer.handlerRef ! PeersNetworkMessage(peers) + context.parent ! MsgSent(PeersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } } From 5572e400f2fc4131e093e91fcd713fd869d7ba9b Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 14:50:31 +0300 Subject: [PATCH 058/177] code style updated --- src/main/scala/encry/nvg/ModifiersCache.scala | 114 ++++++++++-------- src/main/scala/encry/nvg/NodeViewHolder.scala | 23 +--- 2 files changed, 69 insertions(+), 68 deletions(-) diff --git a/src/main/scala/encry/nvg/ModifiersCache.scala b/src/main/scala/encry/nvg/ModifiersCache.scala index 97e4d5b21c..07ed709065 100644 --- a/src/main/scala/encry/nvg/ModifiersCache.scala +++ b/src/main/scala/encry/nvg/ModifiersCache.scala @@ -2,7 +2,7 @@ package encry.nvg import com.typesafe.scalalogging.StrictLogging import encry.view.history.History -import encry.view.history.ValidationError.{FatalValidationError, NonFatalValidationError} +import encry.view.history.ValidationError.{ FatalValidationError, NonFatalValidationError } import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.utils.Algos @@ -17,7 +17,7 @@ object ModifiersCache extends StrictLogging { private type Key = mutable.WrappedArray[Byte] - val cache: TrieMap[Key, PersistentModifier] = TrieMap.empty + val cache: TrieMap[Key, PersistentModifier] = TrieMap.empty private var headersCollection: SortedMap[Int, List[ModifierId]] = SortedMap.empty[Int, List[ModifierId]] private var isChainSynced = false @@ -36,18 +36,23 @@ object ModifiersCache extends StrictLogging { value match { case header: Header => val possibleHeadersAtCurrentHeight: List[ModifierId] = headersCollection.getOrElse(header.height, List()) - logger.debug(s"possibleHeadersAtCurrentHeight(${header.height}): ${possibleHeadersAtCurrentHeight.map(Algos.encode).mkString(",")}") + logger.debug( + s"possibleHeadersAtCurrentHeight(${header.height}): ${possibleHeadersAtCurrentHeight.map(Algos.encode).mkString(",")}" + ) val updatedHeadersAtCurrentHeight: List[ModifierId] = header.id :: possibleHeadersAtCurrentHeight - logger.debug(s"updatedHeadersAtCurrentHeight(${header.height}): ${updatedHeadersAtCurrentHeight.map(Algos.encode).mkString(",")}") + logger.debug( + s"updatedHeadersAtCurrentHeight(${header.height}): ${updatedHeadersAtCurrentHeight.map(Algos.encode).mkString(",")}" + ) headersCollection = headersCollection.updated(header.height, updatedHeadersAtCurrentHeight) case _ => } - if (size > history.settings.node.modifiersCacheSize) cache.find { case (_, modifier) => - history.testApplicable(modifier) match { - case Right(_) | Left(_: NonFatalValidationError) => false - case _ => true - } + if (size > history.settings.node.modifiersCacheSize) cache.find { + case (_, modifier) => + history.testApplicable(modifier) match { + case Right(_) | Left(_: NonFatalValidationError) => false + case _ => true + } }.map(mod => remove(mod._1)) } @@ -64,13 +69,19 @@ object ModifiersCache extends StrictLogging { def findCandidateKey(history: History): List[Key] = { - def isApplicable(key: Key): Boolean = cache.get(key).exists(modifier => history.testApplicable(modifier) match { - case Left(_: FatalValidationError) => remove(key); false - case Right(_) => true - case Left(_) => false - }) + def isApplicable(key: Key): Boolean = + cache + .get(key) + .exists( + modifier => + history.testApplicable(modifier) match { + case Left(_: FatalValidationError) => remove(key); false + case Right(_) => true + case Left(_) => false + } + ) - def getHeadersKeysAtHeight(height: Int): List[Key] = { + def getHeadersKeysAtHeight(height: Int): List[Key] = headersCollection.get(height) match { case Some(headersIds) => headersIds.map(new mutable.WrappedArray.ofByte(_)).collect { @@ -79,26 +90,28 @@ object ModifiersCache extends StrictLogging { case None => List.empty[Key] } - } def findApplicablePayloadAtHeight(height: Int): List[Key] = { history.headerIdsAtHeight(height).view.flatMap(history.getHeaderById).collect { case header: Header if isApplicable(new mutable.WrappedArray.ofByte(header.payloadId)) => new mutable.WrappedArray.ofByte(header.payloadId) } - }.toList - - def exhaustiveSearch: List[Key] = List(cache.find { case (k, v) => - v match { - case _: Header if history.getBestHeaderId.exists(headerId => headerId sameElements v.parentId) => true - case _ => - val isApplicableMod: Boolean = isApplicable(k) - isApplicableMod - } - }).collect { case Some(v) => v._1 } + }.toList + + def exhaustiveSearch: List[Key] = + List(cache.find { + case (k, v) => + v match { + case _: Header if history.getBestHeaderId.exists(headerId => headerId sameElements v.parentId) => true + case _ => + val isApplicableMod: Boolean = isApplicable(k) + isApplicableMod + } + }).collect { case Some(v) => v._1 } @tailrec - def applicableBestPayloadChain(atHeight: Int = history.getBestBlockHeight, prevKeys: List[Key] = List.empty[Key]): List[Key] = { + def applicableBestPayloadChain(atHeight: Int = history.getBestBlockHeight, + prevKeys: List[Key] = List.empty[Key]): List[Key] = { val payloads = findApplicablePayloadAtHeight(atHeight) if (payloads.nonEmpty) applicableBestPayloadChain(atHeight + 1, prevKeys ++ payloads) else prevKeys @@ -112,9 +125,9 @@ object ModifiersCache extends StrictLogging { logger.debug(s"Drop height ${history.getBestHeaderHeight + 1} in HeadersCollection") val res = value.map(cache.get(_)).collect { case Some(v: Header) - if (history.getBestHeaderHeight == history.settings.constants.PreGenesisHeight && - (v.parentId sameElements Header.GenesisParentId) || - history.getHeaderById(v.parentId).nonEmpty) && isApplicable(new mutable.WrappedArray.ofByte(v.id)) => + if (history.getBestHeaderHeight == history.settings.constants.PreGenesisHeight && + (v.parentId sameElements Header.GenesisParentId) || + history.getHeaderById(v.parentId).nonEmpty) && isApplicable(new mutable.WrappedArray.ofByte(v.id)) => logger.debug(s"Find new bestHeader in cache: ${Algos.encode(v.id)}") new mutable.WrappedArray.ofByte(v.id) } @@ -124,26 +137,31 @@ object ModifiersCache extends StrictLogging { logger.debug(s"${history.getBestHeader}") logger.debug(s"${history.getBestHeaderHeight}") logger.debug(s"${headersCollection.get(history.getBestHeaderHeight + 1).map(_.map(Algos.encode))}") - logger.debug(s"No header in cache at height ${history.getBestHeaderHeight + 1}. " + - s"Trying to find in range [${history.getBestHeaderHeight - history.settings.constants.MaxRollbackDepth}, ${history.getBestHeaderHeight}]") - (history.getBestHeaderHeight - history.settings.constants.MaxRollbackDepth to history.getBestHeaderHeight).flatMap(height => - getHeadersKeysAtHeight(height) - ).toList + logger.debug( + s"No header in cache at height ${history.getBestHeaderHeight + 1}. " + + s"Trying to find in range [${history.getBestHeaderHeight - history.settings.constants.MaxRollbackDepth}, ${history.getBestHeaderHeight}]" + ) + (history.getBestHeaderHeight - history.settings.constants.MaxRollbackDepth to history.getBestHeaderHeight) + .flatMap(height => getHeadersKeysAtHeight(height)) + .toList } } if (bestHeadersIds.nonEmpty) bestHeadersIds - else history.headerIdsAtHeight(history.getBestBlockHeight + 1).headOption match { - case Some(id) => history.getHeaderById(id) match { - case Some(header: Header) if isApplicable(new mutable.WrappedArray.ofByte(header.payloadId)) => - List(new mutable.WrappedArray.ofByte(header.payloadId)) - case _ if history.isFullChainSynced => exhaustiveSearch - case _ => List.empty[Key] + else + history.headerIdsAtHeight(history.getBestBlockHeight + 1).headOption match { + case Some(id) => + history.getHeaderById(id) match { + case Some(header: Header) if isApplicable(new mutable.WrappedArray.ofByte(header.payloadId)) => + List(new mutable.WrappedArray.ofByte(header.payloadId)) + case _ if history.isFullChainSynced => exhaustiveSearch + case _ => List.empty[Key] + } + case None if isChainSynced => + logger.debug(s"No payloads for current history") + exhaustiveSearch + case None => + logger.debug(s"No payloads for current history") + List.empty[Key] } - case None if isChainSynced => - logger.debug(s"No payloads for current history") - exhaustiveSearch - case None => logger.debug(s"No payloads for current history") - List.empty[Key] - } } -} \ No newline at end of file +} diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index bfed745d26..8d6c367294 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -14,26 +14,9 @@ import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.nvg.NodeViewHolder.{ - NodeView, - RollbackFailed, - RollbackSucceed, - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification, - UpdateHistoryReader, - UpdateInformation -} -import encry.nvg.SnapshotProcessor.{ - FastSyncDone, - FastSyncFinished, - HeaderChainIsSynced, - RemoveRedundantManifestIds, - RequiredManifestHeightAndId, - SnapshotChunk, - TreeChunks -} +import encry.nvg.NodeViewHolder._ import encry.nvg.SnapshotProcessor.SnapshotManifest.ManifestId +import encry.nvg.SnapshotProcessor._ import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag @@ -48,9 +31,9 @@ import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } From d40d1af2eed1fb589078f0f30375ae2d087a4505 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 15:14:35 +0300 Subject: [PATCH 059/177] change GetPeerWith(Older/Equal/Younger)History --- .../scala/encry/network/MessageBuilder.scala | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 941cb03de1..801c8b4441 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -9,7 +9,7 @@ import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.{Equal, Older, Younger} import encry.network.ConnectedPeersCollection.PeerInfo import encry.network.DM.{IsRequested, RequestSent, RequestStatus} -import encry.network.MessageBuilder.{GetPeerInfo, GetPeerWithEqualHistory, GetPeerWithOlderHistory, GetPeers, MsgSent} +import encry.network.MessageBuilder.{GetPeerInfo, GetPeers, MsgSent} import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendPeers, SendSyncInfo} import encry.network.PeerConnectionHandler.ConnectedPeer import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, PeersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} @@ -40,7 +40,7 @@ case class MessageBuilder(peersKeeper: ActorRef, } case RequestFromLocal(None, modTypeId, modsIds) => Try { - (peersKeeper ? (GetPeerWithOlderHistory() || GetPeerWithEqualHistory())).mapTo[ConnectedPeer].foreach { peer => + (peersKeeper ? (MessageBuilder.PeerWithOlderHistory || MessageBuilder.PeerWithEqualHistory)).mapTo[ConnectedPeer].foreach { peer => (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> status.notRequested) modsIds.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) @@ -86,25 +86,29 @@ object MessageBuilder { case class GetPeerInfo(peerIp: InetSocketAddress) trait GetPeerByPredicate { + def predicate: PeerInfo => Boolean - def ||(that: GetPeerByPredicate): GetPeerByPredicate = new GetPeerByPredicate { - override def predicate: PeerInfo => Boolean = info => this.predicate(info) || that.predicate(info) + + def ||(that: GetPeerByPredicate): GetPeerByPredicate = { + println("invoke!") + GetPeerByPredicate((info: PeerInfo) => predicate(info) || that.predicate(info)) } - def &&(that: GetPeerByPredicate): GetPeerByPredicate =new GetPeerByPredicate { - override def predicate: PeerInfo => Boolean = info => this.predicate(info) && that.predicate(info) + def &&(that: GetPeerByPredicate): GetPeerByPredicate = { + println("invoke!") + val newPredicate = (info: PeerInfo) => this.predicate.andThen(res => that.predicate(info) && res)(info) + GetPeerByPredicate(newPredicate) } } - final case class GetPeerWithEqualHistory() extends GetPeerByPredicate { - override def predicate: PeerInfo => Boolean = (info: PeerInfo) => info.historyComparisonResult == Equal - } - final case class GetPeerWithOlderHistory() extends GetPeerByPredicate { - override def predicate: PeerInfo => Boolean = (info: PeerInfo) => info.historyComparisonResult == Older + object GetPeerByPredicate { + def apply(peerPredicate: PeerInfo => Boolean): GetPeerByPredicate = new GetPeerByPredicate { + override def predicate: PeerInfo => Boolean = peerPredicate + } } - final case class GetPeerWithYoungerHistory() extends GetPeerByPredicate { - override def predicate: PeerInfo => Boolean = (info: PeerInfo) => info.historyComparisonResult == Younger - } + val PeerWithEqualHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Equal) + val PeerWithOlderHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Older) + val PeerWithYoungerHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Younger) def props(peersKeeper: ActorRef, deliveryManager: ActorRef): Props = Props(new MessageBuilder(peersKeeper, deliveryManager)) From 9ce0eb9d29100c8e7743b274718d326fe50d95d5 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 15:52:15 +0300 Subject: [PATCH 060/177] fix payload downloading --- src/main/scala/encry/network/MessageBuilder.scala | 9 ++++----- src/main/scala/encry/network/PK.scala | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 801c8b4441..544af268ea 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -29,7 +29,7 @@ case class MessageBuilder(peersKeeper: ActorRef, case RequestFromLocal(Some(peer), modTypeId, modsIds) => Try { (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].map { peer => - logger.info(s"Going to req mods from ${peer.socketAddress}") + logger.info(s"Going to req mods from ${peer.socketAddress} of type ${modTypeId}") (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> status.notRequested) status.notRequested.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) @@ -40,15 +40,16 @@ case class MessageBuilder(peersKeeper: ActorRef, } case RequestFromLocal(None, modTypeId, modsIds) => Try { - (peersKeeper ? (MessageBuilder.PeerWithOlderHistory || MessageBuilder.PeerWithEqualHistory)).mapTo[ConnectedPeer].foreach { peer => + (peersKeeper ? (MessageBuilder.PeerWithOlderHistory || MessageBuilder.PeerWithEqualHistory)).mapTo[ConnectedPeer].map { peer => + logger.info(s"Going to req mods from ${peer.socketAddress} of type ${modTypeId}") (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => + logger.info(s"Requested or received: ${status.requested.length}. Not request or not received: ${status.notRequested.length}") peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> status.notRequested) modsIds.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) context.parent ! MsgSent(RequestModifiersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } } } - context.stop(self) case SendSyncInfo(syncInfo) => (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => peers.foreach(_.handlerRef ! SyncInfoNetworkMessage(syncInfo)) @@ -90,11 +91,9 @@ object MessageBuilder { def predicate: PeerInfo => Boolean def ||(that: GetPeerByPredicate): GetPeerByPredicate = { - println("invoke!") GetPeerByPredicate((info: PeerInfo) => predicate(info) || that.predicate(info)) } def &&(that: GetPeerByPredicate): GetPeerByPredicate = { - println("invoke!") val newPredicate = (info: PeerInfo) => this.predicate.andThen(res => that.predicate(info) && res)(info) GetPeerByPredicate(newPredicate) } diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 8a87c7cb19..0ab62b5224 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -125,7 +125,7 @@ class PK(networkSettings: NetworkSettings, case predicate: GetPeerByPredicate => connectedPeers.getAll.find { case (_, info) => predicate.predicate(info) }.map { - case (_, info) => sender() ! info.connectedPeer.handlerRef + case (_, info) => sender() ! info.connectedPeer } case GetPeers => sender() ! connectedPeers.getAll.map(_._2.connectedPeer) case GetPeerInfo(peerIp) => connectedPeers.getAll.find(_._1 == peerIp).map { From e3d03f0fbfec48f3ddf247dcfebce42ee7fdc2c8 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 16:22:12 +0300 Subject: [PATCH 061/177] start pipeline of transactions downloading --- src/main/scala/encry/network/NetworkRouter.scala | 15 ++++++++++----- .../encry/view/mempool/IntermediaryMempool.scala | 16 +++++++++++----- .../encry/view/mempool/MemoryPoolProcessor.scala | 3 --- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index d1b5968234..be370e7aae 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -9,9 +9,9 @@ import akka.io.Tcp.SO.KeepAlive import com.typesafe.scalalogging.StrictLogging import encry.network.BlackList.BanReason.InvalidNetworkMessage import encry.network.Messages.MessageToNetwork -import encry.network.MessageBuilder.{GetPeerInfo, GetPeerWithEqualHistory, GetPeerWithOlderHistory, GetPeers, MsgSent} +import encry.network.MessageBuilder.MsgSent import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} -import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} +import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling, RegisterForTxHandling} import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} @@ -19,6 +19,7 @@ import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionVerified, N import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConnection, RequestPeerForConnection} import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.{BlackListSettings, NetworkSettings} +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} @@ -33,6 +34,7 @@ class NetworkRouter(settings: NetworkSettings, var messagesHandlers: Map[Seq[Byte], ActorRef] = Map.empty var handlerForMods: ActorRef = ActorRef.noSender + var txsHandler: ActorRef = ActorRef.noSender IO(Tcp) ! Bind(self, settings.bindAddress, options = KeepAlive(true) :: Nil, pullMode = false) @@ -45,9 +47,10 @@ class NetworkRouter(settings: NetworkSettings, logger.info(s"Registering handlers for ${types.mkString(",")}.") val ids = types.map(_._1) messagesHandlers += (ids -> handler) - case msg: MsgSent => context.stop(sender()) + case _: MsgSent => context.stop(sender()) case CommandFailed(cmd: Tcp.Command) => logger.info(s"Failed to execute: $cmd.") case RegisterForModsHandling => handlerForMods = sender() + case RegisterForTxHandling => txsHandler = sender() case msg => logger.warn(s"NetworkController: got something strange $msg.") } @@ -62,13 +65,14 @@ class NetworkRouter(settings: NetworkSettings, def businessLogic: Receive = { case MessageFromNetwork(message, Some(remote)) if message.isValid(settings.syncPacketLength) => - logger.debug(s"Got ${message.messageName} on the NetworkController.") + logger.debug(s"Got ${message.messageName} on the NetworkRouter.") findHandler(message, message.NetworkMessageTypeID, remote, messagesHandlers) case MessageFromNetwork(message, Some(remote)) => peersKeeper ! BanPeer(remote.socketAddress, InvalidNetworkMessage(message.messageName)) logger.info(s"Invalid message type: ${message.messageName} from remote $remote.") case msg: SemanticallySuccessfulModifier => deliveryManager ! msg - case msg: ModifierFromNetwork => handlerForMods ! msg + case msg: ModifierFromNetwork if msg.modTypeId != Transaction.modifierTypeId => handlerForMods ! msg + case msg: ModifierFromNetwork => txsHandler ! msg case msg: OtherNodeSyncingStatus => peersKeeper ! msg case msg: MessageToNetwork => context.actorOf( @@ -127,6 +131,7 @@ object NetworkRouter { modBytes: Array[Byte]) case object RegisterForModsHandling + case object RegisterForTxHandling def props(settings: NetworkSettings, blackListSettings: BlackListSettings): Props = Props(new NetworkRouter(settings, blackListSettings)) diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala index e798dbd897..62b41dc8bb 100644 --- a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala +++ b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala @@ -4,20 +4,21 @@ import akka.actor.{Actor, ActorRef, Props} import com.typesafe.scalalogging.StrictLogging import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForTxHandling} import encry.network.PeersKeeper.BanPeer import encry.settings.EncryAppSettings import encry.stats.StatsSender.ValidatedModifierFromNetwork import encry.utils.NetworkTimeProvider import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.{RolledBackTransactions, TransactionProcessing, TransactionsForMiner} -import encry.view.mempool.MemoryPoolProcessor.RequestedModifiersForRemote import encry.view.mempool.TransactionsValidator.{InvalidModifier, ModifiersForValidating} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction class IntermediaryMempool(settings: EncryAppSettings, networkTimeProvider: NetworkTimeProvider, minerReference: ActorRef, - influxReference: Option[ActorRef]) + influxReference: Option[ActorRef], + networkRouter: ActorRef) extends Actor with StrictLogging { @@ -29,6 +30,10 @@ class IntermediaryMempool(settings: EncryAppSettings, val mempoolProcessor: ActorRef = context.actorOf(MemoryPoolProcessor.props(settings, networkTimeProvider), name = "mempool-processor") + override def preStart(): Unit = { + networkRouter ! RegisterForTxHandling + } + override def receive(): Receive = { case msg @ InvalidModifier(_) => // to nvsh case msg @ BanPeer(_, _) => // to peersKeeper @@ -37,8 +42,8 @@ class IntermediaryMempool(settings: EncryAppSettings, case msg @ RolledBackTransactions(_) => memoryPool ! msg // to mempool case msg @ ModifiersForValidating(_, _, _) => memoryPool ! msg // to mempool case msg @ DataFromPeer(_, _) => mempoolProcessor ! msg // to mempool processor - case msg @ RequestedModifiersForRemote(_, _) => // to network case msg @ RequestFromLocal(_, _, _) => // to network + case msg @ ModifierFromNetwork(_, _, _, _) => txValidator ! msg case msg @ TransactionProcessing(_) => mempoolProcessor ! msg // to mempool processor case msg @ IsChainSynced(_) => mempoolProcessor ! msg } @@ -48,8 +53,9 @@ object IntermediaryMempool { def props(settings: EncryAppSettings, networkTimeProvider: NetworkTimeProvider, minerReference: ActorRef, - influxReference: Option[ActorRef]) = - Props(new IntermediaryMempool(settings, networkTimeProvider, minerReference, influxReference)) + influxReference: Option[ActorRef], + networkRouter: ActorRef) = + Props(new IntermediaryMempool(settings, networkTimeProvider, minerReference, influxReference, networkRouter)) final case class TransactionsForValidating(tx: Transaction) diff --git a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala index 0c919244b5..4736009fbd 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala +++ b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala @@ -104,7 +104,4 @@ object MemoryPoolProcessor { def props(settings: EncryAppSettings, ntp: NetworkTimeProvider) = Props(new MemoryPoolProcessor(settings, ntp)) case object CleanupBloomFilter - - final case class RequestedModifiersForRemote(peer: InetSocketAddress, txs: Seq[Transaction]) - } From b493b1715408b228134a705cb2d199fa0a9e83b2 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 16:25:44 +0300 Subject: [PATCH 062/177] code style updated --- .../benches/SnapshotAssemblerBench.scala | 2 +- .../encry/api/http/DataHolderForApi.scala | 2 +- src/main/scala/encry/local/miner/Miner.scala | 2 +- src/main/scala/encry/network/DM.scala | 2 +- .../scala/encry/network/NetworkRouter.scala | 3 +- .../encry/network/NodeViewSynchronizer.scala | 2 +- .../scala/encry/nvg/IntermediaryNVH.scala | 35 +-- .../scala/encry/nvg/ModifiersValidator.scala | 2 +- .../encry/nvg/NetworkMessagesProcessor.scala | 18 +- .../{ => fast/sync}/SnapshotDownloader.scala | 26 +- .../sync}/SnapshotIntermediary.scala | 2 +- .../{ => fast/sync}/SnapshotProcessor.scala | 16 +- .../scala/encry/nvg/nvhg/HistoryApplier.scala | 5 + .../encry/nvg/{ => nvhg}/NodeViewHolder.scala | 27 +- .../nvg/nvhg/NodeViewHolderController.scala | 5 + .../scala/encry/nvg/nvhg/StateApplier.scala | 5 + .../sync/SnapshotDownloadController.scala | 4 +- ...SnapshotDownloadControllerStorageAPI.scala | 2 +- .../encry/view/fast/sync/SnapshotHolder.scala | 27 +- .../sync/SnapshotProcessorStorageAPI.scala | 2 +- .../scala/encry/view/mempool/MemoryPool.scala | 2 +- .../view/mempool/MemoryPoolProcessor.scala | 3 +- .../encry/view/state/avlTree/AvlTree.scala | 254 +++++++++--------- 23 files changed, 225 insertions(+), 223 deletions(-) rename src/main/scala/encry/nvg/{ => fast/sync}/SnapshotDownloader.scala (90%) rename src/main/scala/encry/nvg/{ => fast/sync}/SnapshotIntermediary.scala (82%) rename src/main/scala/encry/nvg/{ => fast/sync}/SnapshotProcessor.scala (96%) create mode 100644 src/main/scala/encry/nvg/nvhg/HistoryApplier.scala rename src/main/scala/encry/nvg/{ => nvhg}/NodeViewHolder.scala (97%) create mode 100644 src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala create mode 100644 src/main/scala/encry/nvg/nvhg/StateApplier.scala diff --git a/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala b/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala index bcc7d55023..c46e2d7749 100644 --- a/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala +++ b/benchmarks/src/test/scala/benches/SnapshotAssemblerBench.scala @@ -7,7 +7,7 @@ import benches.SnapshotAssemblerBench.SnapshotAssemblerBenchState import encry.view.state.avlTree.utils.implicits.Instances._ import benches.StateBenches.{StateBenchState, benchSettings} import benches.Utils.{getRandomTempDir, utxoFromBoxHolder} -import encry.nvg.SnapshotProcessor +import encry.nvg.fast.sync.SnapshotProcessor import encry.settings.Settings import encry.storage.{RootNodesStorage, VersionalStorage} import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} diff --git a/src/main/scala/encry/api/http/DataHolderForApi.scala b/src/main/scala/encry/api/http/DataHolderForApi.scala index 935b592d8d..f29e856311 100644 --- a/src/main/scala/encry/api/http/DataHolderForApi.scala +++ b/src/main/scala/encry/api/http/DataHolderForApi.scala @@ -21,7 +21,7 @@ import encry.network.ConnectedPeersCollection import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeerFromAPI -import encry.nvg.NodeViewHolder.NodeViewChange +import encry.nvg.nvhg.NodeViewHolder.NodeViewChange import encry.settings.EncryAppSettings import encry.utils.{NetworkTime, NetworkTimeProvider} import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index 660cb5324e..c61d078e4a 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -14,7 +14,7 @@ import encry.local.miner.Miner._ import encry.local.miner.Worker.NextChallenge import encry.modifiers.mempool.TransactionFactory import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.NetworkTime.Time diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 889a14f7b0..5ab898a9c8 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -14,7 +14,7 @@ import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMess import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import cats.syntax.option._ -import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier} +import encry.nvg.nvhg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier} import scala.collection.mutable import scala.concurrent.Future diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index be370e7aae..bb4ad9ecfd 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -10,6 +10,7 @@ import com.typesafe.scalalogging.StrictLogging import encry.network.BlackList.BanReason.InvalidNetworkMessage import encry.network.Messages.MessageToNetwork import encry.network.MessageBuilder.MsgSent +import encry.network.MessageBuilder.{GetPeerInfo, GetPeers, MsgSent} import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling, RegisterForTxHandling} import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus @@ -17,7 +18,7 @@ import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionVerified, NewConnection, OutgoingConnectionFailed} import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConnection, RequestPeerForConnection} -import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index e3d4b4bbbb..9313e51524 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -33,7 +33,7 @@ import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import scala.concurrent.duration._ import encry.network.ModifiersToNetworkUtils._ -import encry.nvg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} +import encry.nvg.nvhg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} import scala.util.Try diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 92a7054b5a..c2de5291eb 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -1,41 +1,28 @@ package encry.nvg -import akka.actor.{ Actor, ActorRef, Props } +import akka.actor.{Actor, ActorRef, Props} import akka.routing.BalancingPool import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.local.miner.Miner.{DisableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } -import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } -import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } +import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} +import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeersKeeper.BanPeer -import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation } -import encry.nvg.NodeViewHolder.{ - RollbackFailed, - RollbackSucceed, - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification, - UpdateHistoryReader -} -import encry.nvg.SnapshotProcessor.{ FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks } +import encry.nvg.ModifiersValidator.{InvalidModifierBytes, ModifierForValidation} +import encry.nvg.fast.sync.SnapshotProcessor +import encry.nvg.nvhg.NodeViewHolder.{RollbackFailed, RollbackSucceed, SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification, UpdateHistoryReader} +import encry.nvg.fast.sync.SnapshotProcessor.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks} +import encry.nvg.nvhg.NodeViewHolder import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions -import org.encryfoundation.common.network.BasicMessagesRepo.{ - InvNetworkMessage, - RequestChunkMessage, - RequestManifestMessage, - RequestModifiersNetworkMessage, - ResponseChunkMessage, - ResponseManifestMessage, - SyncInfoNetworkMessage -} +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestChunkMessage, RequestManifestMessage, RequestModifiersNetworkMessage, ResponseChunkMessage, ResponseManifestMessage, SyncInfoNetworkMessage} import org.encryfoundation.common.utils.Algos class IntermediaryNVH( diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index e3990ec576..e63f0eb669 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -16,7 +16,7 @@ import encry.network.BlackList.BanReason.{ } import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation, ValidatedModifier } -import encry.nvg.NodeViewHolder.SyntacticallyFailedModification +import encry.nvg.nvhg.NodeViewHolder.SyntacticallyFailedModification import encry.settings.EncryAppSettings import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 8415387d83..75d88241b7 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -1,26 +1,22 @@ package encry.nvg -import akka.actor.{ Actor, Cancellable, Props } +import akka.actor.{Actor, Cancellable, Props} import com.typesafe.scalalogging.StrictLogging -import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } +import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Younger} import cats.syntax.option._ import encry.network.DeliveryManager.CheckPayloadsToDownload -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } +import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus -import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, UpdateHistoryReader } +import encry.nvg.nvhg.NodeViewHolder +import encry.nvg.nvhg.NodeViewHolder.{SemanticallySuccessfulModifier, UpdateHistoryReader} import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } -import org.encryfoundation.common.network.BasicMessagesRepo.{ - InvNetworkMessage, - ModifiersNetworkMessage, - RequestModifiersNetworkMessage, - SyncInfoNetworkMessage -} +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId diff --git a/src/main/scala/encry/nvg/SnapshotDownloader.scala b/src/main/scala/encry/nvg/fast/sync/SnapshotDownloader.scala similarity index 90% rename from src/main/scala/encry/nvg/SnapshotDownloader.scala rename to src/main/scala/encry/nvg/fast/sync/SnapshotDownloader.scala index 902bba67c3..bcc4ed26c5 100644 --- a/src/main/scala/encry/nvg/SnapshotDownloader.scala +++ b/src/main/scala/encry/nvg/fast/sync/SnapshotDownloader.scala @@ -1,26 +1,14 @@ -package encry.nvg +package encry.nvg.fast.sync -import akka.actor.{ Actor, Cancellable } -import cats.syntax.either._ -import cats.syntax.option._ +import akka.actor.{Actor, Cancellable} import com.typesafe.scalalogging.StrictLogging -import encry.network.BlackList.BanReason.{ InvalidChunkMessage, InvalidStateAfterFastSync } import encry.network.Messages.MessageToNetwork.BroadcastManifestRequest -import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.PeersKeeper.BanPeer -import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier -import encry.nvg.SnapshotProcessor._ +import encry.nvg.fast.sync.SnapshotProcessor.{BroadcastManifestRequestMessage, RequiredManifestHeightAndId} +import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.EncryAppSettings -import encry.view.fast.sync.FastSyncExceptions.{ ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage } -import encry.view.fast.sync.{ SnapshotDownloadController, SnapshotHolder } -import encry.view.history.{ History, HistoryReader } -import encry.view.wallet.EncryWallet -import org.encryfoundation.common.modifiers.history.{ Header, Payload } -import org.encryfoundation.common.network.BasicMessagesRepo.{ - RequestChunkMessage, - ResponseChunkMessage, - ResponseManifestMessage -} +import encry.view.fast.sync.{SnapshotDownloadController, SnapshotHolder} +import encry.view.history.HistoryReader +import org.encryfoundation.common.modifiers.history.{Header, Payload} import org.encryfoundation.common.utils.Algos class SnapshotDownloader(settings: EncryAppSettings) extends Actor with StrictLogging { diff --git a/src/main/scala/encry/nvg/SnapshotIntermediary.scala b/src/main/scala/encry/nvg/fast/sync/SnapshotIntermediary.scala similarity index 82% rename from src/main/scala/encry/nvg/SnapshotIntermediary.scala rename to src/main/scala/encry/nvg/fast/sync/SnapshotIntermediary.scala index 9e08c81b36..bf8c04b28e 100644 --- a/src/main/scala/encry/nvg/SnapshotIntermediary.scala +++ b/src/main/scala/encry/nvg/fast/sync/SnapshotIntermediary.scala @@ -1,4 +1,4 @@ -package encry.nvg +package encry.nvg.fast.sync import akka.actor.Actor diff --git a/src/main/scala/encry/nvg/SnapshotProcessor.scala b/src/main/scala/encry/nvg/fast/sync/SnapshotProcessor.scala similarity index 96% rename from src/main/scala/encry/nvg/SnapshotProcessor.scala rename to src/main/scala/encry/nvg/fast/sync/SnapshotProcessor.scala index ed002d7f03..945c6cb544 100644 --- a/src/main/scala/encry/nvg/SnapshotProcessor.scala +++ b/src/main/scala/encry/nvg/fast/sync/SnapshotProcessor.scala @@ -1,23 +1,19 @@ -package encry.nvg +package encry.nvg.fast.sync import SnapshotChunkProto.SnapshotChunkMessage import SnapshotManifestProto.SnapshotManifestProtoMessage import akka.actor.{Actor, ActorRef, Cancellable, Props} -import cats.syntax.either._ import cats.syntax.option._ import com.google.protobuf.ByteString import com.typesafe.scalalogging.StrictLogging -import encry.network.BlackList.BanReason.{InvalidChunkMessage, InvalidResponseManifestMessage, InvalidStateAfterFastSync} -import encry.network.Broadcast -import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} +import encry.network.DeliveryManager.CheckDelivery +import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.ChangedHistory -import encry.network.PeersKeeper.{BanPeer, SendToNetwork} -import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier -import encry.nvg.SnapshotProcessor.SnapshotManifest.{ChunkId, ManifestId} -import encry.nvg.SnapshotProcessor._ +import encry.nvg.fast.sync.SnapshotProcessor.{DropProcessedCount, HeaderChainIsSynced, RemoveRedundantManifestIds, TreeChunks} +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.{ChunkId, ManifestId} +import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.EncryAppSettings import encry.storage.VersionalStorage.{StorageKey, StorageValue} -import encry.view.fast.sync.FastSyncExceptions.{ApplicableChunkIsAbsent, FastSyncException, UnexpectedChunkMessage} import encry.view.fast.sync.{RequestsPerPeriodProcessor, SnapshotDownloadController, SnapshotHolder} import encry.view.history.{History, HistoryReader} import encry.view.state.UtxoState diff --git a/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala b/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala new file mode 100644 index 0000000000..3f47124fd2 --- /dev/null +++ b/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala @@ -0,0 +1,5 @@ +package encry.nvg.nvhg + +class HistoryApplier { + +} diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala similarity index 97% rename from src/main/scala/encry/nvg/NodeViewHolder.scala rename to src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala index 8d6c367294..3dd5f03522 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala @@ -1,4 +1,4 @@ -package encry.nvg +package encry.nvg.nvhg import java.io.File @@ -12,11 +12,28 @@ import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ +import encry.nvg.ModifiersCache import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.nvg.NodeViewHolder._ -import encry.nvg.SnapshotProcessor.SnapshotManifest.ManifestId -import encry.nvg.SnapshotProcessor._ +import encry.nvg.fast.sync.SnapshotProcessor.{ + FastSyncDone, + FastSyncFinished, + HeaderChainIsSynced, + RemoveRedundantManifestIds, + SnapshotChunk, + TreeChunks +} +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId +import encry.nvg.nvhg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } +import encry.nvg.nvhg.NodeViewHolder.{ + NodeView, + RollbackFailed, + RollbackSucceed, + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification, + UpdateHistoryReader, + UpdateInformation +} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag diff --git a/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala b/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala new file mode 100644 index 0000000000..f64e04a22f --- /dev/null +++ b/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala @@ -0,0 +1,5 @@ +package encry.nvg.nvhg + +class NodeViewHolderController { + +} diff --git a/src/main/scala/encry/nvg/nvhg/StateApplier.scala b/src/main/scala/encry/nvg/nvhg/StateApplier.scala new file mode 100644 index 0000000000..5316a2b46a --- /dev/null +++ b/src/main/scala/encry/nvg/nvhg/StateApplier.scala @@ -0,0 +1,5 @@ +package encry.nvg.nvhg + +class StateApplier { + +} diff --git a/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala b/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala index d2635d4f21..45ec35fc51 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotDownloadController.scala @@ -8,8 +8,8 @@ import SnapshotManifestProto.SnapshotManifestProtoMessage import cats.syntax.either._ import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessor.SnapshotManifest.ChunkId -import encry.nvg.SnapshotProcessor.{ SnapshotChunk, SnapshotChunkSerializer, SnapshotManifestSerializer } +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ChunkId +import encry.nvg.fast.sync.SnapshotProcessor.{ SnapshotChunk, SnapshotChunkSerializer, SnapshotManifestSerializer } import encry.settings.EncryAppSettings import encry.storage.levelDb.versionalLevelDB.LevelDbFactory import encry.view.fast.sync.FastSyncExceptions._ diff --git a/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala b/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala index 79d3b4d1df..7bee1e8eaa 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotDownloadControllerStorageAPI.scala @@ -1,7 +1,7 @@ package encry.view.fast.sync import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessor.SnapshotManifest.ChunkId +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ChunkId import encry.settings.EncryAppSettings import org.encryfoundation.common.utils.Algos import org.iq80.leveldb.DB diff --git a/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala b/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala index ef047908a7..adbb1112b1 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotHolder.scala @@ -7,36 +7,33 @@ import cats.syntax.either._ import cats.syntax.option._ import com.google.common.primitives.Ints import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessor.{ - SnapshotChunk, - SnapshotChunkSerializer, - SnapshotManifest, - SnapshotManifestSerializer -} -import encry.nvg.SnapshotProcessor.SnapshotManifest.ManifestId +import encry.nvg.fast.sync.SnapshotProcessor.{SnapshotChunk, SnapshotChunkSerializer, SnapshotManifest, SnapshotManifestSerializer} +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotChunk +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId import encry.settings.EncryAppSettings -import encry.storage.{ RootNodesStorage, VersionalStorage } -import encry.storage.VersionalStorage.{ StorageKey, StorageType, StorageValue, StorageVersion } +import encry.storage.{RootNodesStorage, VersionalStorage} +import encry.storage.VersionalStorage.{StorageKey, StorageType, StorageValue, StorageVersion} import encry.storage.iodb.versionalIODB.IODBWrapper -import encry.storage.levelDb.versionalLevelDB.{ LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion } +import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} import encry.view.fast.sync.FastSyncExceptions._ import encry.view.history.History import encry.view.state.UtxoState import encry.view.state.avlTree._ import encry.view.state.avlTree.utils.implicits.Instances._ import encry.view.wallet.EncryWallet -import io.iohk.iodb.{ ByteArrayWrapper, LSMStore } +import io.iohk.iodb.{ByteArrayWrapper, LSMStore} import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.state.StateModifierSerializer import org.encryfoundation.common.modifiers.state.box.EncryBaseBox import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } -import org.iq80.leveldb.{ DB, Options } +import org.encryfoundation.common.utils.TaggedTypes.{Height, ModifierId} +import org.iq80.leveldb.{DB, Options} import scorex.utils.Random -import scala.collection.immutable.{ HashMap, HashSet } +import scala.collection.immutable.{HashMap, HashSet} import scala.language.postfixOps -import scala.util.{ Failure, Success } +import scala.util.{Failure, Success} final case class SnapshotHolder( settings: EncryAppSettings, diff --git a/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala b/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala index f52505f78c..26b7e0ca63 100644 --- a/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala +++ b/src/main/scala/encry/view/fast/sync/SnapshotProcessorStorageAPI.scala @@ -3,7 +3,7 @@ package encry.view.fast.sync import SnapshotChunkProto.SnapshotChunkMessage import SnapshotManifestProto.SnapshotManifestProtoMessage import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessor.{ SnapshotManifest, SnapshotManifestSerializer } +import encry.nvg.fast.sync.SnapshotProcessor.{ SnapshotManifest, SnapshotManifestSerializer } import encry.storage.VersionalStorage import encry.storage.VersionalStorage.{ StorageKey, StorageValue } import org.encryfoundation.common.utils.Algos diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index 695ded5134..0c4ae8dfec 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -5,7 +5,7 @@ import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import cats.syntax.either._ import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.nvg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} +import encry.nvg.nvhg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.ReceivableMessages.CompareViews diff --git a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala index 4736009fbd..23a0fbbc00 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala +++ b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala @@ -12,7 +12,7 @@ import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.TransactionProcessing -import encry.view.mempool.MemoryPoolProcessor.{CleanupBloomFilter, RequestedModifiersForRemote} +import encry.view.mempool.MemoryPoolProcessor.{CleanupBloomFilter} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestModifiersNetworkMessage} import org.encryfoundation.common.utils.Algos @@ -55,7 +55,6 @@ class MemoryPoolProcessor(settings: EncryAppSettings, ntp: NetworkTimeProvider) .map(Algos.encode) .collect { case id if memoryPool.contains(id) => memoryPool.get(id) } .flatten - context.parent ! RequestedModifiersForRemote(remote, modifiersIds) logger.debug( s"MemoryPool got request modifiers message. Number of requested ids is ${requestedIds.size}." + s" Number of sent transactions is ${modifiersIds.size}. Request was from $remote." diff --git a/src/main/scala/encry/view/state/avlTree/AvlTree.scala b/src/main/scala/encry/view/state/avlTree/AvlTree.scala index 9b97e83d45..9ab46c63ef 100644 --- a/src/main/scala/encry/view/state/avlTree/AvlTree.scala +++ b/src/main/scala/encry/view/state/avlTree/AvlTree.scala @@ -4,8 +4,8 @@ import cats.syntax.order._ import cats.{Monoid, Order} import com.google.common.primitives.Ints import com.typesafe.scalalogging.StrictLogging -import encry.nvg.SnapshotProcessor.SnapshotChunk -import encry.nvg.SnapshotProcessor.SnapshotManifest.ChunkId +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotChunk +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ChunkId import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} import encry.storage.{RootNodesStorage, VersionalStorage} import encry.view.state.UtxoState @@ -18,10 +18,12 @@ import org.encryfoundation.common.utils.TaggedTypes.Height import scala.util.Try -final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], - avlStorage: VersionalStorage, - rootNodesStorage: RootNodesStorage[K, V], - saveRootNodes: Boolean = false) extends AutoCloseable with StrictLogging { +final case class AvlTree[K: Hashable: Order, V](rootNode: Node[K, V], + avlStorage: VersionalStorage, + rootNodesStorage: RootNodesStorage[K, V], + saveRootNodes: Boolean = false) + extends AutoCloseable + with StrictLogging { implicit def nodeOrder(implicit ord: Order[K]): Order[Node[K, V]] = new Order[Node[K, V]] { override def compare(x: Node[K, V], y: Node[K, V]): Int = ord.compare(x.key, y.key) @@ -31,31 +33,29 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], val rootHash: Array[Byte] = rootNode.hash - def insertAndDeleteMany(version: StorageVersion, - toInsert: List[(K, V)], - toDelete: List[K], - stateHeight: Height = Height @@ 0, - saveRootNodesFlag: Boolean = false) - (implicit kSer: Serializer[K], - vSer: Serializer[V], - kM: Monoid[K], - vM: Monoid[V]): AvlTree[K, V] = { + def insertAndDeleteMany( + version: StorageVersion, + toInsert: List[(K, V)], + toDelete: List[K], + stateHeight: Height = Height @@ 0, + saveRootNodesFlag: Boolean = false + )(implicit kSer: Serializer[K], vSer: Serializer[V], kM: Monoid[K], vM: Monoid[V]): AvlTree[K, V] = { val deleteStartTime = System.nanoTime() val rootAfterDelete = toDelete.foldLeft(rootNode) { case (prevRoot, toDeleteKey) => deleteKey(toDeleteKey, prevRoot) } val avlDeleteTime = System.nanoTime() - deleteStartTime - logger.debug(s"avlDeleteTime: ${avlDeleteTime/1000000L} ms") + logger.debug(s"avlDeleteTime: ${avlDeleteTime / 1000000L} ms") val insertStartTime = System.nanoTime() val newRoot = toInsert.foldLeft(rootAfterDelete) { case (prevRoot, (keyToInsert, valueToInsert)) => insert(keyToInsert, valueToInsert, prevRoot) } val insertTime = System.nanoTime() - insertStartTime - logger.debug(s"avlInsertTime: ${insertTime/1000000L} ms") + logger.debug(s"avlInsertTime: ${insertTime / 1000000L} ms") val startPackingTime = System.nanoTime() - logger.debug(s"Packing time: ${(System.nanoTime() - startPackingTime)/1000000} ms") + logger.debug(s"Packing time: ${(System.nanoTime() - startPackingTime) / 1000000} ms") val startInsertTime = System.nanoTime() logger.debug(s"Insert in avl version ${Algos.encode(version)}") avlStorage.insert( @@ -64,18 +64,19 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], case (key, value) => StorageKey @@ AvlTree.elementKey(kSer.toBytes(key)) -> StorageValue @@ vSer.toBytes(value) } ++ - List(AvlTree.rootNodeKey -> StorageValue @@ newRoot.hash, - UtxoState.bestHeightKey -> StorageValue @@ Ints.toByteArray(stateHeight)), - toDelete.map(key => - StorageKey @@ AvlTree.elementKey(kSer.toBytes(key)) - ) + List(AvlTree.rootNodeKey -> StorageValue @@ newRoot.hash, + UtxoState.bestHeightKey -> StorageValue @@ Ints.toByteArray(stateHeight)), + toDelete.map(key => StorageKey @@ AvlTree.elementKey(kSer.toBytes(key))) ) - logger.debug(s"Insertion time: ${(System.nanoTime() - startInsertTime)/1000000L} ms") - val newRootNodesStorage = if (saveRootNodesFlag || saveRootNodes) rootNodesStorage.insert( - version, - newRoot, - stateHeight - ) else rootNodesStorage + logger.debug(s"Insertion time: ${(System.nanoTime() - startInsertTime) / 1000000L} ms") + val newRootNodesStorage = + if (saveRootNodesFlag || saveRootNodes) + rootNodesStorage.insert( + version, + newRoot, + stateHeight + ) + else rootNodesStorage AvlTree(newRoot, avlStorage, newRootNodesStorage, saveRootNodesFlag || saveRootNodes) } @@ -95,10 +96,10 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], } def findKey(key: K, node: Node[K, V]): Option[V] = node match { - case internalNode: InternalNode[K, V] if key > internalNode.key=> findKey(key, internalNode.rightChild) - case internalNode: InternalNode[K, V] if key < internalNode.key=> findKey(key, internalNode.leftChild) - case n: Node[K, V] if n.key === key => Some(n.value) - case _ => None + case internalNode: InternalNode[K, V] if key > internalNode.key => findKey(key, internalNode.rightChild) + case internalNode: InternalNode[K, V] if key < internalNode.key => findKey(key, internalNode.leftChild) + case n: Node[K, V] if n.key === key => Some(n.value) + case _ => None } def get(k: K)(implicit kSer: Serializer[K], vSer: Serializer[V]): Option[V] = @@ -107,10 +108,10 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], def contains(k: K)(implicit kSer: Serializer[K]): Boolean = avlStorage.get(StorageKey !@@ AvlTree.elementKey(kSer.toBytes(k))).isDefined - def deleteKey(key: K, node: Node[K, V])(implicit m: Monoid[K], - v: Monoid[V], - kSer: Serializer[K], - vSer: Serializer[V]): Node[K, V] = delete(node, key) + def deleteKey( + key: K, + node: Node[K, V] + )(implicit m: Monoid[K], v: Monoid[V], kSer: Serializer[K], vSer: Serializer[V]): Node[K, V] = delete(node, key) private def delete(node: Node[K, V], key: K)( implicit m: Monoid[K], @@ -132,7 +133,7 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], balance(childUpdated) } else if (internalNode.key < key) { val newRightChild = delete(internalNode.rightChild, key) - val childUpdated = internalNode.updateChilds(newRightChild = newRightChild) + val childUpdated = internalNode.updateChilds(newRightChild = newRightChild) balance(childUpdated) } else { val theClosestValue = findTheClosestValue(internalNode, internalNode.key) @@ -188,8 +189,10 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], } } - def rollbackTo(to: StorageVersion, additionalBlocks: List[Block]) - (implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Try[AvlTree[K, V]] = + def rollbackTo( + to: StorageVersion, + additionalBlocks: List[Block] + )(implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Try[AvlTree[K, V]] = Try { logger.info(s"Rollback avl to version: ${Algos.encode(to)}") logger.info(s"Versions in storage: ${avlStorage.versions.map(Algos.encode).mkString(",")}") @@ -198,18 +201,21 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], avlStorage.rollbackTo(to) logger.info(s"Storage success rolled back") logger.info(s"rootNodeKey: ${Algos.encode(avlStorage.get(AvlTree.rootNodeKey).getOrElse(Array.emptyByteArray))}") - val (newStorage, newRoot) = rootNodesStorage.rollbackToSafePoint(RootNodesStorage.blocks2InsInfo[K, V](additionalBlocks)) + val (newStorage, newRoot) = + rootNodesStorage.rollbackToSafePoint(RootNodesStorage.blocks2InsInfo[K, V](additionalBlocks)) logger.info(s"root node hash after rollback: ${Algos.encode(newRoot.hash)}") AvlTree[K, V](newRoot, avlStorage, newStorage) } - def restore(additionalBlocks: List[Block]) - (implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Try[AvlTree[K, V]] = - Try { - val (newStorage, newRoot) = rootNodesStorage.rollbackToSafePoint(RootNodesStorage.blocks2InsInfo[K, V](additionalBlocks)) - logger.info(s"root node hash after restore: ${Algos.encode(newRoot.hash)}") - AvlTree[K, V](newRoot, avlStorage, newStorage) - } + def restore( + additionalBlocks: List[Block] + )(implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Try[AvlTree[K, V]] = + Try { + val (newStorage, newRoot) = + rootNodesStorage.rollbackToSafePoint(RootNodesStorage.blocks2InsInfo[K, V](additionalBlocks)) + logger.info(s"root node hash after restore: ${Algos.encode(newRoot.hash)}") + AvlTree[K, V](newRoot, avlStorage, newStorage) + } private def getRightPath(node: Node[K, V]): List[Node[K, V]] = node match { case shadowNode: ShadowNode[K, V] => @@ -231,39 +237,39 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], case _: EmptyNode[K, V] => List.empty } - private def insert(newKey: K, newValue: V, node: Node[K, V]) - (implicit kMonoid: Monoid[K], - kSer: Serializer[K], - vMonoid: Monoid[V], - vSer: Serializer[V]): Node[K, V] = node match { - case shadowNode: ShadowNode[K, V] => - val restoredNode = shadowNode.restoreFullNode(avlStorage) - insert(newKey, newValue, restoredNode) - case _: EmptyNode[K, V] => LeafNode[K, V](newKey, newValue) - case leafNode: LeafNode[K, V] => - if (leafNode.key === newKey) leafNode.copy(value = newValue) - else { - val newInternalNode = InternalNode[K, V](leafNode.key, leafNode.value, height = 1, balance = 0) - insert( - newKey, - newValue, - newInternalNode - ) - } - case internalNode: InternalNode[K, V] => - if (internalNode.key > newKey) { - val newLeftChild = insert(newKey, newValue, internalNode.leftChild) - val newNode = internalNode.updateChilds(newLeftChild = newLeftChild) - balance(newNode) - } else { - val newRightChild = insert(newKey, newValue, internalNode.rightChild) - val newNode = internalNode.updateChilds(newRightChild = newRightChild) - balance(newNode) - } - } + private def insert(newKey: K, newValue: V, node: Node[K, V])(implicit kMonoid: Monoid[K], + kSer: Serializer[K], + vMonoid: Monoid[V], + vSer: Serializer[V]): Node[K, V] = node match { + case shadowNode: ShadowNode[K, V] => + val restoredNode = shadowNode.restoreFullNode(avlStorage) + insert(newKey, newValue, restoredNode) + case _: EmptyNode[K, V] => LeafNode[K, V](newKey, newValue) + case leafNode: LeafNode[K, V] => + if (leafNode.key === newKey) leafNode.copy(value = newValue) + else { + val newInternalNode = InternalNode[K, V](leafNode.key, leafNode.value, height = 1, balance = 0) + insert( + newKey, + newValue, + newInternalNode + ) + } + case internalNode: InternalNode[K, V] => + if (internalNode.key > newKey) { + val newLeftChild = insert(newKey, newValue, internalNode.leftChild) + val newNode = internalNode.updateChilds(newLeftChild = newLeftChild) + balance(newNode) + } else { + val newRightChild = insert(newKey, newValue, internalNode.rightChild) + val newNode = internalNode.updateChilds(newRightChild = newRightChild) + balance(newNode) + } + } - private def balance(node: Node[K, V]) - (implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Node[K, V] = { + private def balance( + node: Node[K, V] + )(implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Node[K, V] = { node match { case shadowNode: ShadowNode[K, V] => val restoredNode = shadowNode.restoreFullNode(avlStorage) @@ -310,11 +316,9 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], case _ => -1 } - private def rightRotation(node: Node[K, V]) - (implicit kMonoid: Monoid[K], - kSer: Serializer[K], - vMonoid: Monoid[V], - vSer: Serializer[V]): Node[K, V] = { + private def rightRotation( + node: Node[K, V] + )(implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Node[K, V] = { node match { case shadowNode: ShadowNode[K, V] => val restoredNode = shadowNode.restoreFullNode(avlStorage) @@ -331,16 +335,14 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], } } val newLeftChildForPrevRoot = newRoot.rightChild.selfInspection - val prevRoot = internalNode.updateChilds(newLeftChild = newLeftChildForPrevRoot) + val prevRoot = internalNode.updateChilds(newLeftChild = newLeftChildForPrevRoot) newRoot.updateChilds(newRightChild = prevRoot) } }.selfInspection - private def leftRotation(node: Node[K, V]) - (implicit kMonoid: Monoid[K], - kSer: Serializer[K], - vMonoid: Monoid[V], - vSer: Serializer[V]): Node[K, V] = { + private def leftRotation( + node: Node[K, V] + )(implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Node[K, V] = { node match { case shadowNode: ShadowNode[K, V] => val restoredNode = shadowNode.restoreFullNode(avlStorage) @@ -357,16 +359,14 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], } } val newRightChildForPrevRoot = newRoot.leftChild.selfInspection - val prevRoot = internalNode.updateChilds(newRightChild = newRightChildForPrevRoot) + val prevRoot = internalNode.updateChilds(newRightChild = newRightChildForPrevRoot) newRoot.updateChilds(newLeftChild = prevRoot) } }.selfInspection - private def rlRotation(node: Node[K, V]) - (implicit kMonoid: Monoid[K], - kSer: Serializer[K], - vMonoid: Monoid[V], - vSer: Serializer[V]): Node[K, V] = { + private def rlRotation( + node: Node[K, V] + )(implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Node[K, V] = { node match { case shadowNode: ShadowNode[K, V] => val restoredNode = shadowNode.restoreFullNode(avlStorage) @@ -380,11 +380,9 @@ final case class AvlTree[K : Hashable : Order, V](rootNode: Node[K, V], } }.selfInspection - private def lrRotation(node: Node[K, V]) - (implicit kMonoid: Monoid[K], - kSer: Serializer[K], - vMonoid: Monoid[V], - vSer: Serializer[V]): Node[K, V] = { + private def lrRotation( + node: Node[K, V] + )(implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Node[K, V] = { node match { case shadowNode: ShadowNode[K, V] => val restoredNode = shadowNode.restoreFullNode(avlStorage) @@ -416,8 +414,12 @@ object AvlTree extends StrictLogging { AvlTree(rootNode, avlStorage, rootNodesStorage) } - def rollbackTo[K: Hashable: Order, V](to: StorageVersion, additionalBlocks: List[Block], avlStorage: VersionalStorage, rootNodesStorage: RootNodesStorage[K, V]) - (implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Try[AvlTree[K, V]] = + def rollbackTo[K: Hashable: Order, V]( + to: StorageVersion, + additionalBlocks: List[Block], + avlStorage: VersionalStorage, + rootNodesStorage: RootNodesStorage[K, V] + )(implicit kMonoid: Monoid[K], kSer: Serializer[K], vMonoid: Monoid[V], vSer: Serializer[V]): Try[AvlTree[K, V]] = Try { logger.info(s"Rollback avl to version: ${Algos.encode(to)}") logger.info(s"Versions in storage: ${avlStorage.versions.map(Algos.encode).mkString(",")}") @@ -425,7 +427,8 @@ object AvlTree extends StrictLogging { avlStorage.rollbackTo(to) logger.info(s"Storage success rolled back") logger.info(s"rootNodeKey: ${Algos.encode(avlStorage.get(AvlTree.rootNodeKey).getOrElse(Array.emptyByteArray))}") - val (newStorage, newRoot) = rootNodesStorage.rollbackToSafePoint(RootNodesStorage.blocks2InsInfo[K, V](additionalBlocks)) + val (newStorage, newRoot) = + rootNodesStorage.rollbackToSafePoint(RootNodesStorage.blocks2InsInfo[K, V](additionalBlocks)) logger.info(s"root node hash after rollback: ${Algos.encode(newRoot.hash)}") AvlTree[K, V](newRoot, avlStorage, newStorage) } @@ -439,28 +442,30 @@ object AvlTree extends StrictLogging { case object EMPTY extends Direction } - def apply[K: Monoid: Order: Hashable : Serializer, - V: Monoid : Serializer](avlStorage: VersionalStorage, rootNodesStorage: RootNodesStorage[K, V]): AvlTree[K, V] = - { - rootNodesStorage.insert(StorageVersion @@ Array.fill(32)(0: Byte), EmptyNode(), Height @@ 0) - new AvlTree[K, V](EmptyNode(), avlStorage, rootNodesStorage) - } + def apply[K: Monoid: Order: Hashable: Serializer, V: Monoid: Serializer]( + avlStorage: VersionalStorage, + rootNodesStorage: RootNodesStorage[K, V] + ): AvlTree[K, V] = { + rootNodesStorage.insert(StorageVersion @@ Array.fill(32)(0: Byte), EmptyNode(), Height @@ 0) + new AvlTree[K, V](EmptyNode(), avlStorage, rootNodesStorage) + } def elementKey(key: Array[Byte]): Array[Byte] = (0: Byte) +: key def nodeKey(key: Array[Byte]): Array[Byte] = (1: Byte) +: key - def getChunks(node: Node[StorageKey, StorageValue], - currentChunkHeight: Int, - avlStorage: VersionalStorage) - (implicit kSer: Serializer[StorageKey], - vSer: Serializer[StorageValue], - kM: Monoid[StorageKey], - vM: Monoid[StorageValue], - hashKey: Hashable[StorageKey]): List[SnapshotChunk] = { - - def restoreNodesUntilDepthAndReturnLeafs(depth: Int, - node: Node[StorageKey, StorageValue]): (Node[StorageKey, StorageValue], List[Node[StorageKey, StorageValue]]) = node match { + def getChunks(node: Node[StorageKey, StorageValue], currentChunkHeight: Int, avlStorage: VersionalStorage)( + implicit kSer: Serializer[StorageKey], + vSer: Serializer[StorageValue], + kM: Monoid[StorageKey], + vM: Monoid[StorageValue], + hashKey: Hashable[StorageKey] + ): List[SnapshotChunk] = { + + def restoreNodesUntilDepthAndReturnLeafs( + depth: Int, + node: Node[StorageKey, StorageValue] + ): (Node[StorageKey, StorageValue], List[Node[StorageKey, StorageValue]]) = node match { case shadowNode: ShadowNode[StorageKey, StorageValue] => val newNode = shadowNode.restoreFullNode(avlStorage) restoreNodesUntilDepthAndReturnLeafs(depth, newNode) @@ -475,11 +480,12 @@ object AvlTree extends StrictLogging { ) -> (rightSubTreeChildren ++ leftSubTreeChildren) case internalNode: InternalNode[StorageKey, StorageValue] => internalNode -> List(internalNode.leftChild, internalNode.rightChild) - case leaf: LeafNode[StorageKey, StorageValue] => leaf -> List.empty[Node[StorageKey, StorageValue]] + case leaf: LeafNode[StorageKey, StorageValue] => leaf -> List.empty[Node[StorageKey, StorageValue]] case emptyNode: EmptyNode[StorageKey, StorageValue] => emptyNode -> List.empty } - val (rootChunk: Node[StorageKey, StorageValue], rootChunkChildren) = restoreNodesUntilDepthAndReturnLeafs(currentChunkHeight, node) + val (rootChunk: Node[StorageKey, StorageValue], rootChunkChildren) = + restoreNodesUntilDepthAndReturnLeafs(currentChunkHeight, node) SnapshotChunk(rootChunk, ChunkId @@ rootChunk.hash) :: rootChunkChildren.flatMap(node => getChunks(node, currentChunkHeight, avlStorage)) } From d92a85c466eb4f9be41cd1c21ec82f305f9bf7d9 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 17:41:53 +0300 Subject: [PATCH 063/177] logs improved --- src/main/scala/encry/network/DM.scala | 4 ++-- src/main/scala/encry/nvg/ModifiersValidator.scala | 2 ++ src/main/scala/encry/view/state/UtxoState.scala | 8 ++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 5ab898a9c8..72804c6a5c 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -42,8 +42,8 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging } case RequestSent(peer, modTypeId, modId) => expectedModifiers += toKey(modId) - context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! - AwaitingRequest(peer, modTypeId, modId, 1) + context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)( + self ! AwaitingRequest(peer, modTypeId, modId, 1) ) case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts <= networkSettings.maxDeliveryChecks && expectedModifiers.contains(toKey(modId))=> context.parent ! RequestFromLocal(peer.some, modTypeId, List(modId)) diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index e63f0eb669..5dfa628e8b 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -55,6 +55,8 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings } } + + private def isPreSemanticValid( modifier: PersistentModifier, historyReader: HistoryReader, diff --git a/src/main/scala/encry/view/state/UtxoState.scala b/src/main/scala/encry/view/state/UtxoState.scala index 614098a537..ec9d901776 100644 --- a/src/main/scala/encry/view/state/UtxoState.scala +++ b/src/main/scala/encry/view/state/UtxoState.scala @@ -76,7 +76,7 @@ final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], case block: Block => logger.info(s"\n\nStarting to applyModifier as a Block: ${Algos.encode(mod.id)} to state at height ${block.header.height}") logger.info(s"State root should be: ${Algos.encode(block.header.stateRoot)}") - logger.info(s"Current root node hash: ${tree.rootNode.hash}") + logger.info(s"Current root node hash: ${Algos.encode(tree.rootNode.hash)}") val lastTxId = block.payload.txs.last.id val totalFees: Amount = block.payload.txs.init.map(_.fee).sum val validstartTime = System.nanoTime() @@ -120,7 +120,7 @@ final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], s"State root should be ${Algos.encode(block.header.stateRoot)} but got " + s"${Algos.encode(newTree.rootNode.hash)}")).asLeft[UtxoState] } else { - logger.info(s"After applying root node: ${newTree.rootNode.hash}") + logger.info(s"After applying root node: ${Algos.encode(newTree.rootNode.hash)}") UtxoState( newTree, Height @@ block.header.height, @@ -139,7 +139,7 @@ final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], def rollbackTo(version: VersionTag, additionalBlocks: List[Block]): Try[UtxoState] = Try{ logger.info(s"Rollback utxo to version: ${Algos.encode(version)}") val rollbackedAvl = AvlTree.rollbackTo(StorageVersion !@@ version, additionalBlocks, tree.avlStorage, tree.rootNodesStorage).get - logger.info(s"UTXO -> rollbackTo ->${tree.avlStorage.get(UtxoState.bestHeightKey)} ") + logger.info(s"UTXO -> rollbackTo -> ${tree.avlStorage.get(UtxoState.bestHeightKey).map(Ints.fromByteArray)}.") val height: Height = Height !@@ Ints.fromByteArray(tree.avlStorage.get(UtxoState.bestHeightKey).get) UtxoState(rollbackedAvl, height, constants, influxRef) } @@ -147,7 +147,7 @@ final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], def restore(additionalBlocks: List[Block]): Try[UtxoState] = Try { logger.info(s"Rollback utxo from storage: ${Algos.encode(version)}") val rollbackedAvl = tree.restore(additionalBlocks).get - logger.info(s"UTXO -> rollbackTo ->${tree.avlStorage.get(UtxoState.bestHeightKey)} ") + logger.info(s"UTXO -> restore -> ${tree.avlStorage.get(UtxoState.bestHeightKey).map(Ints.fromByteArray)}.") val height: Height = Height !@@ Ints.fromByteArray(tree.avlStorage.get(UtxoState.bestHeightKey).get) UtxoState(rollbackedAvl, height, constants, influxRef) } From da1b14390ddf9648bcb0004e5e31f54656d164aa Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 6 Mar 2020 18:19:30 +0300 Subject: [PATCH 064/177] code style updated --- src/main/scala/encry/Starter.scala | 15 ++++---- .../scala/encry/nvg/IntermediaryNVH.scala | 38 ++++++++++++++----- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 6afb9ca961..17c2813c50 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -2,10 +2,10 @@ package encry import java.net.InetSocketAddress -import akka.actor.{Actor, ActorRef} +import akka.actor.{ Actor, ActorRef } import akka.http.scaladsl.Http import cats.Functor -import cats.data.{NonEmptyChain, Validated} +import cats.data.{ NonEmptyChain, Validated } import cats.instances.future._ import cats.instances.option._ import cats.syntax.apply._ @@ -16,21 +16,20 @@ import encry.Starter.InitNodeResult import encry.api.http.DataHolderForApi import encry.api.http.DataHolderForApi.PassForStorage import encry.cli.ConsoleListener -import encry.cli.ConsoleListener.{StartListening, prompt} +import encry.cli.ConsoleListener.{ prompt, StartListening } import encry.local.miner.Miner import encry.local.miner.Miner.StartMining -import encry.network.{NetworkRouter, NodeViewSynchronizer} +import encry.network.NetworkRouter import encry.nvg.IntermediaryNVH import encry.settings._ import encry.stats.StatsSender -import encry.utils.{Mnemonic, NetworkTimeProvider} -import encry.view.NodeViewHolder +import encry.utils.{ Mnemonic, NetworkTimeProvider } import encry.view.mempool.MemoryPool import encry.view.wallet.AccountManager import scala.concurrent.Future import scala.io.StdIn -import scala.util.{Failure, Success, Try} +import scala.util.{ Failure, Success, Try } class Starter(settings: EncryAppSettings, timeProvider: NetworkTimeProvider, @@ -43,7 +42,7 @@ class Starter(settings: EncryAppSettings, var initHttpApiServer: Option[Future[Http.ServerBinding]] = none - val preview = + val preview: String = """ |XXXXXX XX XX XXXXX XXXXXX XX XX |XX XXXX XX XX XX XX XXXXXX diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index c2de5291eb..5947d99e8a 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -1,28 +1,48 @@ package encry.nvg -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import akka.routing.BalancingPool import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo -import encry.local.miner.Miner.{DisableMining, StartMining} +import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} -import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} -import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } +import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } +import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeersKeeper.BanPeer -import encry.nvg.ModifiersValidator.{InvalidModifierBytes, ModifierForValidation} +import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation } import encry.nvg.fast.sync.SnapshotProcessor -import encry.nvg.nvhg.NodeViewHolder.{RollbackFailed, RollbackSucceed, SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification, UpdateHistoryReader} -import encry.nvg.fast.sync.SnapshotProcessor.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks} +import encry.nvg.nvhg.NodeViewHolder.{ + RollbackFailed, + RollbackSucceed, + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification, + UpdateHistoryReader +} +import encry.nvg.fast.sync.SnapshotProcessor.{ + FastSyncDone, + HeaderChainIsSynced, + RequiredManifestHeightAndId, + TreeChunks +} import encry.nvg.nvhg.NodeViewHolder import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions -import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestChunkMessage, RequestManifestMessage, RequestModifiersNetworkMessage, ResponseChunkMessage, ResponseManifestMessage, SyncInfoNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{ + InvNetworkMessage, + RequestChunkMessage, + RequestManifestMessage, + RequestModifiersNetworkMessage, + ResponseChunkMessage, + ResponseManifestMessage, + SyncInfoNetworkMessage +} import org.encryfoundation.common.utils.Algos class IntermediaryNVH( From adbdff42e0a98e3f18ffe8136d401294e2e42264 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 6 Mar 2020 18:30:08 +0300 Subject: [PATCH 065/177] add some pipelines to mempool --- src/main/scala/encry/Starter.scala | 19 ++++-- .../scala/encry/network/NetworkRouter.scala | 7 +- .../scala/encry/nvg/nvhg/NodeViewHolder.scala | 2 +- .../encry/storage/RootNodesStorage.scala | 22 +++++-- .../scala/encry/view/mempool/MemoryPool.scala | 65 +++++++++++++++++++ .../view/mempool/MemoryPoolProcessor.scala | 11 ++-- .../view/mempool/TransactionsValidator.scala | 35 +++++----- .../encry/view/state/avlTree/AvlTree.scala | 2 +- 8 files changed, 124 insertions(+), 39 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 17c2813c50..265514609c 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -25,6 +25,9 @@ import encry.settings._ import encry.stats.StatsSender import encry.utils.{ Mnemonic, NetworkTimeProvider } import encry.view.mempool.MemoryPool +import encry.utils.{Mnemonic, NetworkTimeProvider} +import encry.view.NodeViewHolder +import encry.view.mempool.{IntermediaryMempool, MemoryPool} import encry.view.wallet.AccountManager import scala.concurrent.Future @@ -408,13 +411,13 @@ class Starter(settings: EncryAppSettings, } lazy val dataHolderForApi = context.system.actorOf(DataHolderForApi.props(newSettings, timeProvider), "dataHolder") - lazy val miner: ActorRef = + val miner: ActorRef = context.system.actorOf(Miner.props(dataHolderForApi, influxRef, newSettings), "miner") - lazy val memoryPool: ActorRef = context.system.actorOf( - MemoryPool - .props(newSettings, timeProvider, miner, influxRef) - .withDispatcher("mempool-dispatcher") - ) +// lazy val memoryPool: ActorRef = context.system.actorOf( +// MemoryPool +// .props(newSettings, timeProvider, miner, influxRef) +// .withDispatcher("mempool-dispatcher") +// ) // val nodeViewHolder: ActorRef = context.system.actorOf( // NodeViewHolder // .props(memoryPool, influxRef, dataHolderForApi, newSettings) @@ -442,6 +445,10 @@ class Starter(settings: EncryAppSettings, IntermediaryNVH.props(newSettings, networkRouter, timeProvider, influxRef) ) + val memoryPool = context.system.actorOf( + IntermediaryMempool.props(newSettings, timeProvider, miner, influxRef, networkRouter) + ) + if (newSettings.node.mining) miner ! StartMining if (newSettings.node.useCli) { context.system diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index bb4ad9ecfd..a98a3393f5 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -10,7 +10,7 @@ import com.typesafe.scalalogging.StrictLogging import encry.network.BlackList.BanReason.InvalidNetworkMessage import encry.network.Messages.MessageToNetwork import encry.network.MessageBuilder.MsgSent -import encry.network.MessageBuilder.{GetPeerInfo, GetPeers, MsgSent} +import encry.network.MessageBuilder.{GetPeerInfo, GetPeers, MsgSent} import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling, RegisterForTxHandling} import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus @@ -21,7 +21,7 @@ import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConn import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.NetworkMessage +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, NetworkMessage} import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import scala.concurrent.duration._ @@ -65,6 +65,9 @@ class NetworkRouter(settings: NetworkSettings, } def businessLogic: Receive = { + case mfn@MessageFromNetwork(inv@InvNetworkMessage(data), Some(_)) if data._1 == Transaction.modifierTypeId && inv.isValid(settings.syncPacketLength) => + logger.debug(s"Got ${inv.messageName} on the NetworkRouter.") + txsHandler ! mfn case MessageFromNetwork(message, Some(remote)) if message.isValid(settings.syncPacketLength) => logger.debug(s"Got ${message.messageName} on the NetworkRouter.") findHandler(message, message.NetworkMessageTypeID, remote, messagesHandlers) diff --git a/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala b/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala index 3dd5f03522..6dcf1745e5 100644 --- a/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala @@ -472,7 +472,7 @@ class NodeViewHolder( logger.info( s"State and history are inconsistent." + s" Going to rollback to ${rollbackId.map(Algos.encode)} and " + - s"apply ${newChain.length} modifiers" + s"apply ${newChain.length} modifiers. State safe point: ${safePointHeight}. ${newChain.headers.head.height}. ${newChain.headers.last.height}" ) val additionalBlocks = (state.safePointHeight + 1 to historyBestBlock.header.height).foldLeft(List.empty[Block]) { diff --git a/src/main/scala/encry/storage/RootNodesStorage.scala b/src/main/scala/encry/storage/RootNodesStorage.scala index c47fccc0ad..66528341e1 100644 --- a/src/main/scala/encry/storage/RootNodesStorage.scala +++ b/src/main/scala/encry/storage/RootNodesStorage.scala @@ -1,17 +1,20 @@ package encry.storage -import java.io.{ BufferedOutputStream, File, FileOutputStream } -import java.nio.file.{ Files, Paths } -import cats.kernel.{ Monoid, Order } +import java.io.{BufferedOutputStream, File, FileOutputStream} +import java.nio.file.{Files, Paths} + +import cats.kernel.{Monoid, Order} import com.google.common.primitives.Ints +import com.typesafe.scalalogging.StrictLogging import encry.storage.VersionalStorage.StorageVersion -import encry.view.state.avlTree.utils.implicits.{ Hashable, Serializer } -import encry.view.state.avlTree.{ AvlTree, EmptyNode, Node, NodeSerilalizer } +import encry.view.state.avlTree.utils.implicits.{Hashable, Serializer} +import encry.view.state.avlTree.{AvlTree, EmptyNode, Node, NodeSerilalizer} import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.Height -import org.iq80.leveldb.{ DB, ReadOptions } +import org.iq80.leveldb.{DB, ReadOptions} import scorex.utils.Random + import scala.util.Try trait RootNodesStorage[K, V] extends AutoCloseable { @@ -29,7 +32,7 @@ object RootNodesStorage { def apply[K: Serializer: Monoid: Hashable: Order, V: Serializer: Monoid: Hashable](storage: DB, rollbackDepth: Int, - rootsPath: File): RootNodesStorage[K, V] = new RootNodesStorage[K, V] with AutoCloseable { + rootsPath: File): RootNodesStorage[K, V] = new RootNodesStorage[K, V] with AutoCloseable with StrictLogging { private def atHeightKey(height: Height): Array[Byte] = Ints.toByteArray(height) @@ -47,6 +50,8 @@ object RootNodesStorage { val bos = new BufferedOutputStream(new FileOutputStream(fileToAdd)) try { val newSafePointHeight = Math.max(0, height - rollbackDepth) + logger.info(s"new safe point height: ${newSafePointHeight}") + logger.info(s"write to file root node with hash: ${Algos.encode(rootNode.hash)}") val newSafePointSerialized = Ints.toByteArray(newSafePointHeight) val fileToDelete = new File(rootsPath.getAbsolutePath ++ s"/${newSafePointHeight - rollbackDepth}") if (fileToDelete.exists()) fileToDelete.delete() @@ -73,11 +78,14 @@ object RootNodesStorage { val newRootNode = insertionInfo .foldLeft(avlTree) { case (tree, (height, (toInsert, toDelete))) => + logger.info(s"Previous tree hash: ${Algos.encode(tree.rootNode.hash)}") val newTree = tree.insertAndDeleteMany( StorageVersion @@ Random.randomBytes(), toInsert, toDelete ) + logger.info(s"Current safe point: ${safePointHeight}") + logger.info(s"After insertion at height ${height} state root: ${Algos.encode(newTree.rootNode.hash)}") if (height == currentSafePoint + rollbackDepth) { batch.put(Ints.toByteArray(currentSafePoint + rollbackDepth), NodeSerilalizer.toBytes(newTree.rootNode)) } diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index 0c4ae8dfec..f8e3f4915e 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -3,16 +3,24 @@ package encry.view.mempool import akka.actor.{Actor, ActorRef, ActorSystem, Props} import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} import cats.syntax.either._ +import com.google.common.base.Charsets +import com.google.common.hash.{BloomFilter, Funnels} import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging +import encry.network.Messages.MessageToNetwork.{RequestFromLocal, ResponseFromLocal} +import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.nvg.nvhg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.ReceivableMessages.CompareViews +import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.MemoryPoolStateType.NotProcessingNewTransactions import encry.view.mempool.MemoryPool._ import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.IndexedSeq @@ -25,8 +33,12 @@ class MemoryPool(settings: EncryAppSettings, var memoryPool: MemoryPoolStorage = MemoryPoolStorage.empty(settings, networkTimeProvider) + var bloomFilterForTransactionsIds: BloomFilter[String] = initBloomFilter + var canProcessTransactions: Boolean = false + var chainSynced: Boolean = false + override def preStart(): Unit = { logger.debug(s"Starting MemoryPool. Initializing all schedulers") context.system.scheduler.schedule( @@ -47,6 +59,47 @@ class MemoryPool(settings: EncryAppSettings, def disableTransactionsProcessor: Receive = auxiliaryReceive(MemoryPoolStateType.NotProcessingNewTransactions) def transactionsProcessor(currentNumberOfProcessedTransactions: Int): Receive = { + case DataFromPeer(message, remote) => + message match { + case RequestModifiersNetworkMessage((_, requestedIds)) => + val modifiersIds: Seq[Transaction] = requestedIds + .map(Algos.encode) + .collect { case id if memoryPool.contains(id) => memoryPool.get(id) } + .flatten + logger.debug( + s"MemoryPool got request modifiers message. Number of requested ids is ${requestedIds.size}." + + s" Number of sent transactions is ${modifiersIds.size}. Request was from $remote." + ) + context.parent ! ResponseFromLocal( + remote, + Transaction.modifierTypeId, + modifiersIds.map(tx => tx.id -> tx.bytes).toMap + ) + case InvNetworkMessage((_, txs)) => + val notYetRequestedTransactions: IndexedSeq[ModifierId] = notRequestedYet(txs.toIndexedSeq) + if (notYetRequestedTransactions.nonEmpty) { + sender ! RequestFromLocal(Some(remote), Transaction.modifierTypeId, notYetRequestedTransactions.toList) + logger.debug( + s"MemoryPool got inv message with ${txs.size} ids." + + s" Not yet requested ids size is ${notYetRequestedTransactions.size}." + ) + } else + logger.debug( + s"MemoryPool got inv message with ${txs.size} ids." + + s" There are no not yet requested ids." + ) + + case InvNetworkMessage(invData) => + logger.debug( + s"Get inv with tx: ${invData._2.map(Algos.encode).mkString(",")}, but " + + s"chainSynced is $chainSynced and canProcessTransactions is $canProcessTransactions." + ) + + case _ => logger.debug(s"MemoryPoolProcessor got invalid type of DataFromPeer message!") + } + + case IsChainSynced(info) => chainSynced = info + case NewTransaction(transaction) => val (newMemoryPool: MemoryPoolStorage, validatedTransaction: Option[Transaction]) = memoryPool.validateTransaction(transaction) @@ -134,6 +187,18 @@ class MemoryPool(settings: EncryAppSettings, case message => logger.debug(s"MemoryPool got unhandled message $message.") } + + def notRequestedYet(ids: IndexedSeq[ModifierId]): IndexedSeq[ModifierId] = ids.collect { + case id: ModifierId if !bloomFilterForTransactionsIds.mightContain(Algos.encode(id)) => + bloomFilterForTransactionsIds.put(Algos.encode(id)) + id + } + + def initBloomFilter: BloomFilter[String] = BloomFilter.create( + Funnels.stringFunnel(Charsets.UTF_8), + settings.mempool.bloomFilterCapacity, + settings.mempool.bloomFilterFailureProbability + ) } object MemoryPool { diff --git a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala index 23a0fbbc00..c0055dd99b 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala +++ b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala @@ -6,13 +6,13 @@ import akka.actor.{Actor, Props} import com.google.common.base.Charsets import com.google.common.hash.{BloomFilter, Funnels} import com.typesafe.scalalogging.StrictLogging -import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.Messages.MessageToNetwork.{RequestFromLocal, ResponseFromLocal} import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.TransactionProcessing -import encry.view.mempool.MemoryPoolProcessor.{CleanupBloomFilter} +import encry.view.mempool.MemoryPoolProcessor.CleanupBloomFilter import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestModifiersNetworkMessage} import org.encryfoundation.common.utils.Algos @@ -49,7 +49,6 @@ class MemoryPoolProcessor(settings: EncryAppSettings, ntp: NetworkTimeProvider) case DataFromPeer(message, remote) => message match { - case RequestModifiersNetworkMessage((_, requestedIds)) => val modifiersIds: Seq[Transaction] = requestedIds .map(Algos.encode) @@ -59,7 +58,11 @@ class MemoryPoolProcessor(settings: EncryAppSettings, ntp: NetworkTimeProvider) s"MemoryPool got request modifiers message. Number of requested ids is ${requestedIds.size}." + s" Number of sent transactions is ${modifiersIds.size}. Request was from $remote." ) - + context.parent ! ResponseFromLocal( + remote, + Transaction.modifierTypeId, + modifiersIds.map(tx => tx.id -> tx.bytes).toMap + ) case InvNetworkMessage((_, txs)) => val notYetRequestedTransactions: IndexedSeq[ModifierId] = notRequestedYet(txs.toIndexedSeq) if (notYetRequestedTransactions.nonEmpty) { diff --git a/src/main/scala/encry/view/mempool/TransactionsValidator.scala b/src/main/scala/encry/view/mempool/TransactionsValidator.scala index 4268635170..c9c7cc571c 100644 --- a/src/main/scala/encry/view/mempool/TransactionsValidator.scala +++ b/src/main/scala/encry/view/mempool/TransactionsValidator.scala @@ -4,6 +4,7 @@ import TransactionProto.TransactionProtoMessage import akka.actor.{Actor, ActorRef, Props} import com.typesafe.scalalogging.StrictLogging import encry.network.BlackList.BanReason.{CorruptedSerializedBytes, SyntacticallyInvalidTransaction} +import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeer import encry.settings.EncryAppSettings @@ -11,7 +12,9 @@ import encry.utils.NetworkTimeProvider import encry.view.mempool.MemoryPool.NewTransaction import encry.view.mempool.TransactionsValidator.{InvalidModifier, ModifiersForValidating} import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionProtoSerializer} +import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} + import scala.util.{Failure, Success, Try} class TransactionsValidator(settings: EncryAppSettings, @@ -21,26 +24,22 @@ class TransactionsValidator(settings: EncryAppSettings, with StrictLogging { override def receive(): Receive = { - case ModifiersForValidating(remote, typeId, filteredModifiers) => - typeId match { - case Transaction.modifierTypeId => - filteredModifiers.foreach { - case (id, bytes) => - Try(TransactionProtoSerializer.fromProto(TransactionProtoMessage.parseFrom(bytes))).flatten match { - case Success(tx) if tx.semanticValidity.isSuccess => memPool ! NewTransaction(tx) - case Success(tx) => - logger.info(s"Transaction with id: ${tx.encodedId} invalid cause of: ${tx.semanticValidity}.") - context.parent ! BanPeer(remote.socketAddress, SyntacticallyInvalidTransaction) - context.parent ! InvalidModifier(id) - case Failure(ex) => - context.parent ! BanPeer(remote.socketAddress, CorruptedSerializedBytes) - context.parent ! InvalidModifier(id) - logger.info(s"Received modifier from $remote can't be parsed cause of: ${ex.getMessage}.") - } + case DataFromPeer(ModifiersNetworkMessage(data), remote) if data._1 == Transaction.modifierTypeId => + data._2.foreach { + case (id, bytes) => + Try(TransactionProtoSerializer.fromProto(TransactionProtoMessage.parseFrom(bytes))).flatten match { + case Success(tx) if tx.semanticValidity.isSuccess => memPool ! NewTransaction(tx) + case Success(tx) => + logger.info(s"Transaction with id: ${tx.encodedId} invalid cause of: ${tx.semanticValidity}.") + context.parent ! BanPeer(remote, SyntacticallyInvalidTransaction) + context.parent ! InvalidModifier(id) + case Failure(ex) => + context.parent ! BanPeer(remote, CorruptedSerializedBytes) + context.parent ! InvalidModifier(id) + logger.info(s"Received modifier from $remote can't be parsed cause of: ${ex.getMessage}.") } - } + } } - } object TransactionsValidator { diff --git a/src/main/scala/encry/view/state/avlTree/AvlTree.scala b/src/main/scala/encry/view/state/avlTree/AvlTree.scala index 9ab46c63ef..1dec310e47 100644 --- a/src/main/scala/encry/view/state/avlTree/AvlTree.scala +++ b/src/main/scala/encry/view/state/avlTree/AvlTree.scala @@ -446,7 +446,7 @@ object AvlTree extends StrictLogging { avlStorage: VersionalStorage, rootNodesStorage: RootNodesStorage[K, V] ): AvlTree[K, V] = { - rootNodesStorage.insert(StorageVersion @@ Array.fill(32)(0: Byte), EmptyNode(), Height @@ 0) + rootNodesStorage.insert(StorageVersion @@ Array.fill(32)(0: Byte), EmptyNode(), Height @@ -1) new AvlTree[K, V](EmptyNode(), avlStorage, rootNodesStorage) } From ad844c62c3de2749b39f608f7d9e802af6f8dc6a Mon Sep 17 00:00:00 2001 From: aleksandr Date: Sat, 7 Mar 2020 11:41:30 +0300 Subject: [PATCH 066/177] miner pipeline --- src/main/scala/encry/Starter.scala | 6 +- src/main/scala/encry/local/miner/Miner.scala | 83 +++++++++++-------- .../scala/encry/network/DeliveryManager.scala | 4 +- .../scala/encry/nvg/IntermediaryNVH.scala | 41 +++------ .../scala/encry/nvg/nvhg/NodeViewHolder.scala | 57 ++++++------- .../view/mempool/IntermediaryMempool.scala | 30 ++++--- .../view/mempool/TransactionsValidator.scala | 9 +- 7 files changed, 114 insertions(+), 116 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 265514609c..587b5bfff3 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -411,8 +411,6 @@ class Starter(settings: EncryAppSettings, } lazy val dataHolderForApi = context.system.actorOf(DataHolderForApi.props(newSettings, timeProvider), "dataHolder") - val miner: ActorRef = - context.system.actorOf(Miner.props(dataHolderForApi, influxRef, newSettings), "miner") // lazy val memoryPool: ActorRef = context.system.actorOf( // MemoryPool // .props(newSettings, timeProvider, miner, influxRef) @@ -446,9 +444,11 @@ class Starter(settings: EncryAppSettings, ) val memoryPool = context.system.actorOf( - IntermediaryMempool.props(newSettings, timeProvider, miner, influxRef, networkRouter) + IntermediaryMempool.props(newSettings, timeProvider, influxRef, networkRouter) ) + val miner: ActorRef = + context.system.actorOf(Miner.props(dataHolderForApi, memoryPool, nvhRouter, influxRef, newSettings), "miner") if (newSettings.node.mining) miner ! StartMining if (newSettings.node.useCli) { context.system diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index c61d078e4a..b43d9c6453 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -13,14 +13,15 @@ import encry.consensus.{CandidateBlock, EncrySupplyController, EquihashPowScheme import encry.local.miner.Miner._ import encry.local.miner.Worker.NextChallenge import encry.modifiers.mempool.TransactionFactory -import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.network.DeliveryManager +import encry.network.DeliveryManager.{BlockchainStatus, FullBlockChainIsSynced} +import encry.nvg.nvhg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier +import encry.nvg.nvhg.NodeViewHolder.{GetDataFromCurrentView, SemanticallySuccessfulModifier} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.NetworkTime.Time import encry.view.state.avlTree.utils.implicits.Instances._ import encry.view.NodeViewHolder.CurrentView -import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import encry.view.history.History import encry.view.mempool.MemoryPool.TransactionsForMiner import encry.view.state.UtxoState @@ -40,6 +41,8 @@ import scala.collection._ import scala.concurrent.duration._ class Miner(dataHolder: ActorRef, + mempool: ActorRef, + nvh: ActorRef, influx: Option[ActorRef], settings: EncryAppSettings) extends Actor with StrictLogging { @@ -62,6 +65,7 @@ class Miner(dataHolder: ActorRef, override def preStart(): Unit = { context.system.eventStream.subscribe(self, classOf[ClIMiner]) context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) + context.system.eventStream.subscribe(self, classOf[BlockchainStatus]) context.system.scheduler.schedule(5.seconds, 5.seconds)( influx.foreach(_ ! InfoAboutTransactionsFromMiner(transactionsPool.size)) ) @@ -79,7 +83,10 @@ class Miner(dataHolder: ActorRef, def needNewCandidate(b: Block): Boolean = !candidateOpt.flatMap(_.parentOpt).map(_.id).exists(id => Algos.encode(id) == Algos.encode(b.header.id)) - override def receive: Receive = if (settings.node.mining && syncingDone) miningEnabled else miningDisabled + override def receive: Receive = { + logger.info(s"settings.node.mining: ${settings.node.mining}. syncingDone: ${syncingDone}") + if (settings.node.mining && syncingDone) miningEnabled else miningDisabled + } def mining: Receive = { case StartMining if context.children.nonEmpty & syncingDone => @@ -113,7 +120,7 @@ class Miner(dataHolder: ActorRef, s" from worker $workerIdx with nonce: ${block.header.nonce}.") logger.debug(s"Set previousSelfMinedBlockId: ${Algos.encode(block.id)}") killAllWorkers() - //context.actorSelection("/user/nodeViewHolder") ! LocallyGeneratedModifier(block) + nvh ! LocallyGeneratedModifier(block) if (settings.influxDB.isDefined) { context.actorSelection("/user/statsSender") ! MiningEnd(block.header, workerIdx, context.children.size) context.actorSelection("/user/statsSender") ! MiningTime(System.currentTimeMillis() - startTime) @@ -131,9 +138,11 @@ class Miner(dataHolder: ActorRef, def miningDisabled: Receive = { case EnableMining => + logger.info("Enable mining on miner!") context.become(miningEnabled) self ! StartMining case FullBlockChainIsSynced => + logger.info("Set syncingDone to true") syncingDone = true if (settings.node.mining) self ! EnableMining case DisableMining | SemanticallySuccessfulModifier(_) => @@ -144,7 +153,7 @@ class Miner(dataHolder: ActorRef, logger.info(s"Got new block. Starting to produce candidate at height: ${mod.header.height + 1} " + s"at ${dateFormat.format(new Date(System.currentTimeMillis()))}") produceCandidate() - case SemanticallySuccessfulModifier(_) => + case SemanticallySuccessfulModifier(_) => logger.info("Got new block. But needNewCandidate - false") } def receiverCandidateBlock: Receive = { @@ -160,7 +169,9 @@ class Miner(dataHolder: ActorRef, } def chainEvents: Receive = { - case FullBlockChainIsSynced => syncingDone = true + case FullBlockChainIsSynced => + logger.info("Set syncingDone on miner to true") + syncingDone = true } def procCandidateBlock(c: CandidateBlock): Unit = { @@ -216,31 +227,33 @@ class Miner(dataHolder: ActorRef, candidate } - def produceCandidate(): Unit = - context.actorSelection("/user/nodeViewHolder") ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope] { - nodeView => - val producingStartTime: Time = System.currentTimeMillis() - startTime = producingStartTime - val bestHeaderOpt: Option[Header] = nodeView.history.getBestBlock.map(_.header) - bestHeaderOpt match { - case Some(h) => logger.info(s"Best header at height ${h.height}") - case None => logger.info(s"No best header opt") - } - val candidate: CandidateEnvelope = - if ((bestHeaderOpt.isDefined && - (syncingDone || nodeView.history.isFullChainSynced)) || settings.node.offlineGeneration) { - logger.info(s"Starting candidate generation at " + - s"${dateFormat.format(new Date(System.currentTimeMillis()))}") - if (settings.influxDB.isDefined) - context.actorSelection("user/statsSender") ! SleepTime(System.currentTimeMillis() - sleepTime) - logger.info("Going to calculate last block:") - val envelope: CandidateEnvelope = - CandidateEnvelope - .fromCandidate(createCandidate(nodeView, bestHeaderOpt)) - envelope - } else CandidateEnvelope.empty - candidate + def produceCandidate(): Unit = { + val lambda = (nodeView: CurrentView[History, UtxoState, EncryWallet]) => + { + val producingStartTime: Time = System.currentTimeMillis() + startTime = producingStartTime + val bestHeaderOpt: Option[Header] = nodeView.history.getBestBlock.map(_.header) + bestHeaderOpt match { + case Some(h) => logger.info(s"Best header at height ${h.height}") + case None => logger.info(s"No best header opt") + } + val candidate: CandidateEnvelope = + if ((bestHeaderOpt.isDefined && + (syncingDone || nodeView.history.isFullChainSynced)) || settings.node.offlineGeneration) { + logger.info(s"Starting candidate generation at " + + s"${dateFormat.format(new Date(System.currentTimeMillis()))}") + if (settings.influxDB.isDefined) + context.actorSelection("user/statsSender") ! SleepTime(System.currentTimeMillis() - sleepTime) + logger.info("Going to calculate last block:") + val envelope: CandidateEnvelope = + CandidateEnvelope + .fromCandidate(createCandidate(nodeView, bestHeaderOpt)) + envelope + } else CandidateEnvelope.empty + candidate } + nvh ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope] (lambda, self) + } } object Miner { @@ -276,6 +289,10 @@ object Miner { "candidateBlock" -> r.candidateBlock.map(_.asJson).getOrElse("None".asJson) ).asJson - def props(dataHolder: ActorRef, influx: Option[ActorRef], settings: EncryAppSettings): Props = - Props(new Miner(dataHolder, influx, settings)) + def props(dataHolder: ActorRef, + mempool: ActorRef, + nvh: ActorRef, + influx: Option[ActorRef], + settings: EncryAppSettings): Props = + Props(new Miner(dataHolder, mempool, nvh, influx, settings)) } \ No newline at end of file diff --git a/src/main/scala/encry/network/DeliveryManager.scala b/src/main/scala/encry/network/DeliveryManager.scala index dd7f913792..64ade81e21 100644 --- a/src/main/scala/encry/network/DeliveryManager.scala +++ b/src/main/scala/encry/network/DeliveryManager.scala @@ -511,7 +511,9 @@ object DeliveryManager { case object CheckPayloadsToDownload - final case object FullBlockChainIsSynced + trait BlockchainStatus + + final case object FullBlockChainIsSynced extends BlockchainStatus final case class CheckModifiersWithQueueSize(size: Int) extends AnyVal diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 5947d99e8a..8480ba5681 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -1,48 +1,29 @@ package encry.nvg -import akka.actor.{ Actor, ActorRef, Props } +import akka.actor.{Actor, ActorRef, Props} import akka.routing.BalancingPool import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.local.miner.Miner.{DisableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } -import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } -import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } +import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} +import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeersKeeper.BanPeer -import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation } +import encry.nvg.ModifiersValidator.{InvalidModifierBytes, ModifierForValidation} import encry.nvg.fast.sync.SnapshotProcessor -import encry.nvg.nvhg.NodeViewHolder.{ - RollbackFailed, - RollbackSucceed, - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification, - UpdateHistoryReader -} -import encry.nvg.fast.sync.SnapshotProcessor.{ - FastSyncDone, - HeaderChainIsSynced, - RequiredManifestHeightAndId, - TreeChunks -} +import encry.nvg.nvhg.NodeViewHolder.{GetDataFromCurrentView, RollbackFailed, RollbackSucceed, SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification, UpdateHistoryReader} +import encry.nvg.fast.sync.SnapshotProcessor.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks} import encry.nvg.nvhg.NodeViewHolder +import encry.nvg.nvhg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions -import org.encryfoundation.common.network.BasicMessagesRepo.{ - InvNetworkMessage, - RequestChunkMessage, - RequestManifestMessage, - RequestModifiersNetworkMessage, - ResponseChunkMessage, - ResponseManifestMessage, - SyncInfoNetworkMessage -} +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestChunkMessage, RequestManifestMessage, RequestModifiersNetworkMessage, ResponseChunkMessage, ResponseManifestMessage, SyncInfoNetworkMessage} import org.encryfoundation.common.utils.Algos class IntermediaryNVH( @@ -107,6 +88,7 @@ class IntermediaryNVH( case msg @ UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! msg + case msg: LocallyGeneratedModifier => nodeViewHolder ! msg case msg @ BanPeer(_, _) => intermediaryNetwork ! msg case msg @ InvalidModifierBytes(_) => intermediaryNetwork ! msg case msg @ OtherNodeSyncingStatus(_, _, _) => intermediaryNetwork ! msg @@ -125,6 +107,7 @@ class IntermediaryNVH( case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder case msg @ RolledBackTransactions(_) => //+ to memory pool case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) + case msg @ GetDataFromCurrentView(_, _) => nodeViewHolder ! msg case msg @ RollbackSucceed(_) => case msg @ RollbackFailed(_) => case msg @ SemanticallySuccessfulModifier(_) => diff --git a/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala b/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala index 6dcf1745e5..d9121897d4 100644 --- a/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala @@ -2,61 +2,48 @@ package encry.nvg.nvhg import java.io.File -import akka.actor.{ Actor, ActorRef, Props } +import akka.actor.{Actor, ActorRef, Props} +import akka.pattern._ import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.local.miner.Miner.{DisableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersCache import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.fast.sync.SnapshotProcessor.{ - FastSyncDone, - FastSyncFinished, - HeaderChainIsSynced, - RemoveRedundantManifestIds, - SnapshotChunk, - TreeChunks -} +import encry.nvg.fast.sync.SnapshotProcessor.{FastSyncDone, FastSyncFinished, HeaderChainIsSynced, RemoveRedundantManifestIds, SnapshotChunk, TreeChunks} import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId -import encry.nvg.nvhg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.nvg.nvhg.NodeViewHolder.{ - NodeView, - RollbackFailed, - RollbackSucceed, - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification, - UpdateHistoryReader, - UpdateInformation -} +import encry.nvg.nvhg.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} +import encry.nvg.nvhg.NodeViewHolder.{GetDataFromCurrentView, NodeView, RollbackFailed, RollbackSucceed, SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification, UpdateHistoryReader, UpdateInformation} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.NodeViewErrors.ModifierApplyError import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError +import encry.view.NodeViewHolder.CurrentView import encry.view.history.storage.HistoryStorage -import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } +import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} import encry.view.mempool.MemoryPool.RolledBackTransactions import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } +import org.encryfoundation.common.modifiers.{PersistentModifier, PersistentNodeViewModifier} import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } +import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} -import scala.collection.{ mutable, IndexedSeq, Seq } +import scala.collection.{IndexedSeq, Seq, mutable} +import scala.concurrent.Future import scala.concurrent.duration._ -import scala.util.{ Failure, Success, Try } +import scala.util.{Failure, Success, Try} class NodeViewHolder( settings: EncryAppSettings, @@ -121,6 +108,13 @@ class NodeViewHolder( s"is ${(System.currentTimeMillis() - startTime) / 1000}s." ) + case GetDataFromCurrentView(f, sender) => + logger.info("Receive GetDataFromCurrentView on nvh") + f(CurrentView(nodeView.history, nodeView.state, nodeView.wallet)) match { + case resultFuture: Future[_] => resultFuture.pipeTo(sender) + case result => sender ! result + } + case FastSyncFinished(state: UtxoState, wallet: EncryWallet) => val startTime: Long = System.currentTimeMillis() logger.info(s"Got a signal about finishing fast sync process.") @@ -290,7 +284,7 @@ class NodeViewHolder( } if (settings.node.mining && progressInfo.chainSwitchingNeeded) context.parent ! StartMining - context.parent ! SemanticallySuccessfulModifier(modToApply) + context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) if (newHis.getBestHeaderId.exists( bestHeaderId => newHis.getBestBlockId.exists(bId => ByteArrayWrapper(bId) == ByteArrayWrapper(bestHeaderId)) @@ -359,16 +353,17 @@ class NodeViewHolder( logger.debug(s"Persistent modifier ${modifier.encodedId} was applied successfully.") newHistory.getBestHeader.foreach(context.parent ! BestHeaderInChain(_)) if (newHistory.isFullChainSynced) { - logger.debug(s"BlockChain is synced on nvh at the height ${newHistory.getBestHeaderHeight}.") + logger.info(s"BlockChain is synced on nvh at the height ${newHistory.getBestHeaderHeight}.") ModifiersCache.setChainSynced() context.parent ! FullBlockChainIsSynced + context.system.eventStream.publish(FullBlockChainIsSynced) } updateNodeView(newHistory.some, newState.some, nodeView.wallet.some) } else { logger.info(s"Progress info is empty.") context.parent ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height) if (!isLocallyGenerated) requestDownloads(progressInfo, modifier.id.some) - context.parent ! SemanticallySuccessfulModifier(modifier) + context.system.eventStream.publish(SemanticallySuccessfulModifier(modifier)) updateNodeView(updatedHistory = historyBeforeStUpdate.some) } case Left(e: Throwable) => @@ -525,6 +520,8 @@ object NodeViewHolder { case class SemanticallySuccessfulModifier(modifier: PersistentNodeViewModifier) extends ModificationOutcome + case class GetDataFromCurrentView[HIS, MS, VL, A](f: CurrentView[HIS, MS, VL] => A, sender: ActorRef) + final case class DownloadRequest( modifierTypeId: ModifierTypeId, modifierIds: List[ModifierId] diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala index 62b41dc8bb..acf38c9982 100644 --- a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala +++ b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala @@ -11,12 +11,11 @@ import encry.stats.StatsSender.ValidatedModifierFromNetwork import encry.utils.NetworkTimeProvider import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.{RolledBackTransactions, TransactionProcessing, TransactionsForMiner} -import encry.view.mempool.TransactionsValidator.{InvalidModifier, ModifiersForValidating} +import encry.view.mempool.TransactionsValidator.{InvalidTransaction, ModifiersForValidating} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction class IntermediaryMempool(settings: EncryAppSettings, networkTimeProvider: NetworkTimeProvider, - minerReference: ActorRef, influxReference: Option[ActorRef], networkRouter: ActorRef) extends Actor @@ -35,27 +34,26 @@ class IntermediaryMempool(settings: EncryAppSettings, } override def receive(): Receive = { - case msg @ InvalidModifier(_) => // to nvsh - case msg @ BanPeer(_, _) => // to peersKeeper - case msg @ ValidatedModifierFromNetwork(_) => // to influx - case msg @ TransactionsForMiner(_) => minerReference ! msg// to miner - case msg @ RolledBackTransactions(_) => memoryPool ! msg // to mempool - case msg @ ModifiersForValidating(_, _, _) => memoryPool ! msg // to mempool - case msg @ DataFromPeer(_, _) => mempoolProcessor ! msg // to mempool processor - case msg @ RequestFromLocal(_, _, _) => // to network - case msg @ ModifierFromNetwork(_, _, _, _) => txValidator ! msg - case msg @ TransactionProcessing(_) => mempoolProcessor ! msg // to mempool processor - case msg @ IsChainSynced(_) => mempoolProcessor ! msg + case msg: InvalidTransaction => networkRouter ! msg + case msg: BanPeer => networkRouter ! msg + case msg: ValidatedModifierFromNetwork => // to influx + case msg: RolledBackTransactions => memoryPool ! msg // to mempool + case msg: ModifiersForValidating => memoryPool ! msg // to mempool + case msg: DataFromPeer => mempoolProcessor ! msg // to mempool processor + case msg: RequestFromLocal => networkRouter ! msg + case msg: ModifierFromNetwork => txValidator ! msg + case msg: TransactionProcessing => mempoolProcessor ! msg // to mempool processor + case msg: IsChainSynced => mempoolProcessor ! msg } } object IntermediaryMempool { + def props(settings: EncryAppSettings, networkTimeProvider: NetworkTimeProvider, - minerReference: ActorRef, influxReference: Option[ActorRef], - networkRouter: ActorRef) = - Props(new IntermediaryMempool(settings, networkTimeProvider, minerReference, influxReference, networkRouter)) + networkRouter: ActorRef): Props = + Props(new IntermediaryMempool(settings, networkTimeProvider, influxReference, networkRouter)) final case class TransactionsForValidating(tx: Transaction) diff --git a/src/main/scala/encry/view/mempool/TransactionsValidator.scala b/src/main/scala/encry/view/mempool/TransactionsValidator.scala index c9c7cc571c..fe00f24a46 100644 --- a/src/main/scala/encry/view/mempool/TransactionsValidator.scala +++ b/src/main/scala/encry/view/mempool/TransactionsValidator.scala @@ -7,10 +7,11 @@ import encry.network.BlackList.BanReason.{CorruptedSerializedBytes, Syntacticall import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeer +import encry.nvg.ModifiersValidator.InvalidModifierBytes import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.mempool.MemoryPool.NewTransaction -import encry.view.mempool.TransactionsValidator.{InvalidModifier, ModifiersForValidating} +import encry.view.mempool.TransactionsValidator.{InvalidTransaction, ModifiersForValidating} import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionProtoSerializer} import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} @@ -32,10 +33,10 @@ class TransactionsValidator(settings: EncryAppSettings, case Success(tx) => logger.info(s"Transaction with id: ${tx.encodedId} invalid cause of: ${tx.semanticValidity}.") context.parent ! BanPeer(remote, SyntacticallyInvalidTransaction) - context.parent ! InvalidModifier(id) + context.parent ! InvalidTransaction(id) case Failure(ex) => context.parent ! BanPeer(remote, CorruptedSerializedBytes) - context.parent ! InvalidModifier(id) + context.parent ! InvalidModifierBytes(id) logger.info(s"Received modifier from $remote can't be parsed cause of: ${ex.getMessage}.") } } @@ -48,7 +49,7 @@ object TransactionsValidator { typeId: ModifierTypeId, modifiers: Map[ModifierId, Array[Byte]]) - final case class InvalidModifier(ids: ModifierId) extends AnyVal + final case class InvalidTransaction(ids: ModifierId) extends AnyVal def props(settings: EncryAppSettings, memPool: ActorRef, ntp: NetworkTimeProvider): Props = Props(new TransactionsValidator(settings, memPool, ntp)) From 901439c10f3cbe05b2f7679167afc52d67b23605 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 10 Mar 2020 11:19:18 +0300 Subject: [PATCH 067/177] add logging + fix re requesting --- src/main/scala/encry/network/DM.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 72804c6a5c..d8cf9f126f 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -40,18 +40,21 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging receivedModifier += toKey(id) } else logger.info("Receive spam!") } - case RequestSent(peer, modTypeId, modId) => + case RequestSent(peer, modTypeId, modId) if !expectedModifiers.contains(toKey(modId))=> expectedModifiers += toKey(modId) context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)( self ! AwaitingRequest(peer, modTypeId, modId, 1) ) + case RequestSent(_, _, _) => //do nothing case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts <= networkSettings.maxDeliveryChecks && expectedModifiers.contains(toKey(modId))=> context.parent ! RequestFromLocal(peer.some, modTypeId, List(modId)) + logger.info(s"Re-request modifier ${Algos.encode(modId)}") context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! AwaitingRequest(peer, modTypeId, modId, attempts + 1) ) - case AwaitingRequest(peer, _, modId, _) => - logger.info(s"Stop requesting modifier ${Algos.encode(modId)} from peer $peer") + case AwaitingRequest(peer, _, modId, attempts) => + logger.info(s"Stop requesting modifier ${Algos.encode(modId)} from peer $peer, qty of attempts $attempts." + + s" Expected modifier contains: ${expectedModifiers.contains(toKey(modId))}") case ModifierFromNetwork(source, modTypeId, modId, modBytes) => if (expectedModifiers.contains(toKey(modId))) { expectedModifiers -= toKey(modId) From b61733f114adf2c25a694afc6ba359dc00ecfb6e Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 10 Mar 2020 11:27:55 +0300 Subject: [PATCH 068/177] remove mod from expected after failed re-requesting --- src/main/scala/encry/network/DM.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index d8cf9f126f..4a4e0510ce 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -55,6 +55,7 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging case AwaitingRequest(peer, _, modId, attempts) => logger.info(s"Stop requesting modifier ${Algos.encode(modId)} from peer $peer, qty of attempts $attempts." + s" Expected modifier contains: ${expectedModifiers.contains(toKey(modId))}") + expectedModifiers -= toKey(modId) case ModifierFromNetwork(source, modTypeId, modId, modBytes) => if (expectedModifiers.contains(toKey(modId))) { expectedModifiers -= toKey(modId) From 55a5414000082852a767cfadda6ebd432864ae00 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 10 Mar 2020 11:53:30 +0300 Subject: [PATCH 069/177] state/history separation frame --- src/main/scala/encry/nvg/ModifiersCache.scala | 17 +- .../scala/encry/nvg/nvhg/HistoryApplier.scala | 6 +- .../nvg/nvhg/NodeViewHolderController.scala | 179 +++++++++++++++++- .../scala/encry/nvg/nvhg/StateApplier.scala | 6 +- 4 files changed, 189 insertions(+), 19 deletions(-) diff --git a/src/main/scala/encry/nvg/ModifiersCache.scala b/src/main/scala/encry/nvg/ModifiersCache.scala index 07ed709065..24006111cb 100644 --- a/src/main/scala/encry/nvg/ModifiersCache.scala +++ b/src/main/scala/encry/nvg/ModifiersCache.scala @@ -62,7 +62,7 @@ object ModifiersCache extends StrictLogging { } def popCandidate(history: History): List[PersistentModifier] = synchronized { - findCandidateKey(history).flatMap(k => remove(k)) + findCandidateKey(history).take(1).flatMap(k => remove(k)) } override def toString: String = cache.keys.map(key => Algos.encode(key.toArray)).mkString(",") @@ -91,13 +91,6 @@ object ModifiersCache extends StrictLogging { List.empty[Key] } - def findApplicablePayloadAtHeight(height: Int): List[Key] = { - history.headerIdsAtHeight(height).view.flatMap(history.getHeaderById).collect { - case header: Header if isApplicable(new mutable.WrappedArray.ofByte(header.payloadId)) => - new mutable.WrappedArray.ofByte(header.payloadId) - } - }.toList - def exhaustiveSearch: List[Key] = List(cache.find { case (k, v) => @@ -109,14 +102,6 @@ object ModifiersCache extends StrictLogging { } }).collect { case Some(v) => v._1 } - @tailrec - def applicableBestPayloadChain(atHeight: Int = history.getBestBlockHeight, - prevKeys: List[Key] = List.empty[Key]): List[Key] = { - val payloads = findApplicablePayloadAtHeight(atHeight) - if (payloads.nonEmpty) applicableBestPayloadChain(atHeight + 1, prevKeys ++ payloads) - else prevKeys - } - val bestHeadersIds: List[Key] = { headersCollection.get(history.getBestHeaderHeight + 1) match { case Some(value) => diff --git a/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala b/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala index 3f47124fd2..4e75ae2909 100644 --- a/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala +++ b/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala @@ -1,5 +1,9 @@ package encry.nvg.nvhg -class HistoryApplier { +import akka.actor.Actor +class HistoryApplier extends Actor { + override def receive: Receive = { + case _ => + } } diff --git a/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala b/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala index f64e04a22f..80b43b6539 100644 --- a/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala +++ b/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala @@ -1,5 +1,182 @@ package encry.nvg.nvhg -class NodeViewHolderController { +import java.io.File +import akka.actor.{ Actor, ActorRef } +import cats.syntax.option._ +import com.typesafe.scalalogging.StrictLogging +import encry.nvg.ModifiersCache +import encry.nvg.ModifiersValidator.ValidatedModifier +import encry.nvg.nvhg.NodeViewHolder.NodeView +import encry.nvg.nvhg.NodeViewHolderController.{ ApplicableModifier, GetNewApplicable, HistoryIsReady, StateIsReady } +import encry.settings.EncryAppSettings +import encry.utils.CoreTaggedTypes.VersionTag +import encry.utils.NetworkTimeProvider +import encry.view.history.History +import encry.view.state.UtxoState +import encry.view.wallet.EncryWallet +import org.apache.commons.io.FileUtils +import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.history.Block +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.ADDigest + +import scala.collection.mutable + +class NodeViewHolderController( + settings: EncryAppSettings, + ntp: NetworkTimeProvider, + influxRef: Option[ActorRef] +) extends Actor + with StrictLogging { + + var nodeView: NodeView = restoreState().getOrElse(genesisState) + + val historyApplier: ActorRef = Actor.noSender + + val stateApplier: ActorRef = Actor.noSender + + override def receive: Receive = mainReceive.orElse(processModifierFromNetwork) + + def mainReceive: Receive = { + case GetNewApplicable => + val mod: List[PersistentModifier] = ModifiersCache.popCandidate(nodeView.history) + mod.headOption.foreach { mod: PersistentModifier => + historyApplier ! ApplicableModifier(mod) + stateApplier ! ApplicableModifier(mod) + logger.info(s"Applicable modifier ${mod.encodedId} of type ${mod.modifierTypeId} sent to history and state.") + context.become( + awaitApplication(historyIsReady = false, stateIsReady = false).orElse(processModifierFromNetwork) + ) + } + } + + def awaitApplication(historyIsReady: Boolean, stateIsReady: Boolean): Receive = { + case HistoryIsReady => + if (stateIsReady) { + self ! GetNewApplicable + context.become(mainReceive.orElse(processModifierFromNetwork)) + } else context.become(awaitApplication(historyIsReady = true, stateIsReady).orElse(processModifierFromNetwork)) + case StateIsReady => + if (historyIsReady) { + self ! GetNewApplicable + context.become(mainReceive.orElse(processModifierFromNetwork)) + } else context.become(awaitApplication(historyIsReady, stateIsReady = true).orElse(processModifierFromNetwork)) + } + + def processModifierFromNetwork: Receive = { + case ValidatedModifier(modifier: PersistentModifier) => + val wrappedKey: mutable.WrappedArray.ofByte = NodeViewHolder.toKey(modifier.id) + val isInHistory: Boolean = nodeView.history.isModifierDefined(modifier.id) + val isInCache: Boolean = ModifiersCache.contains(wrappedKey) + if (isInHistory || isInCache) + logger.info( + s"Modifier ${modifier.encodedId} can't be placed into the cache cause: " + + s"contains in cache: $isInCache, contains in history: $isInHistory." + ) + else { + ModifiersCache.put(wrappedKey, modifier, nodeView.history) + self ! GetNewApplicable + } + } + + def genesisState: NodeView = { + val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdir() + val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() + assert(stateDir.listFiles().isEmpty, s"Genesis directory $stateDir should always be empty.") + val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) + val history: History = History.readOrGenerate(settings, ntp) + val wallet: EncryWallet = + EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) + NodeView(history, state, wallet) + } + + def restoreState(influxRef: Option[ActorRef] = none): Option[NodeView] = + if (History.getHistoryIndexDir(settings).listFiles.nonEmpty) + try { + val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdirs() + val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() + val history: History = History.readOrGenerate(settings, ntp) + val wallet: EncryWallet = + EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) + val state: UtxoState = restoreConsistentState( + UtxoState.create(stateDir, rootsDir, settings, influxRef), + history, + influxRef + ) + history.updateIdsForSyncInfo() + logger.info(s"History best block height: ${history.getBestBlockHeight}") + logger.info(s"History best header height: ${history.getBestHeaderHeight}") + NodeView(history, state, wallet).some + } catch { + case ex: Throwable => + logger.info(s"${ex.getMessage} during state restore. Recover from Modifiers holder!") + new File(settings.directory).listFiles.foreach(dir => FileUtils.cleanDirectory(dir)) + genesisState.some + } else { + none + } + + def getRecreatedState( + version: Option[VersionTag] = none, + digest: Option[ADDigest] = none, + influxRef: Option[ActorRef] + ): UtxoState = { + val dir: File = UtxoState.getStateDir(settings) + dir.mkdirs() + dir.listFiles.foreach(_.delete()) + val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdirs() + val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() + UtxoState.create(stateDir, rootsDir, settings, influxRef) + } + + def restoreConsistentState( + stateIn: UtxoState, + history: History, + influxRefActor: Option[ActorRef] + ): UtxoState = + (stateIn.version, history.getBestBlock, stateIn, stateIn.safePointHeight) match { + case (stateId, None, _, _) if stateId sameElements Array.emptyByteArray => + logger.info(s"State and history are both empty on startup") + stateIn + case (_, None, _, _) => + logger.info( + s"State and history are inconsistent." + + s" History is empty on startup, rollback state to genesis." + ) + getRecreatedState(influxRef = influxRefActor) + case (_, Some(historyBestBlock), state: UtxoState, safePointHeight) => + val headerAtSafePointHeight = history.getBestHeaderAtHeight(safePointHeight) + val (rollbackId, newChain) = history.getChainToHeader(headerAtSafePointHeight, historyBestBlock.header) + logger.info( + s"State and history are inconsistent." + + s" Going to rollback to ${rollbackId.map(Algos.encode)} and " + + s"apply ${newChain.length} modifiers. State safe point: ${safePointHeight}. ${newChain.headers.head.height}. ${newChain.headers.last.height}" + ) + val additionalBlocks = + (state.safePointHeight + 1 to historyBestBlock.header.height).foldLeft(List.empty[Block]) { + case (blocks, height) => + val headerAtHeight = history.getBestHeaderAtHeight(height).get + val blockAtHeight = history.getBlockByHeader(headerAtHeight).get + blocks :+ blockAtHeight + } + logger.info(s"Qty of additional blocks: ${additionalBlocks.length}") + rollbackId + .map(_ => state.restore(additionalBlocks).get) + .getOrElse(getRecreatedState(influxRef = influxRefActor)) + } +} + +object NodeViewHolderController { + + final case class ApplicableModifier(modifier: PersistentModifier) + case object HistoryIsReady + case object StateIsReady + case object GetNewApplicable } diff --git a/src/main/scala/encry/nvg/nvhg/StateApplier.scala b/src/main/scala/encry/nvg/nvhg/StateApplier.scala index 5316a2b46a..a94bfee293 100644 --- a/src/main/scala/encry/nvg/nvhg/StateApplier.scala +++ b/src/main/scala/encry/nvg/nvhg/StateApplier.scala @@ -1,5 +1,9 @@ package encry.nvg.nvhg -class StateApplier { +import akka.actor.Actor +class StateApplier extends Actor { + override def receive: Receive = { + case _ => + } } From aafceb3e1779fd85177bcd1992da8b7fd91984ec Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 10 Mar 2020 11:55:08 +0300 Subject: [PATCH 070/177] state/history separation frame --- src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala b/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala index d9121897d4..91afd77947 100644 --- a/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala @@ -39,7 +39,6 @@ import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.modifiers.{PersistentModifier, PersistentNodeViewModifier} import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} - import scala.collection.{IndexedSeq, Seq, mutable} import scala.concurrent.Future import scala.concurrent.duration._ From da7a7c718b7f1f2e584508b3cf458940194aa1e9 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 10 Mar 2020 13:34:43 +0300 Subject: [PATCH 071/177] removed history\state separation --- .../encry/api/http/DataHolderForApi.scala | 2 +- src/main/scala/encry/local/miner/Miner.scala | 4 +- src/main/scala/encry/network/DM.scala | 2 +- .../scala/encry/network/NetworkRouter.scala | 2 +- .../encry/network/NodeViewSynchronizer.scala | 2 +- .../scala/encry/nvg/IntermediaryNVH.scala | 42 ++-- src/main/scala/encry/nvg/ModifiersCache.scala | 2 - .../scala/encry/nvg/ModifiersValidator.scala | 4 +- .../encry/nvg/NetworkMessagesProcessor.scala | 18 +- .../encry/nvg/{nvhg => }/NodeViewHolder.scala | 31 ++- .../nvg/fast/sync/SnapshotDownloader.scala | 2 +- .../nvg/fast/sync/SnapshotProcessor.scala | 2 +- .../scala/encry/nvg/nvhg/HistoryApplier.scala | 9 - .../nvg/nvhg/NodeViewHolderController.scala | 182 ------------------ .../scala/encry/nvg/nvhg/StateApplier.scala | 9 - .../scala/encry/view/mempool/MemoryPool.scala | 2 +- 16 files changed, 67 insertions(+), 248 deletions(-) rename src/main/scala/encry/nvg/{nvhg => }/NodeViewHolder.scala (95%) delete mode 100644 src/main/scala/encry/nvg/nvhg/HistoryApplier.scala delete mode 100644 src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala delete mode 100644 src/main/scala/encry/nvg/nvhg/StateApplier.scala diff --git a/src/main/scala/encry/api/http/DataHolderForApi.scala b/src/main/scala/encry/api/http/DataHolderForApi.scala index f29e856311..935b592d8d 100644 --- a/src/main/scala/encry/api/http/DataHolderForApi.scala +++ b/src/main/scala/encry/api/http/DataHolderForApi.scala @@ -21,7 +21,7 @@ import encry.network.ConnectedPeersCollection import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeerFromAPI -import encry.nvg.nvhg.NodeViewHolder.NodeViewChange +import encry.nvg.NodeViewHolder.NodeViewChange import encry.settings.EncryAppSettings import encry.utils.{NetworkTime, NetworkTimeProvider} import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index b43d9c6453..8512ddfd23 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -15,8 +15,8 @@ import encry.local.miner.Worker.NextChallenge import encry.modifiers.mempool.TransactionFactory import encry.network.DeliveryManager import encry.network.DeliveryManager.{BlockchainStatus, FullBlockChainIsSynced} -import encry.nvg.nvhg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier -import encry.nvg.nvhg.NodeViewHolder.{GetDataFromCurrentView, SemanticallySuccessfulModifier} +import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier +import encry.nvg.NodeViewHolder.{GetDataFromCurrentView, SemanticallySuccessfulModifier} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.NetworkTime.Time diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 4a4e0510ce..b766d63723 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -14,7 +14,7 @@ import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMess import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import cats.syntax.option._ -import encry.nvg.nvhg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier} +import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier} import scala.collection.mutable import scala.concurrent.Future diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index a98a3393f5..9ceb8a9576 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -18,7 +18,7 @@ import encry.network.PeerConnectionHandler.ReceivableMessages.StartInteraction import encry.network.PeerConnectionHandler.{ConnectedPeer, MessageFromNetwork} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionVerified, NewConnection, OutgoingConnectionFailed} import encry.network.PeersKeeper.{BanPeer, ConnectionStatusMessages, PeerForConnection, RequestPeerForConnection} -import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, NetworkMessage} diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index 9313e51524..e3d4b4bbbb 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -33,7 +33,7 @@ import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import scala.concurrent.duration._ import encry.network.ModifiersToNetworkUtils._ -import encry.nvg.nvhg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} +import encry.nvg.NodeViewHolder.{NodeViewChange, NodeViewHolderEvent, SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} import scala.util.Try diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 8480ba5681..3539027feb 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -1,29 +1,49 @@ package encry.nvg -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import akka.routing.BalancingPool import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo -import encry.local.miner.Miner.{DisableMining, StartMining} +import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} -import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} -import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } +import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } +import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeersKeeper.BanPeer -import encry.nvg.ModifiersValidator.{InvalidModifierBytes, ModifierForValidation} +import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation } import encry.nvg.fast.sync.SnapshotProcessor -import encry.nvg.nvhg.NodeViewHolder.{GetDataFromCurrentView, RollbackFailed, RollbackSucceed, SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification, UpdateHistoryReader} -import encry.nvg.fast.sync.SnapshotProcessor.{FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks} -import encry.nvg.nvhg.NodeViewHolder -import encry.nvg.nvhg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier +import encry.nvg.NodeViewHolder.{ + GetDataFromCurrentView, + RollbackFailed, + RollbackSucceed, + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification, + UpdateHistoryReader +} +import encry.nvg.fast.sync.SnapshotProcessor.{ + FastSyncDone, + HeaderChainIsSynced, + RequiredManifestHeightAndId, + TreeChunks +} +import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader import encry.view.mempool.MemoryPool.RolledBackTransactions -import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestChunkMessage, RequestManifestMessage, RequestModifiersNetworkMessage, ResponseChunkMessage, ResponseManifestMessage, SyncInfoNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{ + InvNetworkMessage, + RequestChunkMessage, + RequestManifestMessage, + RequestModifiersNetworkMessage, + ResponseChunkMessage, + ResponseManifestMessage, + SyncInfoNetworkMessage +} import org.encryfoundation.common.utils.Algos class IntermediaryNVH( diff --git a/src/main/scala/encry/nvg/ModifiersCache.scala b/src/main/scala/encry/nvg/ModifiersCache.scala index 24006111cb..9a2df13159 100644 --- a/src/main/scala/encry/nvg/ModifiersCache.scala +++ b/src/main/scala/encry/nvg/ModifiersCache.scala @@ -7,8 +7,6 @@ import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId - -import scala.annotation.tailrec import scala.collection.concurrent.TrieMap import scala.collection.immutable.SortedMap import scala.collection.mutable diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index 5dfa628e8b..e3990ec576 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -16,7 +16,7 @@ import encry.network.BlackList.BanReason.{ } import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation, ValidatedModifier } -import encry.nvg.nvhg.NodeViewHolder.SyntacticallyFailedModification +import encry.nvg.NodeViewHolder.SyntacticallyFailedModification import encry.settings.EncryAppSettings import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier @@ -55,8 +55,6 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings } } - - private def isPreSemanticValid( modifier: PersistentModifier, historyReader: HistoryReader, diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 75d88241b7..8415387d83 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -1,22 +1,26 @@ package encry.nvg -import akka.actor.{Actor, Cancellable, Props} +import akka.actor.{ Actor, Cancellable, Props } import com.typesafe.scalalogging.StrictLogging -import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Younger} +import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } import cats.syntax.option._ import encry.network.DeliveryManager.CheckPayloadsToDownload -import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo} +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus -import encry.nvg.nvhg.NodeViewHolder -import encry.nvg.nvhg.NodeViewHolder.{SemanticallySuccessfulModifier, UpdateHistoryReader} +import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, UpdateHistoryReader } import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} -import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.network.BasicMessagesRepo.{ + InvNetworkMessage, + ModifiersNetworkMessage, + RequestModifiersNetworkMessage, + SyncInfoNetworkMessage +} import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId diff --git a/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala similarity index 95% rename from src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala rename to src/main/scala/encry/nvg/NodeViewHolder.scala index 91afd77947..237cf9959c 100644 --- a/src/main/scala/encry/nvg/nvhg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -1,24 +1,23 @@ -package encry.nvg.nvhg +package encry.nvg import java.io.File -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import akka.pattern._ import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{DisableMining, StartMining} +import encry.local.miner.Miner.{ DisableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ -import encry.nvg.ModifiersCache import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.fast.sync.SnapshotProcessor.{FastSyncDone, FastSyncFinished, HeaderChainIsSynced, RemoveRedundantManifestIds, SnapshotChunk, TreeChunks} +import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } +import encry.nvg.NodeViewHolder._ import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId -import encry.nvg.nvhg.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} -import encry.nvg.nvhg.NodeViewHolder.{GetDataFromCurrentView, NodeView, RollbackFailed, RollbackSucceed, SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification, UpdateHistoryReader, UpdateInformation} +import encry.nvg.fast.sync.SnapshotProcessor._ import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag @@ -27,22 +26,23 @@ import encry.view.NodeViewErrors.ModifierApplyError import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.NodeViewHolder.CurrentView import encry.view.history.storage.HistoryStorage -import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} +import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } import encry.view.mempool.MemoryPool.RolledBackTransactions import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.modifiers.{PersistentModifier, PersistentNodeViewModifier} +import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} -import scala.collection.{IndexedSeq, Seq, mutable} +import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } + +import scala.collection.{ mutable, IndexedSeq, Seq } import scala.concurrent.Future import scala.concurrent.duration._ -import scala.util.{Failure, Success, Try} +import scala.util.{ Failure, Success, Try } class NodeViewHolder( settings: EncryAppSettings, @@ -63,7 +63,7 @@ class NodeViewHolder( context.system.scheduler.schedule(1.seconds, 10.seconds) { logger.info( - s"\n History best header id is: ${nodeView.history.getBestHeaderId.map(Algos.encode)}.\n " + + s"\n\n History best header id is: ${nodeView.history.getBestHeaderId.map(Algos.encode)}.\n " + s"History best header height is: ${nodeView.history.getBestHeaderHeight}.\n " + s"History best block id is: ${nodeView.history.getBestBlockId.map(Algos.encode)}.\n " + s"History best block height is: ${nodeView.history.getBestBlockHeight}.\n " + @@ -111,7 +111,7 @@ class NodeViewHolder( logger.info("Receive GetDataFromCurrentView on nvh") f(CurrentView(nodeView.history, nodeView.state, nodeView.wallet)) match { case resultFuture: Future[_] => resultFuture.pipeTo(sender) - case result => sender ! result + case result => sender ! result } case FastSyncFinished(state: UtxoState, wallet: EncryWallet) => @@ -220,7 +220,6 @@ class NodeViewHolder( val additionalBlocks: List[Block] = (state.safePointHeight + 1 to branchPointHeight).foldLeft(List.empty[Block]) { case (blocks: List[Block], height: Int) => - //todo get best header id instead of best header val headerAtHeight: Header = history.getBestHeaderAtHeight(height).get val blockAtHeight: Block = history.getBlockByHeader(headerAtHeight).get blocks :+ blockAtHeight diff --git a/src/main/scala/encry/nvg/fast/sync/SnapshotDownloader.scala b/src/main/scala/encry/nvg/fast/sync/SnapshotDownloader.scala index bcc4ed26c5..ceba485782 100644 --- a/src/main/scala/encry/nvg/fast/sync/SnapshotDownloader.scala +++ b/src/main/scala/encry/nvg/fast/sync/SnapshotDownloader.scala @@ -4,7 +4,7 @@ import akka.actor.{Actor, Cancellable} import com.typesafe.scalalogging.StrictLogging import encry.network.Messages.MessageToNetwork.BroadcastManifestRequest import encry.nvg.fast.sync.SnapshotProcessor.{BroadcastManifestRequestMessage, RequiredManifestHeightAndId} -import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.EncryAppSettings import encry.view.fast.sync.{SnapshotDownloadController, SnapshotHolder} import encry.view.history.HistoryReader diff --git a/src/main/scala/encry/nvg/fast/sync/SnapshotProcessor.scala b/src/main/scala/encry/nvg/fast/sync/SnapshotProcessor.scala index 945c6cb544..5c6fd18db7 100644 --- a/src/main/scala/encry/nvg/fast/sync/SnapshotProcessor.scala +++ b/src/main/scala/encry/nvg/fast/sync/SnapshotProcessor.scala @@ -11,7 +11,7 @@ import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.ChangedHistory import encry.nvg.fast.sync.SnapshotProcessor.{DropProcessedCount, HeaderChainIsSynced, RemoveRedundantManifestIds, TreeChunks} import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.{ChunkId, ManifestId} -import encry.nvg.nvhg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.EncryAppSettings import encry.storage.VersionalStorage.{StorageKey, StorageValue} import encry.view.fast.sync.{RequestsPerPeriodProcessor, SnapshotDownloadController, SnapshotHolder} diff --git a/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala b/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala deleted file mode 100644 index 4e75ae2909..0000000000 --- a/src/main/scala/encry/nvg/nvhg/HistoryApplier.scala +++ /dev/null @@ -1,9 +0,0 @@ -package encry.nvg.nvhg - -import akka.actor.Actor - -class HistoryApplier extends Actor { - override def receive: Receive = { - case _ => - } -} diff --git a/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala b/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala deleted file mode 100644 index 80b43b6539..0000000000 --- a/src/main/scala/encry/nvg/nvhg/NodeViewHolderController.scala +++ /dev/null @@ -1,182 +0,0 @@ -package encry.nvg.nvhg - -import java.io.File - -import akka.actor.{ Actor, ActorRef } -import cats.syntax.option._ -import com.typesafe.scalalogging.StrictLogging -import encry.nvg.ModifiersCache -import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.nvhg.NodeViewHolder.NodeView -import encry.nvg.nvhg.NodeViewHolderController.{ ApplicableModifier, GetNewApplicable, HistoryIsReady, StateIsReady } -import encry.settings.EncryAppSettings -import encry.utils.CoreTaggedTypes.VersionTag -import encry.utils.NetworkTimeProvider -import encry.view.history.History -import encry.view.state.UtxoState -import encry.view.wallet.EncryWallet -import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.Block -import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.ADDigest - -import scala.collection.mutable - -class NodeViewHolderController( - settings: EncryAppSettings, - ntp: NetworkTimeProvider, - influxRef: Option[ActorRef] -) extends Actor - with StrictLogging { - - var nodeView: NodeView = restoreState().getOrElse(genesisState) - - val historyApplier: ActorRef = Actor.noSender - - val stateApplier: ActorRef = Actor.noSender - - override def receive: Receive = mainReceive.orElse(processModifierFromNetwork) - - def mainReceive: Receive = { - case GetNewApplicable => - val mod: List[PersistentModifier] = ModifiersCache.popCandidate(nodeView.history) - mod.headOption.foreach { mod: PersistentModifier => - historyApplier ! ApplicableModifier(mod) - stateApplier ! ApplicableModifier(mod) - logger.info(s"Applicable modifier ${mod.encodedId} of type ${mod.modifierTypeId} sent to history and state.") - context.become( - awaitApplication(historyIsReady = false, stateIsReady = false).orElse(processModifierFromNetwork) - ) - } - } - - def awaitApplication(historyIsReady: Boolean, stateIsReady: Boolean): Receive = { - case HistoryIsReady => - if (stateIsReady) { - self ! GetNewApplicable - context.become(mainReceive.orElse(processModifierFromNetwork)) - } else context.become(awaitApplication(historyIsReady = true, stateIsReady).orElse(processModifierFromNetwork)) - case StateIsReady => - if (historyIsReady) { - self ! GetNewApplicable - context.become(mainReceive.orElse(processModifierFromNetwork)) - } else context.become(awaitApplication(historyIsReady, stateIsReady = true).orElse(processModifierFromNetwork)) - } - - def processModifierFromNetwork: Receive = { - case ValidatedModifier(modifier: PersistentModifier) => - val wrappedKey: mutable.WrappedArray.ofByte = NodeViewHolder.toKey(modifier.id) - val isInHistory: Boolean = nodeView.history.isModifierDefined(modifier.id) - val isInCache: Boolean = ModifiersCache.contains(wrappedKey) - if (isInHistory || isInCache) - logger.info( - s"Modifier ${modifier.encodedId} can't be placed into the cache cause: " + - s"contains in cache: $isInCache, contains in history: $isInHistory." - ) - else { - ModifiersCache.put(wrappedKey, modifier, nodeView.history) - self ! GetNewApplicable - } - } - - def genesisState: NodeView = { - val stateDir: File = UtxoState.getStateDir(settings) - stateDir.mkdir() - val rootsDir: File = UtxoState.getRootsDir(settings) - rootsDir.mkdir() - assert(stateDir.listFiles().isEmpty, s"Genesis directory $stateDir should always be empty.") - val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) - val history: History = History.readOrGenerate(settings, ntp) - val wallet: EncryWallet = - EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) - NodeView(history, state, wallet) - } - - def restoreState(influxRef: Option[ActorRef] = none): Option[NodeView] = - if (History.getHistoryIndexDir(settings).listFiles.nonEmpty) - try { - val stateDir: File = UtxoState.getStateDir(settings) - stateDir.mkdirs() - val rootsDir: File = UtxoState.getRootsDir(settings) - rootsDir.mkdir() - val history: History = History.readOrGenerate(settings, ntp) - val wallet: EncryWallet = - EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) - val state: UtxoState = restoreConsistentState( - UtxoState.create(stateDir, rootsDir, settings, influxRef), - history, - influxRef - ) - history.updateIdsForSyncInfo() - logger.info(s"History best block height: ${history.getBestBlockHeight}") - logger.info(s"History best header height: ${history.getBestHeaderHeight}") - NodeView(history, state, wallet).some - } catch { - case ex: Throwable => - logger.info(s"${ex.getMessage} during state restore. Recover from Modifiers holder!") - new File(settings.directory).listFiles.foreach(dir => FileUtils.cleanDirectory(dir)) - genesisState.some - } else { - none - } - - def getRecreatedState( - version: Option[VersionTag] = none, - digest: Option[ADDigest] = none, - influxRef: Option[ActorRef] - ): UtxoState = { - val dir: File = UtxoState.getStateDir(settings) - dir.mkdirs() - dir.listFiles.foreach(_.delete()) - val stateDir: File = UtxoState.getStateDir(settings) - stateDir.mkdirs() - val rootsDir: File = UtxoState.getRootsDir(settings) - rootsDir.mkdir() - UtxoState.create(stateDir, rootsDir, settings, influxRef) - } - - def restoreConsistentState( - stateIn: UtxoState, - history: History, - influxRefActor: Option[ActorRef] - ): UtxoState = - (stateIn.version, history.getBestBlock, stateIn, stateIn.safePointHeight) match { - case (stateId, None, _, _) if stateId sameElements Array.emptyByteArray => - logger.info(s"State and history are both empty on startup") - stateIn - case (_, None, _, _) => - logger.info( - s"State and history are inconsistent." + - s" History is empty on startup, rollback state to genesis." - ) - getRecreatedState(influxRef = influxRefActor) - case (_, Some(historyBestBlock), state: UtxoState, safePointHeight) => - val headerAtSafePointHeight = history.getBestHeaderAtHeight(safePointHeight) - val (rollbackId, newChain) = history.getChainToHeader(headerAtSafePointHeight, historyBestBlock.header) - logger.info( - s"State and history are inconsistent." + - s" Going to rollback to ${rollbackId.map(Algos.encode)} and " + - s"apply ${newChain.length} modifiers. State safe point: ${safePointHeight}. ${newChain.headers.head.height}. ${newChain.headers.last.height}" - ) - val additionalBlocks = - (state.safePointHeight + 1 to historyBestBlock.header.height).foldLeft(List.empty[Block]) { - case (blocks, height) => - val headerAtHeight = history.getBestHeaderAtHeight(height).get - val blockAtHeight = history.getBlockByHeader(headerAtHeight).get - blocks :+ blockAtHeight - } - logger.info(s"Qty of additional blocks: ${additionalBlocks.length}") - rollbackId - .map(_ => state.restore(additionalBlocks).get) - .getOrElse(getRecreatedState(influxRef = influxRefActor)) - } -} - -object NodeViewHolderController { - - final case class ApplicableModifier(modifier: PersistentModifier) - case object HistoryIsReady - case object StateIsReady - case object GetNewApplicable -} diff --git a/src/main/scala/encry/nvg/nvhg/StateApplier.scala b/src/main/scala/encry/nvg/nvhg/StateApplier.scala deleted file mode 100644 index a94bfee293..0000000000 --- a/src/main/scala/encry/nvg/nvhg/StateApplier.scala +++ /dev/null @@ -1,9 +0,0 @@ -package encry.nvg.nvhg - -import akka.actor.Actor - -class StateApplier extends Actor { - override def receive: Receive = { - case _ => - } -} diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index f8e3f4915e..a9c751ff7b 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -9,7 +9,7 @@ import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.network.Messages.MessageToNetwork.{RequestFromLocal, ResponseFromLocal} import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.nvg.nvhg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} +import encry.nvg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.ReceivableMessages.CompareViews From 140c4bf44a387e3dbafbddc2aca2e99bc0fe72f7 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 10 Mar 2020 13:53:34 +0300 Subject: [PATCH 072/177] added messages to mempool from nvh --- src/main/scala/encry/Starter.scala | 59 ++++++------------- .../scala/encry/nvg/IntermediaryNVH.scala | 12 ++-- 2 files changed, 25 insertions(+), 46 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 587b5bfff3..0128ea4043 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -1,7 +1,6 @@ package encry import java.net.InetSocketAddress - import akka.actor.{ Actor, ActorRef } import akka.http.scaladsl.Http import cats.Functor @@ -24,12 +23,8 @@ import encry.nvg.IntermediaryNVH import encry.settings._ import encry.stats.StatsSender import encry.utils.{ Mnemonic, NetworkTimeProvider } -import encry.view.mempool.MemoryPool -import encry.utils.{Mnemonic, NetworkTimeProvider} -import encry.view.NodeViewHolder -import encry.view.mempool.{IntermediaryMempool, MemoryPool} +import encry.view.mempool.IntermediaryMempool import encry.view.wallet.AccountManager - import scala.concurrent.Future import scala.io.StdIn import scala.util.{ Failure, Success, Try } @@ -411,27 +406,9 @@ class Starter(settings: EncryAppSettings, } lazy val dataHolderForApi = context.system.actorOf(DataHolderForApi.props(newSettings, timeProvider), "dataHolder") -// lazy val memoryPool: ActorRef = context.system.actorOf( -// MemoryPool -// .props(newSettings, timeProvider, miner, influxRef) -// .withDispatcher("mempool-dispatcher") -// ) -// val nodeViewHolder: ActorRef = context.system.actorOf( -// NodeViewHolder -// .props(memoryPool, influxRef, dataHolderForApi, newSettings) -// .withDispatcher("nvh-dispatcher"), -// "nodeViewHolder" -// ) if (nodePass.nonEmpty) dataHolderForApi ! PassForStorage(nodePass) -// context.system.actorOf( -// NodeViewSynchronizer -// .props(influxRef, nodeViewHolder, newSettings, memoryPool, dataHolderForApi) -// .withDispatcher("nvsh-dispatcher"), -// "nodeViewSynchronizer" -// ) - val networkRouter = context.system.actorOf( NetworkRouter .props(networkSettings, settings.blackList) @@ -439,14 +416,12 @@ class Starter(settings: EncryAppSettings, "networkRouter" ) - val nvhRouter = context.system.actorOf( - IntermediaryNVH.props(newSettings, networkRouter, timeProvider, influxRef) - ) - - val memoryPool = context.system.actorOf( + val memoryPool: ActorRef = context.system.actorOf( IntermediaryMempool.props(newSettings, timeProvider, influxRef, networkRouter) ) - + val nvhRouter: ActorRef = context.system.actorOf( + IntermediaryNVH.props(newSettings, networkRouter, timeProvider, influxRef, memoryPool) + ) val miner: ActorRef = context.system.actorOf(Miner.props(dataHolderForApi, memoryPool, nvhRouter, influxRef, newSettings), "miner") if (newSettings.node.mining) miner ! StartMining @@ -461,15 +436,17 @@ class Starter(settings: EncryAppSettings, } object Starter { - final case class InitNodeResult(mnemonic: String, - walletPassword: String, - offlineGeneration: Boolean, - fastSync: Boolean, - snapshotCreation: Boolean, - peers: List[InetSocketAddress], - connectWithOnlyKnownPeers: Boolean, - nodePass: String = "", - nodeName: String, - declaredAddr: Option[InetSocketAddress], - bindAddr: InetSocketAddress) + final case class InitNodeResult( + mnemonic: String, + walletPassword: String, + offlineGeneration: Boolean, + fastSync: Boolean, + snapshotCreation: Boolean, + peers: List[InetSocketAddress], + connectWithOnlyKnownPeers: Boolean, + nodePass: String = "", + nodeName: String, + declaredAddr: Option[InetSocketAddress], + bindAddr: InetSocketAddress + ) } diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 3539027feb..07b78aa7d3 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -50,7 +50,8 @@ class IntermediaryNVH( settings: EncryAppSettings, intermediaryNetwork: ActorRef, timeProvider: NetworkTimeProvider, - influxRef: Option[ActorRef] + influxRef: Option[ActorRef], + mempoolRef: ActorRef ) extends Actor with StrictLogging { @@ -121,11 +122,11 @@ class IntermediaryNVH( case msg @ TreeChunks(_, _) => //+ to fast sync case msg @ FastSyncDone => case msg @ HeaderChainIsSynced => - case msg @ FullBlockChainIsSynced => //+ to miner + case msg @ FullBlockChainIsSynced => mempoolRef ! msg + case msg @ RolledBackTransactions(_) => mempoolRef ! msg case msg @ DisableMining => //+ to miner case msg @ StartMining => //+ to miner case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder - case msg @ RolledBackTransactions(_) => //+ to memory pool case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) case msg @ GetDataFromCurrentView(_, _) => nodeViewHolder ! msg case msg @ RollbackSucceed(_) => @@ -142,6 +143,7 @@ object IntermediaryNVH { settings: EncryAppSettings, intermediaryNetwork: ActorRef, timeProvider: NetworkTimeProvider, - influxRef: Option[ActorRef] - ): Props = Props(new IntermediaryNVH(settings, intermediaryNetwork, timeProvider, influxRef)) + influxRef: Option[ActorRef], + mempoolRef: ActorRef + ): Props = Props(new IntermediaryNVH(settings, intermediaryNetwork, timeProvider, influxRef, mempoolRef)) } From 801e27b05a91e1e799248b5518334ba0e299c463 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 10 Mar 2020 14:20:17 +0300 Subject: [PATCH 073/177] added forwarding instead of ref in message --- src/main/scala/encry/local/miner/Miner.scala | 2 +- src/main/scala/encry/nvg/IntermediaryNVH.scala | 2 +- src/main/scala/encry/nvg/NodeViewHolder.scala | 4 ++-- src/main/scala/encry/view/mempool/MemoryPool.scala | 1 - 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index 8512ddfd23..b2b45cd2f4 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -252,7 +252,7 @@ class Miner(dataHolder: ActorRef, } else CandidateEnvelope.empty candidate } - nvh ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope] (lambda, self) + nvh ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope](lambda) } } diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 07b78aa7d3..d789ef0ed3 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -128,7 +128,7 @@ class IntermediaryNVH( case msg @ StartMining => //+ to miner case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) - case msg @ GetDataFromCurrentView(_, _) => nodeViewHolder ! msg + case msg @ GetDataFromCurrentView(_) => nodeViewHolder.forward(msg) case msg @ RollbackSucceed(_) => case msg @ RollbackFailed(_) => case msg @ SemanticallySuccessfulModifier(_) => diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 237cf9959c..f628a3f571 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -107,7 +107,7 @@ class NodeViewHolder( s"is ${(System.currentTimeMillis() - startTime) / 1000}s." ) - case GetDataFromCurrentView(f, sender) => + case GetDataFromCurrentView(f) => logger.info("Receive GetDataFromCurrentView on nvh") f(CurrentView(nodeView.history, nodeView.state, nodeView.wallet)) match { case resultFuture: Future[_] => resultFuture.pipeTo(sender) @@ -518,7 +518,7 @@ object NodeViewHolder { case class SemanticallySuccessfulModifier(modifier: PersistentNodeViewModifier) extends ModificationOutcome - case class GetDataFromCurrentView[HIS, MS, VL, A](f: CurrentView[HIS, MS, VL] => A, sender: ActorRef) + case class GetDataFromCurrentView[HIS, MS, VL, A](f: CurrentView[HIS, MS, VL] => A) final case class DownloadRequest( modifierTypeId: ModifierTypeId, diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index a9c751ff7b..0da47920ae 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -12,7 +12,6 @@ import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.nvg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.NodeViewHolder.ReceivableMessages.CompareViews import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.MemoryPoolStateType.NotProcessingNewTransactions import encry.view.mempool.MemoryPool._ From a457f6684190eb72a8807f16af95a18e3b5f1474 Mon Sep 17 00:00:00 2001 From: Lior Date: Tue, 10 Mar 2020 14:24:22 +0300 Subject: [PATCH 074/177] code cleanup --- .../encry/view/mempool/IntermediaryMempool.scala | 16 +++++++--------- .../encry/view/mempool/MemoryPoolProcessor.scala | 5 ++--- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala index acf38c9982..1181ffd9f5 100644 --- a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala +++ b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala @@ -2,6 +2,7 @@ package encry.view.mempool import akka.actor.{Actor, ActorRef, Props} import com.typesafe.scalalogging.StrictLogging +import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForTxHandling} @@ -9,7 +10,6 @@ import encry.network.PeersKeeper.BanPeer import encry.settings.EncryAppSettings import encry.stats.StatsSender.ValidatedModifierFromNetwork import encry.utils.NetworkTimeProvider -import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.{RolledBackTransactions, TransactionProcessing, TransactionsForMiner} import encry.view.mempool.TransactionsValidator.{InvalidTransaction, ModifiersForValidating} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction @@ -36,14 +36,14 @@ class IntermediaryMempool(settings: EncryAppSettings, override def receive(): Receive = { case msg: InvalidTransaction => networkRouter ! msg case msg: BanPeer => networkRouter ! msg - case msg: ValidatedModifierFromNetwork => // to influx - case msg: RolledBackTransactions => memoryPool ! msg // to mempool - case msg: ModifiersForValidating => memoryPool ! msg // to mempool - case msg: DataFromPeer => mempoolProcessor ! msg // to mempool processor + case msg: ValidatedModifierFromNetwork => influxReference.foreach(_ ! msg) + case msg: RolledBackTransactions => memoryPool ! msg + case msg: ModifiersForValidating => memoryPool ! msg + case msg: DataFromPeer => mempoolProcessor ! msg case msg: RequestFromLocal => networkRouter ! msg case msg: ModifierFromNetwork => txValidator ! msg - case msg: TransactionProcessing => mempoolProcessor ! msg // to mempool processor - case msg: IsChainSynced => mempoolProcessor ! msg + case msg: TransactionProcessing => mempoolProcessor ! msg + case msg @ FullBlockChainIsSynced => mempoolProcessor ! msg } } @@ -56,6 +56,4 @@ object IntermediaryMempool { Props(new IntermediaryMempool(settings, networkTimeProvider, influxReference, networkRouter)) final case class TransactionsForValidating(tx: Transaction) - - final case class IsChainSynced(info: Boolean) } diff --git a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala index c0055dd99b..fe9ea7dc5e 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala +++ b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala @@ -1,11 +1,10 @@ package encry.view.mempool -import java.net.InetSocketAddress - import akka.actor.{Actor, Props} import com.google.common.base.Charsets import com.google.common.hash.{BloomFilter, Funnels} import com.typesafe.scalalogging.StrictLogging +import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.{RequestFromLocal, ResponseFromLocal} import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.settings.EncryAppSettings @@ -42,7 +41,7 @@ class MemoryPoolProcessor(settings: EncryAppSettings, ntp: NetworkTimeProvider) case TransactionProcessing(info) => canProcessTransactions = info - case IsChainSynced(info) => chainSynced = info + case FullBlockChainIsSynced(info) => chainSynced = info case CleanupBloomFilter => bloomFilterForTransactionsIds = initBloomFilter From 5033ed18d943ecc11aa4ce93d010bc425bddc53c Mon Sep 17 00:00:00 2001 From: Lior Date: Tue, 10 Mar 2020 14:36:22 +0300 Subject: [PATCH 075/177] code cleanup --- src/main/scala/encry/view/mempool/IntermediaryMempool.scala | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala index 1181ffd9f5..e49ef2be36 100644 --- a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala +++ b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala @@ -8,7 +8,6 @@ import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForTxHandling} import encry.network.PeersKeeper.BanPeer import encry.settings.EncryAppSettings -import encry.stats.StatsSender.ValidatedModifierFromNetwork import encry.utils.NetworkTimeProvider import encry.view.mempool.MemoryPool.{RolledBackTransactions, TransactionProcessing, TransactionsForMiner} import encry.view.mempool.TransactionsValidator.{InvalidTransaction, ModifiersForValidating} @@ -36,7 +35,6 @@ class IntermediaryMempool(settings: EncryAppSettings, override def receive(): Receive = { case msg: InvalidTransaction => networkRouter ! msg case msg: BanPeer => networkRouter ! msg - case msg: ValidatedModifierFromNetwork => influxReference.foreach(_ ! msg) case msg: RolledBackTransactions => memoryPool ! msg case msg: ModifiersForValidating => memoryPool ! msg case msg: DataFromPeer => mempoolProcessor ! msg @@ -54,6 +52,4 @@ object IntermediaryMempool { influxReference: Option[ActorRef], networkRouter: ActorRef): Props = Props(new IntermediaryMempool(settings, networkTimeProvider, influxReference, networkRouter)) - - final case class TransactionsForValidating(tx: Transaction) } From 17b85ec5038b6e060b1acea9d5554c779573e3e7 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 10 Mar 2020 15:43:13 +0300 Subject: [PATCH 076/177] added mempool reader --- .../view/mempool/IntermediaryMempool.scala | 55 ++++---- .../scala/encry/view/mempool/MemoryPool.scala | 122 ++++++++++-------- .../view/mempool/MemoryPoolProcessor.scala | 40 +++--- .../encry/view/mempool/MemoryPoolReader.scala | 13 ++ .../view/mempool/MemoryPoolStorage.scala | 18 +-- .../view/mempool/TransactionsValidator.scala | 30 ++--- 6 files changed, 156 insertions(+), 122 deletions(-) create mode 100644 src/main/scala/encry/view/mempool/MemoryPoolReader.scala diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala index e49ef2be36..3e422ae5b8 100644 --- a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala +++ b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala @@ -1,17 +1,16 @@ package encry.view.mempool -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import com.typesafe.scalalogging.StrictLogging import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForTxHandling} +import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForTxHandling } import encry.network.PeersKeeper.BanPeer import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.mempool.MemoryPool.{RolledBackTransactions, TransactionProcessing, TransactionsForMiner} -import encry.view.mempool.TransactionsValidator.{InvalidTransaction, ModifiersForValidating} -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import encry.view.mempool.MemoryPool.{ RolledBackTransactions, TransactionProcessing } +import encry.view.mempool.TransactionsValidator.{ InvalidTransaction, ModifiersForValidating } class IntermediaryMempool(settings: EncryAppSettings, networkTimeProvider: NetworkTimeProvider, @@ -20,36 +19,40 @@ class IntermediaryMempool(settings: EncryAppSettings, extends Actor with StrictLogging { - val memoryPool: ActorRef = - context.actorOf(MemoryPool.props(settings, networkTimeProvider, self, influxReference), name = "mempool") - val txValidator: ActorRef = - context.actorOf(TransactionsValidator.props(settings, memoryPool, networkTimeProvider), - name = "Transaction-validator") val mempoolProcessor: ActorRef = context.actorOf(MemoryPoolProcessor.props(settings, networkTimeProvider), name = "mempool-processor") - override def preStart(): Unit = { - networkRouter ! RegisterForTxHandling - } + val memoryPool: ActorRef = + context.actorOf(MemoryPool.props(settings, networkTimeProvider, influxReference), name = "mempool") + + val txValidator: ActorRef = + context.actorOf( + TransactionsValidator.props(settings, memoryPool, networkTimeProvider), + name = "Transaction-validator" + ) + + override def preStart(): Unit = networkRouter ! RegisterForTxHandling override def receive(): Receive = { - case msg: InvalidTransaction => networkRouter ! msg - case msg: BanPeer => networkRouter ! msg - case msg: RolledBackTransactions => memoryPool ! msg - case msg: ModifiersForValidating => memoryPool ! msg - case msg: DataFromPeer => mempoolProcessor ! msg - case msg: RequestFromLocal => networkRouter ! msg - case msg: ModifierFromNetwork => txValidator ! msg - case msg: TransactionProcessing => mempoolProcessor ! msg - case msg @ FullBlockChainIsSynced => mempoolProcessor ! msg + case msg: InvalidTransaction => networkRouter ! msg + case msg: BanPeer => networkRouter ! msg + case msg: RolledBackTransactions => memoryPool ! msg + case msg: ModifiersForValidating => memoryPool ! msg + case msg: DataFromPeer => mempoolProcessor ! msg + case msg: RequestFromLocal => networkRouter ! msg + case msg: ModifierFromNetwork => txValidator ! msg + case msg: TransactionProcessing => mempoolProcessor ! msg + case msg @ FullBlockChainIsSynced => mempoolProcessor ! msg } } object IntermediaryMempool { - def props(settings: EncryAppSettings, - networkTimeProvider: NetworkTimeProvider, - influxReference: Option[ActorRef], - networkRouter: ActorRef): Props = + def props( + settings: EncryAppSettings, + networkTimeProvider: NetworkTimeProvider, + influxReference: Option[ActorRef], + networkRouter: ActorRef + ): Props = Props(new IntermediaryMempool(settings, networkTimeProvider, influxReference, networkRouter)) } diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index 0da47920ae..47e86b213f 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -1,32 +1,34 @@ package encry.view.mempool -import akka.actor.{Actor, ActorRef, ActorSystem, Props} -import akka.dispatch.{PriorityGenerator, UnboundedStablePriorityMailbox} +import akka.actor.{ Actor, ActorRef, ActorSystem, Props } +import akka.dispatch.{ PriorityGenerator, UnboundedStablePriorityMailbox } import cats.syntax.either._ import com.google.common.base.Charsets -import com.google.common.hash.{BloomFilter, Funnels} +import com.google.common.hash.{ BloomFilter, Funnels } import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.network.Messages.MessageToNetwork.{RequestFromLocal, ResponseFromLocal} +import encry.network.Messages.MessageToNetwork.{ RequestFromLocal, ResponseFromLocal } import encry.network.NetworkController.ReceivableMessages.DataFromPeer -import encry.nvg.NodeViewHolder.{SemanticallySuccessfulModifier, SuccessfulTransaction} +import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, SuccessfulTransaction } import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.mempool.IntermediaryMempool.IsChainSynced import encry.view.mempool.MemoryPool.MemoryPoolStateType.NotProcessingNewTransactions import encry.view.mempool.MemoryPool._ import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestModifiersNetworkMessage } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.IndexedSeq -class MemoryPool(settings: EncryAppSettings, - networkTimeProvider: NetworkTimeProvider, - intermediaryMempool: ActorRef, - influxReference: Option[ActorRef]) extends Actor with StrictLogging { +class MemoryPool( + settings: EncryAppSettings, + networkTimeProvider: NetworkTimeProvider, + influxReference: Option[ActorRef], + mempoolProcessor: ActorRef +) extends Actor + with StrictLogging { import context.dispatcher @@ -42,10 +44,16 @@ class MemoryPool(settings: EncryAppSettings, logger.debug(s"Starting MemoryPool. Initializing all schedulers") context.system.scheduler.schedule( settings.mempool.cleanupInterval, - settings.mempool.cleanupInterval, self, RemoveExpiredFromPool) + settings.mempool.cleanupInterval, + self, + RemoveExpiredFromPool + ) context.system.scheduler.schedule( settings.mempool.txSendingInterval, - settings.mempool.txSendingInterval, self, SendTransactionsToMiner) + settings.mempool.txSendingInterval, + self, + SendTransactionsToMiner + ) context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) } @@ -103,56 +111,61 @@ class MemoryPool(settings: EncryAppSettings, val (newMemoryPool: MemoryPoolStorage, validatedTransaction: Option[Transaction]) = memoryPool.validateTransaction(transaction) memoryPool = newMemoryPool + mempoolProcessor ! UpdateMempoolReader(MemoryPoolReader.apply(memoryPool)) validatedTransaction.foreach(tx => context.system.eventStream.publish(SuccessfulTransaction(tx))) logger.debug(s"MemoryPool got new transactions from remote. New pool size is ${memoryPool.size}.") if (currentNumberOfProcessedTransactions > settings.mempool.transactionsLimit) { - logger.debug(s"MemoryPool has its limit of processed transactions. " + - s"Transit to 'disableTransactionsProcessor' state." + - s"Current number of processed transactions is $currentNumberOfProcessedTransactions.") + logger.debug( + s"MemoryPool has its limit of processed transactions. " + + s"Transit to 'disableTransactionsProcessor' state." + + s"Current number of processed transactions is $currentNumberOfProcessedTransactions." + ) canProcessTransactions = false context.parent ! TransactionProcessing(canProcessTransactions) context.become(disableTransactionsProcessor) } else { val currentTransactionsNumber: Int = currentNumberOfProcessedTransactions + 1 - logger.debug(s"Current number of processed transactions is OK. Continue to process them..." + - s" Current number is $currentTransactionsNumber.") + logger.debug( + s"Current number of processed transactions is OK. Continue to process them..." + + s" Current number is $currentTransactionsNumber." + ) context.become(continueProcessing(currentTransactionsNumber)) } -// case CompareViews(peer, _, transactions) => -// val notYetRequestedTransactions: IndexedSeq[ModifierId] = notRequestedYet(transactions.toIndexedSeq) -// if (notYetRequestedTransactions.nonEmpty) { -// sender ! RequestFromLocal(peer, Transaction.modifierTypeId, notYetRequestedTransactions) -// logger.debug(s"MemoryPool got inv message with ${transactions.size} ids." + -// s" Not yet requested ids size is ${notYetRequestedTransactions.size}.") -// } else logger.debug(s"MemoryPool got inv message with ${transactions.size} ids." + -// s" There are no not yet requested ids.") - case RolledBackTransactions(transactions) => val (newMemoryPool: MemoryPoolStorage, validatedTransactions: Seq[Transaction]) = memoryPool.validateTransactions(transactions) memoryPool = newMemoryPool - logger.debug(s"MemoryPool got rolled back transactions. New pool size is ${memoryPool.size}." + - s"Number of rolled back transactions is ${validatedTransactions.size}.") + mempoolProcessor ! UpdateMempoolReader(MemoryPoolReader.apply(memoryPool)) + logger.debug( + s"MemoryPool got rolled back transactions. New pool size is ${memoryPool.size}." + + s"Number of rolled back transactions is ${validatedTransactions.size}." + ) if (currentNumberOfProcessedTransactions > settings.mempool.transactionsLimit) { - logger.debug(s"MemoryPool has its limit of processed transactions. " + - s"Transit to 'disableTransactionsProcessor' state." + - s"Current number of processed transactions is $currentNumberOfProcessedTransactions.") + logger.debug( + s"MemoryPool has its limit of processed transactions. " + + s"Transit to 'disableTransactionsProcessor' state." + + s"Current number of processed transactions is $currentNumberOfProcessedTransactions." + ) canProcessTransactions = false context.parent ! TransactionProcessing(canProcessTransactions) context.become(disableTransactionsProcessor) } else { val currentTransactionsNumber: Int = currentNumberOfProcessedTransactions + validatedTransactions.size - logger.debug(s"Current number of processed transactions is OK. Continue to process them..." + - s" Current number is $currentTransactionsNumber.") + logger.debug( + s"Current number of processed transactions is OK. Continue to process them..." + + s" Current number is $currentTransactionsNumber." + ) context.become(continueProcessing(currentTransactionsNumber)) } } def auxiliaryReceive(state: MemoryPoolStateType): Receive = { case SemanticallySuccessfulModifier(modifier) if modifier.modifierTypeId == Block.modifierTypeId => - logger.debug(s"MemoryPool got SemanticallySuccessfulModifier with new block while $state." + - s"Transit to a transactionsProcessor state.") + logger.debug( + s"MemoryPool got SemanticallySuccessfulModifier with new block while $state." + + s"Transit to a transactionsProcessor state." + ) if (state == NotProcessingNewTransactions) Either.catchNonFatal(context.system.actorSelection("/user/nodeViewSynchronizer") ! StartTransactionsValidation) canProcessTransactions = true @@ -160,16 +173,21 @@ class MemoryPool(settings: EncryAppSettings, context.become(continueProcessing(currentNumberOfProcessedTransactions = 0)) case SemanticallySuccessfulModifier(_) => - logger.debug(s"MemoryPool got SemanticallySuccessfulModifier with non block modifier" + - s"while $state. Do nothing in this case.") + logger.debug( + s"MemoryPool got SemanticallySuccessfulModifier with non block modifier" + + s"while $state. Do nothing in this case." + ) case SendTransactionsToMiner => val (newMemoryPool: MemoryPoolStorage, transactionsForMiner: Seq[Transaction]) = memoryPool.getTransactionsForMiner memoryPool = newMemoryPool - intermediaryMempool ! TransactionsForMiner(transactionsForMiner) - logger.debug(s"MemoryPool got SendTransactionsToMiner. Size of transactions for miner ${transactionsForMiner.size}." + - s" New pool size is ${memoryPool.size}. Ids ${transactionsForMiner.map(_.encodedId)}") + mempoolProcessor ! UpdateMempoolReader(MemoryPoolReader.apply(memoryPool)) + context.parent ! TransactionsForMiner(transactionsForMiner) + logger.debug( + s"MemoryPool got SendTransactionsToMiner. Size of transactions for miner ${transactionsForMiner.size}." + + s" New pool size is ${memoryPool.size}. Ids ${transactionsForMiner.map(_.encodedId)}" + ) case RemoveExpiredFromPool => memoryPool = memoryPool.filter(memoryPool.isExpired) @@ -220,6 +238,8 @@ object MemoryPool { sealed trait MemoryPoolStateType + final case class UpdateMempoolReader(reader: MemoryPoolReader) + object MemoryPoolStateType { case object ProcessingNewTransaction extends MemoryPoolStateType @@ -228,18 +248,18 @@ object MemoryPool { } - def props(settings: EncryAppSettings, - ntp: NetworkTimeProvider, - intermediaryMempool: ActorRef, - influx: Option[ActorRef]): Props = - Props(new MemoryPool(settings, ntp, intermediaryMempool, influx)) + def props( + settings: EncryAppSettings, + ntp: NetworkTimeProvider, + influx: Option[ActorRef], + mempoolProcessor: ActorRef + ): Props = Props(new MemoryPool(settings, ntp, influx, mempoolProcessor)) class MemoryPoolPriorityQueue(settings: ActorSystem.Settings, config: Config) - extends UnboundedStablePriorityMailbox( - PriorityGenerator { + extends UnboundedStablePriorityMailbox(PriorityGenerator { case RemoveExpiredFromPool | SendTransactionsToMiner => 0 - case NewTransaction(_) => 1 - case otherwise => 2 + case NewTransaction(_) => 1 + case otherwise => 2 }) -} \ No newline at end of file +} diff --git a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala index fe9ea7dc5e..b86b30fa20 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala +++ b/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala @@ -1,19 +1,18 @@ package encry.view.mempool -import akka.actor.{Actor, Props} +import akka.actor.{ Actor, Props } import com.google.common.base.Charsets -import com.google.common.hash.{BloomFilter, Funnels} +import com.google.common.hash.{ BloomFilter, Funnels } import com.typesafe.scalalogging.StrictLogging import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.Messages.MessageToNetwork.{RequestFromLocal, ResponseFromLocal} +import encry.network.Messages.MessageToNetwork.{ RequestFromLocal, ResponseFromLocal } import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.mempool.IntermediaryMempool.IsChainSynced -import encry.view.mempool.MemoryPool.TransactionProcessing +import encry.view.mempool.MemoryPool.{ TransactionProcessing, UpdateMempoolReader } import encry.view.mempool.MemoryPoolProcessor.CleanupBloomFilter import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestModifiersNetworkMessage } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -29,30 +28,27 @@ class MemoryPoolProcessor(settings: EncryAppSettings, ntp: NetworkTimeProvider) var chainSynced: Boolean = false - var memoryPool: MemoryPoolStorage = MemoryPoolStorage.empty(settings, ntp) + var memoryPoolReader: MemoryPoolReader = MemoryPoolReader.empty override def preStart(): Unit = - context.system.scheduler.schedule(settings.mempool.bloomFilterCleanupInterval, - settings.mempool.bloomFilterCleanupInterval, - self, - CleanupBloomFilter) + context.system.scheduler.schedule( + settings.mempool.bloomFilterCleanupInterval, + settings.mempool.bloomFilterCleanupInterval, + self, + CleanupBloomFilter + ) override def receive: Receive = { - + case UpdateMempoolReader(reader) => memoryPoolReader = reader case TransactionProcessing(info) => canProcessTransactions = info - - case FullBlockChainIsSynced(info) => chainSynced = info - - case CleanupBloomFilter => - bloomFilterForTransactionsIds = initBloomFilter - + case FullBlockChainIsSynced => chainSynced = true + case CleanupBloomFilter => bloomFilterForTransactionsIds = initBloomFilter case DataFromPeer(message, remote) => message match { case RequestModifiersNetworkMessage((_, requestedIds)) => val modifiersIds: Seq[Transaction] = requestedIds .map(Algos.encode) - .collect { case id if memoryPool.contains(id) => memoryPool.get(id) } - .flatten + .flatMap(memoryPoolReader.get) logger.debug( s"MemoryPool got request modifiers message. Number of requested ids is ${requestedIds.size}." + s" Number of sent transactions is ${modifiersIds.size}. Request was from $remote." @@ -84,8 +80,8 @@ class MemoryPoolProcessor(settings: EncryAppSettings, ntp: NetworkTimeProvider) case _ => logger.debug(s"MemoryPoolProcessor got invalid type of DataFromPeer message!") } - } + def initBloomFilter: BloomFilter[String] = BloomFilter.create( Funnels.stringFunnel(Charsets.UTF_8), settings.mempool.bloomFilterCapacity, @@ -102,7 +98,7 @@ class MemoryPoolProcessor(settings: EncryAppSettings, ntp: NetworkTimeProvider) object MemoryPoolProcessor { - def props(settings: EncryAppSettings, ntp: NetworkTimeProvider) = Props(new MemoryPoolProcessor(settings, ntp)) + def props(settings: EncryAppSettings, ntp: NetworkTimeProvider): Props = Props(new MemoryPoolProcessor(settings, ntp)) case object CleanupBloomFilter } diff --git a/src/main/scala/encry/view/mempool/MemoryPoolReader.scala b/src/main/scala/encry/view/mempool/MemoryPoolReader.scala new file mode 100644 index 0000000000..470317f68b --- /dev/null +++ b/src/main/scala/encry/view/mempool/MemoryPoolReader.scala @@ -0,0 +1,13 @@ +package encry.view.mempool + +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction + +trait MemoryPoolReader { + def get(elem: String): Option[Transaction] +} + +object MemoryPoolReader { + def apply(pool: MemoryPoolStorage): MemoryPoolReader = (elem: String) => pool.get(elem) + + def empty: MemoryPoolReader = (_: String) => None +} diff --git a/src/main/scala/encry/view/mempool/MemoryPoolStorage.scala b/src/main/scala/encry/view/mempool/MemoryPoolStorage.scala index 8f0d68335a..1c6c1dd005 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolStorage.scala +++ b/src/main/scala/encry/view/mempool/MemoryPoolStorage.scala @@ -5,9 +5,11 @@ import encry.utils.NetworkTimeProvider import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos -final case class MemoryPoolStorage private(transactions: Map[String, Transaction], - settings: EncryAppSettings, - networkTimeProvider: NetworkTimeProvider) { +final case class MemoryPoolStorage private ( + transactions: Map[String, Transaction], + settings: EncryAppSettings, + networkTimeProvider: NetworkTimeProvider +) { lazy val size: Int = transactions.size @@ -57,13 +59,13 @@ final case class MemoryPoolStorage private(transactions: Map[String, Transaction } def getTransactionsForMiner: (MemoryPoolStorage, Seq[Transaction]) = { - val (transactionsForMiner: Seq[Transaction], _) = transactions - .toIndexedSeq - .sortBy { case (_, tx) => tx.fee } + val (transactionsForMiner: Seq[Transaction], _) = transactions.toIndexedSeq.sortBy { case (_, tx) => tx.fee } .foldLeft(Seq.empty[Transaction], Set.empty[String]) { case ((validated, inputs), (_, transaction)) => val transactionInputsIds: Set[String] = transaction.inputs.map(input => Algos.encode(input.boxId)).toSet - if (transactionInputsIds.size == transaction.inputs.size && transactionInputsIds.forall(id => !inputs.contains(id))) + if (transactionInputsIds.size == transaction.inputs.size && transactionInputsIds.forall( + id => !inputs.contains(id) + )) (validated :+ transaction, inputs ++ transactionInputsIds) else (validated, inputs) } @@ -81,4 +83,4 @@ object MemoryPoolStorage { def empty(settings: EncryAppSettings, networkTimeProvider: NetworkTimeProvider): MemoryPoolStorage = MemoryPoolStorage(Map.empty[String, Transaction], settings, networkTimeProvider) -} \ No newline at end of file +} diff --git a/src/main/scala/encry/view/mempool/TransactionsValidator.scala b/src/main/scala/encry/view/mempool/TransactionsValidator.scala index fe00f24a46..4d36f95cc5 100644 --- a/src/main/scala/encry/view/mempool/TransactionsValidator.scala +++ b/src/main/scala/encry/view/mempool/TransactionsValidator.scala @@ -1,9 +1,9 @@ package encry.view.mempool import TransactionProto.TransactionProtoMessage -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import com.typesafe.scalalogging.StrictLogging -import encry.network.BlackList.BanReason.{CorruptedSerializedBytes, SyntacticallyInvalidTransaction} +import encry.network.BlackList.BanReason.{ CorruptedSerializedBytes, SyntacticallyInvalidTransaction } import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeer @@ -11,22 +11,20 @@ import encry.nvg.ModifiersValidator.InvalidModifierBytes import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.mempool.MemoryPool.NewTransaction -import encry.view.mempool.TransactionsValidator.{InvalidTransaction, ModifiersForValidating} -import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionProtoSerializer} +import encry.view.mempool.TransactionsValidator.InvalidTransaction +import org.encryfoundation.common.modifiers.mempool.transaction.TransactionProtoSerializer import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage -import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } -import scala.util.{Failure, Success, Try} +import scala.util.{ Failure, Success, Try } -class TransactionsValidator(settings: EncryAppSettings, - memPool: ActorRef, - networkTimeProvider: NetworkTimeProvider) +class TransactionsValidator(settings: EncryAppSettings, memPool: ActorRef, networkTimeProvider: NetworkTimeProvider) extends Actor with StrictLogging { override def receive(): Receive = { - case DataFromPeer(ModifiersNetworkMessage(data), remote) if data._1 == Transaction.modifierTypeId => - data._2.foreach { + case DataFromPeer(ModifiersNetworkMessage(data), remote) => + data._2.foreach { case (id, bytes) => Try(TransactionProtoSerializer.fromProto(TransactionProtoMessage.parseFrom(bytes))).flatten match { case Success(tx) if tx.semanticValidity.isSuccess => memPool ! NewTransaction(tx) @@ -39,15 +37,17 @@ class TransactionsValidator(settings: EncryAppSettings, context.parent ! InvalidModifierBytes(id) logger.info(s"Received modifier from $remote can't be parsed cause of: ${ex.getMessage}.") } - } + } } } object TransactionsValidator { - final case class ModifiersForValidating(remote: ConnectedPeer, - typeId: ModifierTypeId, - modifiers: Map[ModifierId, Array[Byte]]) + final case class ModifiersForValidating( + remote: ConnectedPeer, + typeId: ModifierTypeId, + modifiers: Map[ModifierId, Array[Byte]] + ) final case class InvalidTransaction(ids: ModifierId) extends AnyVal From c2e858216d53b077d1df6e1a7d848215b6df1a5a Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 10 Mar 2020 15:45:03 +0300 Subject: [PATCH 077/177] some fixes --- src/main/scala/encry/view/mempool/IntermediaryMempool.scala | 2 +- src/main/scala/encry/view/mempool/MemoryPool.scala | 2 -- src/test/scala/encry/view/mempool/MemoryPoolTests.scala | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala index 3e422ae5b8..1872fa92d2 100644 --- a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala +++ b/src/main/scala/encry/view/mempool/IntermediaryMempool.scala @@ -23,7 +23,7 @@ class IntermediaryMempool(settings: EncryAppSettings, context.actorOf(MemoryPoolProcessor.props(settings, networkTimeProvider), name = "mempool-processor") val memoryPool: ActorRef = - context.actorOf(MemoryPool.props(settings, networkTimeProvider, influxReference), name = "mempool") + context.actorOf(MemoryPool.props(settings, networkTimeProvider, influxReference, mempoolProcessor), name = "mempool") val txValidator: ActorRef = context.actorOf( diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/view/mempool/MemoryPool.scala index 47e86b213f..72221153ea 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/view/mempool/MemoryPool.scala @@ -105,8 +105,6 @@ class MemoryPool( case _ => logger.debug(s"MemoryPoolProcessor got invalid type of DataFromPeer message!") } - case IsChainSynced(info) => chainSynced = info - case NewTransaction(transaction) => val (newMemoryPool: MemoryPoolStorage, validatedTransaction: Option[Transaction]) = memoryPool.validateTransaction(transaction) diff --git a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala index f8d05b8623..efdf12453e 100644 --- a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala +++ b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala @@ -63,7 +63,7 @@ class MemoryPoolTests "send transactions to miner" in { val miner1 = TestProbe() val mempool1: TestActorRef[MemoryPool] = - TestActorRef[MemoryPool](MemoryPool.props(testNetSettings, timeProvider, miner1.ref, Some(TestProbe().ref))) + TestActorRef[MemoryPool](MemoryPool.props(testNetSettings, timeProvider, Some(TestProbe().ref), miner1.ref)) val transactions1 = (0 until 4).map { k => val a = coinbaseAt(k) a From b1020ce1d58d365a3d4590201c5b4af5f0f46a78 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 10 Mar 2020 17:46:29 +0300 Subject: [PATCH 078/177] mempool moved --- src/main/resources/TestNetSettings.conf | 2 +- src/main/resources/application.conf | 2 +- src/main/scala/encry/Starter.scala | 14 +++++++----- .../http/routes/TransactionsApiRoute.scala | 2 +- .../api/http/routes/WalletInfoApiRoute.scala | 2 +- .../encry/cli/commands/CreateToken.scala | 2 +- .../scala/encry/cli/commands/Transfer.scala | 2 +- src/main/scala/encry/local/miner/Miner.scala | 2 +- .../mempool => mpg}/IntermediaryMempool.scala | 22 ++++++++++--------- .../{view/mempool => mpg}/MemoryPool.scala | 6 ++--- .../mempool => mpg}/MemoryPoolProcessor.scala | 6 ++--- .../mempool => mpg}/MemoryPoolReader.scala | 2 +- .../mempool => mpg}/MemoryPoolStorage.scala | 2 +- .../TransactionsValidator.scala | 6 ++--- .../scala/encry/network/DeliveryManager.scala | 2 +- .../DownloadedModifiersValidator.scala | 2 +- .../encry/network/NodeViewSynchronizer.scala | 2 +- .../scala/encry/nvg/IntermediaryNVH.scala | 2 +- src/main/scala/encry/nvg/NodeViewHolder.scala | 2 +- .../encry/view/mempool/MemoryPoolTests.scala | 5 +++-- 20 files changed, 46 insertions(+), 41 deletions(-) rename src/main/scala/encry/{view/mempool => mpg}/IntermediaryMempool.scala (78%) rename src/main/scala/encry/{view/mempool => mpg}/MemoryPool.scala (98%) rename src/main/scala/encry/{view/mempool => mpg}/MemoryPoolProcessor.scala (95%) rename src/main/scala/encry/{view/mempool => mpg}/MemoryPoolReader.scala (92%) rename src/main/scala/encry/{view/mempool => mpg}/MemoryPoolStorage.scala (99%) rename src/main/scala/encry/{view/mempool => mpg}/TransactionsValidator.scala (94%) diff --git a/src/main/resources/TestNetSettings.conf b/src/main/resources/TestNetSettings.conf index 5c199c0e6f..0dabd402aa 100644 --- a/src/main/resources/TestNetSettings.conf +++ b/src/main/resources/TestNetSettings.conf @@ -145,7 +145,7 @@ delivery-manager-dispatcher { throughput = 1 } mempool-dispatcher { - mailbox-type = "encry.view.mempool.MemoryPool$MemoryPoolPriorityQueue" + mailbox-type = "encry.mpg.MemoryPool$MemoryPoolPriorityQueue" type = Dispatcher executor = "thread-pool-executor" thread-pool-executor.fixed-pool-size = 2 diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 2c97730e8b..d89602ab55 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -218,7 +218,7 @@ delivery-manager-dispatcher { throughput = 1 } mempool-dispatcher { - mailbox-type = "encry.view.mempool.MemoryPool$MemoryPoolPriorityQueue" + mailbox-type = "encry.mpg.MemoryPool$MemoryPoolPriorityQueue" type = Dispatcher executor = "thread-pool-executor" thread-pool-executor.fixed-pool-size = 2 diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 0128ea4043..b216f699ba 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -1,10 +1,11 @@ package encry import java.net.InetSocketAddress -import akka.actor.{ Actor, ActorRef } + +import akka.actor.{Actor, ActorRef} import akka.http.scaladsl.Http import cats.Functor -import cats.data.{ NonEmptyChain, Validated } +import cats.data.{NonEmptyChain, Validated} import cats.instances.future._ import cats.instances.option._ import cats.syntax.apply._ @@ -15,19 +16,20 @@ import encry.Starter.InitNodeResult import encry.api.http.DataHolderForApi import encry.api.http.DataHolderForApi.PassForStorage import encry.cli.ConsoleListener -import encry.cli.ConsoleListener.{ prompt, StartListening } +import encry.cli.ConsoleListener.{StartListening, prompt} import encry.local.miner.Miner import encry.local.miner.Miner.StartMining +import encry.mpg.IntermediaryMempool import encry.network.NetworkRouter import encry.nvg.IntermediaryNVH import encry.settings._ import encry.stats.StatsSender -import encry.utils.{ Mnemonic, NetworkTimeProvider } -import encry.view.mempool.IntermediaryMempool +import encry.utils.{Mnemonic, NetworkTimeProvider} import encry.view.wallet.AccountManager + import scala.concurrent.Future import scala.io.StdIn -import scala.util.{ Failure, Success, Try } +import scala.util.{Failure, Success, Try} class Starter(settings: EncryAppSettings, timeProvider: NetworkTimeProvider, diff --git a/src/main/scala/encry/api/http/routes/TransactionsApiRoute.scala b/src/main/scala/encry/api/http/routes/TransactionsApiRoute.scala index 4697dec63e..84d1548a59 100644 --- a/src/main/scala/encry/api/http/routes/TransactionsApiRoute.scala +++ b/src/main/scala/encry/api/http/routes/TransactionsApiRoute.scala @@ -5,7 +5,7 @@ import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.server.Route import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport import encry.settings.RESTApiSettings -import encry.view.mempool.MemoryPool.NewTransaction +import encry.mpg.MemoryPool._ import org.encryfoundation.common.modifiers.mempool.transaction.Transaction case class TransactionsApiRoute(dataHolder: ActorRef, memoryPoolRef: ActorRef, settings: RESTApiSettings)( diff --git a/src/main/scala/encry/api/http/routes/WalletInfoApiRoute.scala b/src/main/scala/encry/api/http/routes/WalletInfoApiRoute.scala index 17e4cc51d5..49183f99f9 100644 --- a/src/main/scala/encry/api/http/routes/WalletInfoApiRoute.scala +++ b/src/main/scala/encry/api/http/routes/WalletInfoApiRoute.scala @@ -13,7 +13,7 @@ import encry.modifiers.mempool.TransactionFactory import encry.settings.RESTApiSettings import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import encry.view.history.History -import encry.view.mempool.MemoryPool.NewTransaction +import encry.mpg.MemoryPool._ import encry.view.state.UtxoState import encry.view.wallet.EncryWallet import io.circe.syntax._ diff --git a/src/main/scala/encry/cli/commands/CreateToken.scala b/src/main/scala/encry/cli/commands/CreateToken.scala index c1f9524194..127a8c214d 100644 --- a/src/main/scala/encry/cli/commands/CreateToken.scala +++ b/src/main/scala/encry/cli/commands/CreateToken.scala @@ -11,7 +11,7 @@ import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import encry.view.history.History -import encry.view.mempool.MemoryPool.NewTransaction +import encry.mpg.MemoryPool._ import encry.view.state.UtxoState import encry.view.wallet.EncryWallet import org.encryfoundation.common.crypto.PrivateKey25519 diff --git a/src/main/scala/encry/cli/commands/Transfer.scala b/src/main/scala/encry/cli/commands/Transfer.scala index 0f75eb828a..8ecefb792b 100644 --- a/src/main/scala/encry/cli/commands/Transfer.scala +++ b/src/main/scala/encry/cli/commands/Transfer.scala @@ -13,7 +13,7 @@ import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import encry.view.history.History -import encry.view.mempool.MemoryPool.NewTransaction +import encry.mpg.MemoryPool._ import encry.view.state.UtxoState import encry.view.wallet.EncryWallet import org.encryfoundation.common.crypto.PrivateKey25519 diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index b2b45cd2f4..9001fe3cd2 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -23,7 +23,7 @@ import encry.utils.NetworkTime.Time import encry.view.state.avlTree.utils.implicits.Instances._ import encry.view.NodeViewHolder.CurrentView import encry.view.history.History -import encry.view.mempool.MemoryPool.TransactionsForMiner +import encry.mpg.MemoryPool._ import encry.view.state.UtxoState import encry.utils.implicits.UTXO._ import encry.view.wallet.EncryWallet diff --git a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala b/src/main/scala/encry/mpg/IntermediaryMempool.scala similarity index 78% rename from src/main/scala/encry/view/mempool/IntermediaryMempool.scala rename to src/main/scala/encry/mpg/IntermediaryMempool.scala index 1872fa92d2..2903da3c3f 100644 --- a/src/main/scala/encry/view/mempool/IntermediaryMempool.scala +++ b/src/main/scala/encry/mpg/IntermediaryMempool.scala @@ -1,4 +1,4 @@ -package encry.view.mempool +package encry.mpg import akka.actor.{ Actor, ActorRef, Props } import com.typesafe.scalalogging.StrictLogging @@ -9,21 +9,23 @@ import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForTxHandling import encry.network.PeersKeeper.BanPeer import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.mempool.MemoryPool.{ RolledBackTransactions, TransactionProcessing } -import encry.view.mempool.TransactionsValidator.{ InvalidTransaction, ModifiersForValidating } - -class IntermediaryMempool(settings: EncryAppSettings, - networkTimeProvider: NetworkTimeProvider, - influxReference: Option[ActorRef], - networkRouter: ActorRef) - extends Actor +import encry.mpg.MemoryPool._ +import encry.mpg.TransactionsValidator.{ InvalidTransaction, ModifiersForValidating } + +class IntermediaryMempool( + settings: EncryAppSettings, + networkTimeProvider: NetworkTimeProvider, + influxReference: Option[ActorRef], + networkRouter: ActorRef +) extends Actor with StrictLogging { val mempoolProcessor: ActorRef = context.actorOf(MemoryPoolProcessor.props(settings, networkTimeProvider), name = "mempool-processor") val memoryPool: ActorRef = - context.actorOf(MemoryPool.props(settings, networkTimeProvider, influxReference, mempoolProcessor), name = "mempool") + context.actorOf(MemoryPool.props(settings, networkTimeProvider, influxReference, mempoolProcessor), + name = "mempool") val txValidator: ActorRef = context.actorOf( diff --git a/src/main/scala/encry/view/mempool/MemoryPool.scala b/src/main/scala/encry/mpg/MemoryPool.scala similarity index 98% rename from src/main/scala/encry/view/mempool/MemoryPool.scala rename to src/main/scala/encry/mpg/MemoryPool.scala index 72221153ea..2573685368 100644 --- a/src/main/scala/encry/view/mempool/MemoryPool.scala +++ b/src/main/scala/encry/mpg/MemoryPool.scala @@ -1,4 +1,4 @@ -package encry.view.mempool +package encry.mpg import akka.actor.{ Actor, ActorRef, ActorSystem, Props } import akka.dispatch.{ PriorityGenerator, UnboundedStablePriorityMailbox } @@ -7,18 +7,18 @@ import com.google.common.base.Charsets import com.google.common.hash.{ BloomFilter, Funnels } import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging +import encry.mpg.MemoryPool.MemoryPoolStateType.NotProcessingNewTransactions import encry.network.Messages.MessageToNetwork.{ RequestFromLocal, ResponseFromLocal } import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, SuccessfulTransaction } import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.mempool.MemoryPool.MemoryPoolStateType.NotProcessingNewTransactions -import encry.view.mempool.MemoryPool._ import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestModifiersNetworkMessage } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import encry.mpg.MemoryPool._ import scala.collection.IndexedSeq diff --git a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala b/src/main/scala/encry/mpg/MemoryPoolProcessor.scala similarity index 95% rename from src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala rename to src/main/scala/encry/mpg/MemoryPoolProcessor.scala index b86b30fa20..61e1173538 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolProcessor.scala +++ b/src/main/scala/encry/mpg/MemoryPoolProcessor.scala @@ -1,4 +1,4 @@ -package encry.view.mempool +package encry.mpg import akka.actor.{ Actor, Props } import com.google.common.base.Charsets @@ -9,12 +9,12 @@ import encry.network.Messages.MessageToNetwork.{ RequestFromLocal, ResponseFromL import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.mempool.MemoryPool.{ TransactionProcessing, UpdateMempoolReader } -import encry.view.mempool.MemoryPoolProcessor.CleanupBloomFilter import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestModifiersNetworkMessage } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import encry.mpg.MemoryPool._ +import encry.mpg.MemoryPoolProcessor.CleanupBloomFilter import scala.collection.IndexedSeq diff --git a/src/main/scala/encry/view/mempool/MemoryPoolReader.scala b/src/main/scala/encry/mpg/MemoryPoolReader.scala similarity index 92% rename from src/main/scala/encry/view/mempool/MemoryPoolReader.scala rename to src/main/scala/encry/mpg/MemoryPoolReader.scala index 470317f68b..1e51d26892 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolReader.scala +++ b/src/main/scala/encry/mpg/MemoryPoolReader.scala @@ -1,4 +1,4 @@ -package encry.view.mempool +package encry.mpg import org.encryfoundation.common.modifiers.mempool.transaction.Transaction diff --git a/src/main/scala/encry/view/mempool/MemoryPoolStorage.scala b/src/main/scala/encry/mpg/MemoryPoolStorage.scala similarity index 99% rename from src/main/scala/encry/view/mempool/MemoryPoolStorage.scala rename to src/main/scala/encry/mpg/MemoryPoolStorage.scala index 1c6c1dd005..bcf425c729 100644 --- a/src/main/scala/encry/view/mempool/MemoryPoolStorage.scala +++ b/src/main/scala/encry/mpg/MemoryPoolStorage.scala @@ -1,4 +1,4 @@ -package encry.view.mempool +package encry.mpg import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider diff --git a/src/main/scala/encry/view/mempool/TransactionsValidator.scala b/src/main/scala/encry/mpg/TransactionsValidator.scala similarity index 94% rename from src/main/scala/encry/view/mempool/TransactionsValidator.scala rename to src/main/scala/encry/mpg/TransactionsValidator.scala index 4d36f95cc5..ca583e480d 100644 --- a/src/main/scala/encry/view/mempool/TransactionsValidator.scala +++ b/src/main/scala/encry/mpg/TransactionsValidator.scala @@ -1,4 +1,4 @@ -package encry.view.mempool +package encry.mpg import TransactionProto.TransactionProtoMessage import akka.actor.{ Actor, ActorRef, Props } @@ -10,11 +10,11 @@ import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.InvalidModifierBytes import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.mempool.MemoryPool.NewTransaction -import encry.view.mempool.TransactionsValidator.InvalidTransaction import org.encryfoundation.common.modifiers.mempool.transaction.TransactionProtoSerializer import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } +import encry.mpg.MemoryPool._ +import encry.mpg.TransactionsValidator.InvalidTransaction import scala.util.{ Failure, Success, Try } diff --git a/src/main/scala/encry/network/DeliveryManager.scala b/src/main/scala/encry/network/DeliveryManager.scala index 64ade81e21..47c475f688 100644 --- a/src/main/scala/encry/network/DeliveryManager.scala +++ b/src/main/scala/encry/network/DeliveryManager.scala @@ -27,7 +27,7 @@ import encry.network.PeersKeeper._ import encry.network.PrioritiesCalculator.AccumulatedPeersStatistic import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus.BadNode -import encry.view.mempool.MemoryPool.{StartTransactionsValidation, StopTransactionsValidation} +import encry.mpg.MemoryPool._ import org.encryfoundation.common.modifiers.history.{Block, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo._ diff --git a/src/main/scala/encry/network/DownloadedModifiersValidator.scala b/src/main/scala/encry/network/DownloadedModifiersValidator.scala index 8732b1d7eb..d907297913 100644 --- a/src/main/scala/encry/network/DownloadedModifiersValidator.scala +++ b/src/main/scala/encry/network/DownloadedModifiersValidator.scala @@ -14,7 +14,7 @@ import encry.network.PeersKeeper.BanPeer import encry.settings.EncryAppSettings import encry.stats.StatsSender.ValidatedModifierFromNetwork import encry.view.history.History -import encry.view.mempool.MemoryPool.NewTransaction +import encry.mpg.MemoryPool._ import org.encryfoundation.common.modifiers.mempool.transaction.{Transaction, TransactionProtoSerializer} import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index e3d4b4bbbb..718adb3148 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -10,6 +10,7 @@ import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus._ import encry.local.miner.Miner.{ClIMiner, DisableMining, StartMining} +import encry.mpg.MemoryPool._ import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NodeViewSynchronizer.ReceivableMessages._ @@ -22,7 +23,6 @@ import encry.utils.Utils._ import encry.view.NodeViewHolder.ReceivableMessages.{CompareViews, GetNodeViewChanges} import encry.view.NodeViewErrors.ModifierApplyError import encry.view.history.History -import encry.view.mempool.MemoryPool._ import encry.view.state.UtxoState import org.encryfoundation.common.modifiers.{NodeViewModifier, PersistentNodeViewModifier} import org.encryfoundation.common.modifiers.history._ diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index d789ef0ed3..cb26c80753 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -34,7 +34,7 @@ import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader -import encry.view.mempool.MemoryPool.RolledBackTransactions +import encry.mpg.MemoryPool._ import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestChunkMessage, diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index f628a3f571..30f2126fbe 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -27,7 +27,7 @@ import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.NodeViewHolder.CurrentView import encry.view.history.storage.HistoryStorage import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } -import encry.view.mempool.MemoryPool.RolledBackTransactions +import encry.mpg.MemoryPool._ import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet diff --git a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala index efdf12453e..76fc47ff40 100644 --- a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala +++ b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala @@ -4,9 +4,10 @@ import akka.actor.ActorSystem import akka.testkit.{ TestActorRef, TestProbe } import com.typesafe.scalalogging.StrictLogging import encry.modifiers.InstanceFactory -import encry.settings.{ EncryAppSettings, TestNetSettings } +import encry.mpg.MemoryPool._ +import encry.mpg.{ MemoryPool, MemoryPoolStorage } +import encry.settings.TestNetSettings import encry.utils.NetworkTimeProvider -import encry.view.mempool.MemoryPool.{ NewTransaction, TransactionsForMiner } import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } import scala.concurrent.duration._ From fa9a626371d7a80babbd781b3ac0d645fb7f12df Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 10 Mar 2020 17:57:28 +0300 Subject: [PATCH 079/177] add dataholder --- src/main/scala/encry/Starter.scala | 4 ++-- src/main/scala/encry/network/DM.scala | 6 +++--- src/main/scala/encry/network/MessageBuilder.scala | 1 + src/main/scala/encry/nvg/IntermediaryNVH.scala | 12 ++++++++---- src/main/scala/encry/nvg/NodeViewHolder.scala | 1 + 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index b216f699ba..65ce379acf 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -406,7 +406,7 @@ class Starter(settings: EncryAppSettings, context.system .actorOf(StatsSender.props(influxSettings, newSettings.network, newSettings.constants), "statsSender") } - lazy val dataHolderForApi = + val dataHolderForApi = context.system.actorOf(DataHolderForApi.props(newSettings, timeProvider), "dataHolder") if (nodePass.nonEmpty) dataHolderForApi ! PassForStorage(nodePass) @@ -422,7 +422,7 @@ class Starter(settings: EncryAppSettings, IntermediaryMempool.props(newSettings, timeProvider, influxRef, networkRouter) ) val nvhRouter: ActorRef = context.system.actorOf( - IntermediaryNVH.props(newSettings, networkRouter, timeProvider, influxRef, memoryPool) + IntermediaryNVH.props(newSettings, networkRouter, timeProvider, influxRef, memoryPool, dataHolderForApi) ) val miner: ActorRef = context.system.actorOf(Miner.props(dataHolderForApi, memoryPool, nvhRouter, influxRef, newSettings), "miner") diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index b766d63723..841b48cf5e 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -38,7 +38,7 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging context.parent ! ModifierFromNetwork(source, data._1, id, bytes) expectedModifiers -= toKey(id) receivedModifier += toKey(id) - } else logger.info("Receive spam!") + } else logger.info(s"Receive spam. ModId: ${Algos.encode(id)}!") } case RequestSent(peer, modTypeId, modId) if !expectedModifiers.contains(toKey(modId))=> expectedModifiers += toKey(modId) @@ -67,8 +67,8 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging case IsRequested(modIds) => //logger.info(s"Going to check if ${Algos.encode(modId)} has been requested. Res: ${receivedModifier.contains(toKey(modId))}") sender ! RequestStatus( - modIds.filter(id => receivedModifier.contains(toKey(id)) || expectedModifiers.contains(toKey(id))), - modIds.filter(id => !receivedModifier.contains(toKey(id)) && !expectedModifiers.contains(toKey(id))) + requested = modIds.filter(id => receivedModifier.contains(toKey(id)) || expectedModifiers.contains(toKey(id))), + notRequested = modIds.filter(id => !receivedModifier.contains(toKey(id)) && !expectedModifiers.contains(toKey(id))) ) } diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 544af268ea..3adeba3341 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -33,6 +33,7 @@ case class MessageBuilder(peersKeeper: ActorRef, (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => peer.handlerRef ! RequestModifiersNetworkMessage(modTypeId -> status.notRequested) status.notRequested.foreach(modId => deliveryManager ! RequestSent(peer.socketAddress, modTypeId, modId)) + logger.info(s"Send req for mods: ${status.notRequested.map(Algos.encode).mkString(",")} to peer ${peer.socketAddress}") logger.info(s"Requested or received: ${status.requested.length}. Not request or not received: ${status.notRequested.length}") context.parent ! MsgSent(RequestModifiersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index cb26c80753..18dac26955 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -51,7 +51,8 @@ class IntermediaryNVH( intermediaryNetwork: ActorRef, timeProvider: NetworkTimeProvider, influxRef: Option[ActorRef], - mempoolRef: ActorRef + mempoolRef: ActorRef, + dataHolderRef: ActorRef ) extends Actor with StrictLogging { @@ -109,6 +110,7 @@ class IntermediaryNVH( case msg @ UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader networkMessagesProcessor ! msg + dataHolderRef ! msg case msg: LocallyGeneratedModifier => nodeViewHolder ! msg case msg @ BanPeer(_, _) => intermediaryNetwork ! msg case msg @ InvalidModifierBytes(_) => intermediaryNetwork ! msg @@ -126,7 +128,8 @@ class IntermediaryNVH( case msg @ RolledBackTransactions(_) => mempoolRef ! msg case msg @ DisableMining => //+ to miner case msg @ StartMining => //+ to miner - case msg @ BlockAndHeaderInfo(_, _) => //+ to data holder + case msg @ BlockAndHeaderInfo(_, _) => dataHolderRef ! msg + case msg: UpdatedHistory => dataHolderRef ! msg case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) case msg @ GetDataFromCurrentView(_) => nodeViewHolder.forward(msg) case msg @ RollbackSucceed(_) => @@ -144,6 +147,7 @@ object IntermediaryNVH { intermediaryNetwork: ActorRef, timeProvider: NetworkTimeProvider, influxRef: Option[ActorRef], - mempoolRef: ActorRef - ): Props = Props(new IntermediaryNVH(settings, intermediaryNetwork, timeProvider, influxRef, mempoolRef)) + mempoolRef: ActorRef, + dataHolderRef: ActorRef + ): Props = Props(new IntermediaryNVH(settings, intermediaryNetwork, timeProvider, influxRef, mempoolRef, dataHolderRef)) } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 30f2126fbe..1dc865246d 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -283,6 +283,7 @@ class NodeViewHolder( if (settings.node.mining && progressInfo.chainSwitchingNeeded) context.parent ! StartMining context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) + context.parent ! UpdatedHistory(history) if (newHis.getBestHeaderId.exists( bestHeaderId => newHis.getBestBlockId.exists(bId => ByteArrayWrapper(bId) == ByteArrayWrapper(bestHeaderId)) From e3b3326fac0b6a79e18394a8ce68543e0318ea67 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 10 Mar 2020 18:09:31 +0300 Subject: [PATCH 080/177] use history reader instead of history --- .../encry/api/http/DataHolderForApi.scala | 12 ++++----- .../scala/encry/nvg/IntermediaryNVH.scala | 2 +- src/main/scala/encry/nvg/NodeViewHolder.scala | 1 - .../encry/view/history/HistoryReader.scala | 26 +++++++++++++++++-- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/main/scala/encry/api/http/DataHolderForApi.scala b/src/main/scala/encry/api/http/DataHolderForApi.scala index 935b592d8d..037c3ab2a6 100644 --- a/src/main/scala/encry/api/http/DataHolderForApi.scala +++ b/src/main/scala/encry/api/http/DataHolderForApi.scala @@ -21,11 +21,11 @@ import encry.network.ConnectedPeersCollection import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.ConnectedPeer import encry.network.PeersKeeper.BanPeerFromAPI -import encry.nvg.NodeViewHolder.NodeViewChange +import encry.nvg.NodeViewHolder.{NodeViewChange, UpdateHistoryReader} import encry.settings.EncryAppSettings import encry.utils.{NetworkTime, NetworkTimeProvider} import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView -import encry.view.history.History +import encry.view.history.{History, HistoryReader} import encry.view.state.{UtxoState, UtxoStateReader} import encry.view.wallet.EncryWallet import org.encryfoundation.common.crypto.PrivateKey25519 @@ -55,7 +55,7 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) val launchTimeFuture: Future[NetworkTime.Time] = ntp.time() def awaitNVHRef: Receive = { - case UpdatedHistory(history) => + case UpdateHistoryReader(history) => unstashAll() context.become(workingCycle(nvhRef = sender(), history = Some(history))) case PassForStorage(_) => @@ -66,7 +66,7 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) def workingCycle(nvhRef: ActorRef, blackList: Seq[(InetAddress, (BanReason, BanTime, BanType))] = Seq.empty, connectedPeers: Seq[ConnectedPeer] = Seq.empty, - history: Option[History] = None, + history: Option[HistoryReader] = None, state: Option[UtxoStateReader] = None, transactionsOnMinerActor: Int = 0, minerStatus: MinerStatus = MinerStatus(isMining = false, None), @@ -122,7 +122,7 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) connectedPeersCollection) ) - case ChangedHistory(reader: History) => + case UpdateHistoryReader(reader: HistoryReader) => context.become( workingCycle(nvhRef, blackList, @@ -333,7 +333,7 @@ object DataHolderForApi { //scalastyle:ignore final case class GetLastHeaderIdAtHeightHelper(i: Int) - final case class Readers(h: Option[History], s: Option[UtxoStateReader]) + final case class Readers(h: Option[HistoryReader], s: Option[UtxoStateReader]) final case class PassForStorage(password: String) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 18dac26955..5d4127fe0c 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -129,7 +129,7 @@ class IntermediaryNVH( case msg @ DisableMining => //+ to miner case msg @ StartMining => //+ to miner case msg @ BlockAndHeaderInfo(_, _) => dataHolderRef ! msg - case msg: UpdatedHistory => dataHolderRef ! msg + case msg: UpdateHistoryReader => dataHolderRef ! msg case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) case msg @ GetDataFromCurrentView(_) => nodeViewHolder.forward(msg) case msg @ RollbackSucceed(_) => diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 1dc865246d..30f2126fbe 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -283,7 +283,6 @@ class NodeViewHolder( if (settings.node.mining && progressInfo.chainSwitchingNeeded) context.parent ! StartMining context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) - context.parent ! UpdatedHistory(history) if (newHis.getBestHeaderId.exists( bestHeaderId => newHis.getBestBlockId.exists(bId => ByteArrayWrapper(bId) == ByteArrayWrapper(bestHeaderId)) diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 8e3deb193e..64f9c9edb7 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -1,7 +1,9 @@ package encry.view.history -import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Older } -import org.encryfoundation.common.modifiers.history.Header +import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Older} +import encry.modifiers.history.HeaderChain +import io.iohk.iodb.ByteArrayWrapper +import org.encryfoundation.common.modifiers.history.{Block, Header} import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -19,16 +21,26 @@ trait HistoryReader { def compare(si: SyncInfo): HistoryComparisonResult + def getHeaderById(id: ModifierId): Option[Header] + + def getBlockByHeaderId(id: ModifierId): Option[Block] + + def getBlockByHeader(header: Header): Option[Block] + var isFullChainSynced: Boolean var isHeadersChainSyncedVar: Boolean = false def isModifierDefined(id: ModifierId): Boolean + def headerIdsAtHeight(height: Int): List[ModifierId] + def modifierBytesById(id: ModifierId): Option[Array[Byte]] def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] + def lastHeaders(count: Int): HeaderChain + def syncInfo: SyncInfo def isFastSyncInProcess: Boolean @@ -47,6 +59,11 @@ object HistoryReader { def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty def syncInfo: SyncInfo = SyncInfo(Seq.empty) def isFastSyncInProcess: Boolean = false + def getHeaderById(id: ModifierId): Option[Header] = None + def getBlockByHeaderId(id: ModifierId): Option[Block] = None + def getBlockByHeader(header: Header): Option[Block] = None + def headerIdsAtHeight(height: Int): List[ModifierId] = List.empty[ModifierId] + def lastHeaders(count: Int): HeaderChain = HeaderChain.empty } def apply(history: History): HistoryReader = new HistoryReader { @@ -61,5 +78,10 @@ object HistoryReader { def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = history.payloadsIdsToDownload(howMany, HashSet.empty) def syncInfo: SyncInfo = history.syncInfo def isFastSyncInProcess: Boolean = history.fastSyncInProgress.fastSyncVal + def getHeaderById(id: ModifierId): Option[Header] = history.getHeaderById(id) + def headerIdsAtHeight(height: Int): List[ModifierId] = history.headerIdsAtHeight(height).toList + def getBlockByHeaderId(id: ModifierId): Option[Block] = history.getBlockByHeaderId(id) + def getBlockByHeader(header: Header): Option[Block] = history.getBlockByHeader(header) + def lastHeaders(count: Int): HeaderChain = history.lastHeaders(count) } } From 159b5a83a2ddabc291afc564e9e8a9f5542050b2 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 10 Mar 2020 18:39:39 +0300 Subject: [PATCH 081/177] send info about peers from network router to data holder --- src/main/scala/encry/Starter.scala | 2 +- .../encry/api/http/DataHolderForApi.scala | 21 +++++++------------ .../encry/api/http/routes/PeersApiRoute.scala | 6 +++--- .../scala/encry/network/NetworkRouter.scala | 8 +++++-- src/main/scala/encry/network/PK.scala | 20 ++++++++++++++++-- 5 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 65ce379acf..bebdd2fc7c 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -413,7 +413,7 @@ class Starter(settings: EncryAppSettings, val networkRouter = context.system.actorOf( NetworkRouter - .props(networkSettings, settings.blackList) + .props(networkSettings, settings.blackList, dataHolderForApi) .withDispatcher("nvsh-dispatcher"), "networkRouter" ) diff --git a/src/main/scala/encry/api/http/DataHolderForApi.scala b/src/main/scala/encry/api/http/DataHolderForApi.scala index 037c3ab2a6..4382cab089 100644 --- a/src/main/scala/encry/api/http/DataHolderForApi.scala +++ b/src/main/scala/encry/api/http/DataHolderForApi.scala @@ -19,7 +19,7 @@ import encry.network.BlackList.BanReason.InvalidNetworkMessage import encry.network.BlackList.{BanReason, BanTime, BanType} import encry.network.ConnectedPeersCollection import encry.network.NodeViewSynchronizer.ReceivableMessages._ -import encry.network.PeerConnectionHandler.ConnectedPeer +import encry.network.PeerConnectionHandler.{ConnectedPeer, ConnectionType} import encry.network.PeersKeeper.BanPeerFromAPI import encry.nvg.NodeViewHolder.{NodeViewChange, UpdateHistoryReader} import encry.settings.EncryAppSettings @@ -64,14 +64,14 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) } def workingCycle(nvhRef: ActorRef, - blackList: Seq[(InetAddress, (BanReason, BanTime, BanType))] = Seq.empty, - connectedPeers: Seq[ConnectedPeer] = Seq.empty, + blackList: List[(InetAddress, (BanReason, BanTime, BanType))] = List.empty, + connectedPeers: List[(InetSocketAddress, String, ConnectionType)] = List.empty, history: Option[HistoryReader] = None, state: Option[UtxoStateReader] = None, transactionsOnMinerActor: Int = 0, minerStatus: MinerStatus = MinerStatus(isMining = false, None), blockInfo: BlockAndHeaderInfo = BlockAndHeaderInfo(None, None), - allPeers: Seq[InetSocketAddress] = Seq.empty, + allPeers: List[InetSocketAddress] = List.empty, connectedPeersCollection: ConnectedPeersCollection = ConnectedPeersCollection()): Receive = { case UpdatingTransactionsNumberForApi(qty) => @@ -230,12 +230,7 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) case GetBannedPeersHelper => sender() ! blackList case GetConnectedPeersHelper => sender() ! connectedPeers - .map( - peer => - PeerInfoResponse(peer.socketAddress.toString, - Some(peer.handshake.nodeName), - Some(peer.direction.toString))) - + .map(peer => PeerInfoResponse(peer._1.toString, peer._2, peer._3.toString)) case GetLastHeaderIdAtHeightHelper(i) => sender() ! history.toList.flatMap(_.headerIdsAtHeight(i).map(Algos.encode)) @@ -317,9 +312,9 @@ object DataHolderForApi { //scalastyle:ignore final case class UpdatingMinerStatus(minerStatus: MinerStatus) extends AnyVal - final case class UpdatingPeersInfo(allPeers: Seq[InetSocketAddress], - connectedPeers: Seq[ConnectedPeer], - blackList: Seq[(InetAddress, (BanReason, BanTime, BanType))]) + final case class UpdatingPeersInfo(allPeers: List[InetSocketAddress], + connectedPeers: List[(InetSocketAddress, String, ConnectionType)], + blackList: List[(InetAddress, (BanReason, BanTime, BanType))]) final case class BlockAndHeaderInfo(header: Option[Header], block: Option[Block]) diff --git a/src/main/scala/encry/api/http/routes/PeersApiRoute.scala b/src/main/scala/encry/api/http/routes/PeersApiRoute.scala index 4130e69ac5..d0cf1af184 100755 --- a/src/main/scala/encry/api/http/routes/PeersApiRoute.scala +++ b/src/main/scala/encry/api/http/routes/PeersApiRoute.scala @@ -76,14 +76,14 @@ case class PeersApiRoute(override val settings: RESTApiSettings, dataHolder: Act object PeersApiRoute { - case class PeerInfoResponse(address: String, name: Option[String], connectionType: Option[String]) + case class PeerInfoResponse(address: String, name: String, connectionType: String) object PeerInfoResponse { def fromAddressAndInfo(address: InetSocketAddress, peerInfo: PeerInfo): PeerInfoResponse = PeerInfoResponse( address.toString, - Some(peerInfo.connectedPeer.toString), - Some(peerInfo.connectionType.toString) + peerInfo.connectedPeer.toString, + peerInfo.connectionType.toString ) } diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 9ceb8a9576..419ab5bab3 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -7,6 +7,7 @@ import akka.io.Tcp.{Bind, Bound, CommandFailed, Connect, Connected} import akka.io.{IO, Tcp} import akka.io.Tcp.SO.KeepAlive import com.typesafe.scalalogging.StrictLogging +import encry.api.http.DataHolderForApi.UpdatingPeersInfo import encry.network.BlackList.BanReason.InvalidNetworkMessage import encry.network.Messages.MessageToNetwork import encry.network.MessageBuilder.MsgSent @@ -28,7 +29,8 @@ import scala.concurrent.duration._ import scala.util.Random class NetworkRouter(settings: NetworkSettings, - blackListSettings: BlackListSettings) extends Actor with StrictLogging { + blackListSettings: BlackListSettings, + dataHolderRef: ActorRef) extends Actor with StrictLogging { import context.system import context.dispatcher @@ -78,6 +80,7 @@ class NetworkRouter(settings: NetworkSettings, case msg: ModifierFromNetwork if msg.modTypeId != Transaction.modifierTypeId => handlerForMods ! msg case msg: ModifierFromNetwork => txsHandler ! msg case msg: OtherNodeSyncingStatus => peersKeeper ! msg + case msg: UpdatingPeersInfo => dataHolderRef ! msg case msg: MessageToNetwork => context.actorOf( MessageBuilder.props(peersKeeper, deliveryManager), @@ -138,5 +141,6 @@ object NetworkRouter { case object RegisterForTxHandling def props(settings: NetworkSettings, - blackListSettings: BlackListSettings): Props = Props(new NetworkRouter(settings, blackListSettings)) + blackListSettings: BlackListSettings, + dataHolderRef: ActorRef): Props = Props(new NetworkRouter(settings, blackListSettings, dataHolderRef)) } diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 0ab62b5224..8b175cfcd2 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -2,8 +2,9 @@ package encry.network import java.net.{InetAddress, InetSocketAddress} -import akka.actor.{Actor, Props} +import akka.actor.{Actor, ActorRef, Props} import com.typesafe.scalalogging.StrictLogging +import encry.api.http.DataHolderForApi.UpdatingPeersInfo import encry.network.BlackList.BanReason.SentPeersMessageWithoutRequest import encry.network.BlackList.{BanReason, BanTime, BanType} import encry.network.ConnectedPeersCollection.PeerInfo @@ -12,17 +13,20 @@ import encry.network.Messages.MessageToNetwork.SendPeers import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection -import encry.network.PeerConnectionHandler.{Incoming, Outgoing} +import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming, Outgoing} import encry.network.PeersKeeper.{BanPeer, BanPeerFromAPI, PeerForConnection, RequestPeerForConnection} import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.{GetPeersNetworkMessage, PeersNetworkMessage} +import scala.concurrent.duration._ import scala.util.{Random, Try} class PK(networkSettings: NetworkSettings, blacklistSettings: BlackListSettings) extends Actor with StrictLogging { + import context.dispatcher + val connectWithOnlyKnownPeers: Boolean = networkSettings.connectOnlyWithKnownPeers.getOrElse(true) var connectedPeers: ConnectedPeersCollection = ConnectedPeersCollection() @@ -44,6 +48,15 @@ class PK(networkSettings: NetworkSettings, PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" ), self) + context.system.scheduler.schedule(5.seconds, 5.seconds) { + context.parent ! UpdatingPeersInfo( + peersForConnection.keys.toList, + connectedPeers.collect[ConnectedPeer](getAllPeers, getConnectedPeers).map(peer => + (peer.socketAddress, peer.handshake.nodeName, peer.direction) + ).toList, + blackList.getAll.toList + ) + } } override def receive: Receive = banPeersLogic orElse networkMessagesProcessingLogic orElse { @@ -182,6 +195,9 @@ class PK(networkSettings: NetworkSettings, networkSettings.declaredAddress.contains(address) || InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) || InetAddress.getLoopbackAddress.getAddress.sameElements(address.getAddress.getAddress)).getOrElse(true) + + def getAllPeers: (InetSocketAddress, PeerInfo) => Boolean = (_, _) => true + def getConnectedPeers(add: InetSocketAddress, info: PeerInfo): ConnectedPeer = info.connectedPeer } object PK { From caca490355c7a8d14071e6593bf2dadc7fb111a3 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 12 Mar 2020 17:45:27 +0300 Subject: [PATCH 082/177] stop mining during rollback, send txs for miner by ask --- src/main/scala/encry/local/miner/Miner.scala | 17 ++++++++------- .../scala/encry/mpg/IntermediaryMempool.scala | 1 + src/main/scala/encry/mpg/MemoryPool.scala | 12 +++-------- .../encry/network/NodeViewSynchronizer.scala | 2 +- .../scala/encry/nvg/IntermediaryNVH.scala | 1 - src/main/scala/encry/nvg/NodeViewHolder.scala | 21 ++++++++++--------- 6 files changed, 26 insertions(+), 28 deletions(-) diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index 9001fe3cd2..b8531104b6 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -5,6 +5,7 @@ import java.util.Date import akka.actor.{Actor, ActorRef, Props} import akka.util.Timeout +import akka.pattern._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.EncryApp._ @@ -63,7 +64,7 @@ class Miner(dataHolder: ActorRef, var transactionsPool: IndexedSeq[Transaction] = IndexedSeq.empty[Transaction] override def preStart(): Unit = { - context.system.eventStream.subscribe(self, classOf[ClIMiner]) + context.system.eventStream.subscribe(self, classOf[MinerMiningCommands]) context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) context.system.eventStream.subscribe(self, classOf[BlockchainStatus]) context.system.scheduler.schedule(5.seconds, 5.seconds)( @@ -228,7 +229,7 @@ class Miner(dataHolder: ActorRef, } def produceCandidate(): Unit = { - val lambda = (nodeView: CurrentView[History, UtxoState, EncryWallet]) => + def lambda(txs: List[Transaction]) = (nodeView: CurrentView[History, UtxoState, EncryWallet]) => { val producingStartTime: Time = System.currentTimeMillis() startTime = producingStartTime @@ -252,19 +253,21 @@ class Miner(dataHolder: ActorRef, } else CandidateEnvelope.empty candidate } - nvh ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope](lambda) + (mempool ? SendTransactionsToMiner).mapTo[List[Transaction]].foreach { txs => + nvh ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope](lambda(txs)) + } } } object Miner { - sealed trait ClIMiner + sealed trait MinerMiningCommands - case object DisableMining extends ClIMiner + case object DisableMining extends MinerMiningCommands - case object EnableMining extends ClIMiner + case object EnableMining extends MinerMiningCommands - case object StartMining extends ClIMiner + case object StartMining extends MinerMiningCommands case class MinedBlock(block: Block, workerIdx: Int) diff --git a/src/main/scala/encry/mpg/IntermediaryMempool.scala b/src/main/scala/encry/mpg/IntermediaryMempool.scala index 2903da3c3f..51d4c450a0 100644 --- a/src/main/scala/encry/mpg/IntermediaryMempool.scala +++ b/src/main/scala/encry/mpg/IntermediaryMempool.scala @@ -44,6 +44,7 @@ class IntermediaryMempool( case msg: RequestFromLocal => networkRouter ! msg case msg: ModifierFromNetwork => txValidator ! msg case msg: TransactionProcessing => mempoolProcessor ! msg + case msg @ SendTransactionsToMiner => memoryPool.forward(msg) case msg @ FullBlockChainIsSynced => mempoolProcessor ! msg } } diff --git a/src/main/scala/encry/mpg/MemoryPool.scala b/src/main/scala/encry/mpg/MemoryPool.scala index 2573685368..5bf935e1db 100644 --- a/src/main/scala/encry/mpg/MemoryPool.scala +++ b/src/main/scala/encry/mpg/MemoryPool.scala @@ -48,12 +48,6 @@ class MemoryPool( self, RemoveExpiredFromPool ) - context.system.scheduler.schedule( - settings.mempool.txSendingInterval, - settings.mempool.txSendingInterval, - self, - SendTransactionsToMiner - ) context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) } @@ -181,7 +175,7 @@ class MemoryPool( memoryPool.getTransactionsForMiner memoryPool = newMemoryPool mempoolProcessor ! UpdateMempoolReader(MemoryPoolReader.apply(memoryPool)) - context.parent ! TransactionsForMiner(transactionsForMiner) + sender() ! TransactionsForMiner(transactionsForMiner) logger.debug( s"MemoryPool got SendTransactionsToMiner. Size of transactions for miner ${transactionsForMiner.size}." + s" New pool size is ${memoryPool.size}. Ids ${transactionsForMiner.map(_.encodedId)}" @@ -220,9 +214,9 @@ object MemoryPool { final case class NewTransaction(tx: Transaction) extends AnyVal - final case class RolledBackTransactions(txs: IndexedSeq[Transaction]) extends AnyVal + final case class RolledBackTransactions(txs: IndexedSeq[Transaction]) - final case class TransactionsForMiner(txs: Seq[Transaction]) extends AnyVal + final case class TransactionsForMiner(txs: Seq[Transaction]) case object SendTransactionsToMiner diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index 718adb3148..45fd0cde77 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -9,7 +9,7 @@ import akka.util.Timeout import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus._ -import encry.local.miner.Miner.{ClIMiner, DisableMining, StartMining} +import encry.local.miner.Miner.{MinerMiningCommands, DisableMining, StartMining} import encry.mpg.MemoryPool._ import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 5d4127fe0c..b6b6b65a1f 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -126,7 +126,6 @@ class IntermediaryNVH( case msg @ HeaderChainIsSynced => case msg @ FullBlockChainIsSynced => mempoolRef ! msg case msg @ RolledBackTransactions(_) => mempoolRef ! msg - case msg @ DisableMining => //+ to miner case msg @ StartMining => //+ to miner case msg @ BlockAndHeaderInfo(_, _) => dataHolderRef ! msg case msg: UpdateHistoryReader => dataHolderRef ! msg diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 30f2126fbe..c3e88618b3 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -2,19 +2,19 @@ package encry.nvg import java.io.File -import akka.actor.{ Actor, ActorRef, Props } +import akka.actor.{Actor, ActorRef, Props} import akka.pattern._ import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.local.miner.Miner.{DisableMining, EnableMining, StartMining} import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } +import encry.nvg.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} import encry.nvg.NodeViewHolder._ import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId import encry.nvg.fast.sync.SnapshotProcessor._ @@ -26,23 +26,23 @@ import encry.view.NodeViewErrors.ModifierApplyError import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.NodeViewHolder.CurrentView import encry.view.history.storage.HistoryStorage -import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } +import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} import encry.mpg.MemoryPool._ import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } +import org.encryfoundation.common.modifiers.{PersistentModifier, PersistentNodeViewModifier} import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } +import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} -import scala.collection.{ mutable, IndexedSeq, Seq } +import scala.collection.{IndexedSeq, Seq, mutable} import scala.concurrent.Future import scala.concurrent.duration._ -import scala.util.{ Failure, Success, Try } +import scala.util.{Failure, Success, Try} class NodeViewHolder( settings: EncryAppSettings, @@ -224,7 +224,7 @@ class NodeViewHolder( val blockAtHeight: Block = history.getBlockByHeader(headerAtHeight).get blocks :+ blockAtHeight } - context.parent ! DisableMining + context.system.eventStream.publish(DisableMining) state.rollbackTo(branchPoint, additionalBlocks) -> trimChainSuffix(suffixApplied, ModifierId !@@ branchPoint) } else Success(state) -> IndexedSeq.empty @@ -233,6 +233,7 @@ class NodeViewHolder( stateToApplyTry match { case Success(stateToApply: UtxoState) => context.parent ! RollbackSucceed(branchingPointOpt) + if (settings.node.mining && nodeView.history.isFullChainSynced) context.system.eventStream.publish(EnableMining) val u0: UpdateInformation = UpdateInformation(history, stateToApply, none, none, suffixTrimmed) val uf: UpdateInformation = progressInfo.toApply.foldLeft(u0) { case (u: UpdateInformation, modToApply: PersistentModifier) => From e0bc8cd111a405102c8382cbe936c0001a0028d5 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 12 Mar 2020 17:49:32 +0300 Subject: [PATCH 083/177] ec-fix --- src/main/scala/encry/local/miner/Miner.scala | 4 ++-- src/main/scala/encry/nvg/NodeViewHolder.scala | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index b8531104b6..c6e1974fcd 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -253,8 +253,8 @@ class Miner(dataHolder: ActorRef, } else CandidateEnvelope.empty candidate } - (mempool ? SendTransactionsToMiner).mapTo[List[Transaction]].foreach { txs => - nvh ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope](lambda(txs)) + (mempool ? SendTransactionsToMiner).mapTo[TransactionsForMiner].foreach { txs => + nvh ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope](lambda(txs.txs.toList)) } } } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index c3e88618b3..17b6488ca8 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -233,7 +233,7 @@ class NodeViewHolder( stateToApplyTry match { case Success(stateToApply: UtxoState) => context.parent ! RollbackSucceed(branchingPointOpt) - if (settings.node.mining && nodeView.history.isFullChainSynced) context.system.eventStream.publish(EnableMining) + if (settings.node.mining && nodeView.history.isFullChainSynced && progressInfo.chainSwitchingNeeded) context.system.eventStream.publish(EnableMining) val u0: UpdateInformation = UpdateInformation(history, stateToApply, none, none, suffixTrimmed) val uf: UpdateInformation = progressInfo.toApply.foldLeft(u0) { case (u: UpdateInformation, modToApply: PersistentModifier) => From 77c9107c7d8bc0ed8b8982934280633c5a4a3947 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 12 Mar 2020 17:54:48 +0300 Subject: [PATCH 084/177] remove mempool buffer at miner --- src/main/scala/encry/local/miner/Miner.scala | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index c6e1974fcd..7a59b31a61 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -61,18 +61,13 @@ class Miner(dataHolder: ActorRef, val numberOfWorkers: Int = settings.node.numberOfMiningWorkers val powScheme: EquihashPowScheme = EquihashPowScheme(TestNetConstants.n, TestNetConstants.k, TestNetConstants.Version, TestNetConstants.PreGenesisHeight, TestNetConstants.MaxTarget) - var transactionsPool: IndexedSeq[Transaction] = IndexedSeq.empty[Transaction] override def preStart(): Unit = { context.system.eventStream.subscribe(self, classOf[MinerMiningCommands]) context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) context.system.eventStream.subscribe(self, classOf[BlockchainStatus]) - context.system.scheduler.schedule(5.seconds, 5.seconds)( - influx.foreach(_ ! InfoAboutTransactionsFromMiner(transactionsPool.size)) - ) context.system.scheduler.schedule(5.seconds, 5.seconds) { logger.info(s"data holder: $dataHolder. Context: $context") - dataHolder ! UpdatingTransactionsNumberForApi(transactionsPool.length) dataHolder ! UpdatingMinerStatus(MinerStatus(context.children.nonEmpty && candidateOpt.nonEmpty, candidateOpt)) } } @@ -104,7 +99,6 @@ class Miner(dataHolder: ActorRef, logger.info("Candidate is empty! Producing new candidate!") produceCandidate() } - case TransactionsForMiner(txs) => transactionsPool = transactionsPool ++ txs case StartMining => logger.info("Can't start mining because of chain is not synced!") case DisableMining if context.children.nonEmpty => println(s"Miner -> Disable mining context.children.nonEmpty") @@ -182,10 +176,11 @@ class Miner(dataHolder: ActorRef, } def createCandidate(view: CurrentView[History, UtxoState, EncryWallet], + txsFromMempool: List[Transaction], bestHeaderOpt: Option[Header]): CandidateBlock = { val height: Height = Height @@ (bestHeaderOpt.map(_.height).getOrElse(TestNetConstants.PreGenesisHeight) + 1) val timestamp: Time = timeProvider.estimatedTime - val txsU: IndexedSeq[Transaction] = transactionsPool.filter(view.state.validate(_, timestamp, height).isRight).distinct + val txsU: List[Transaction] = txsFromMempool.filter(view.state.validate(_, timestamp, height).isRight).distinct val filteredTxsWithoutDuplicateInputs = txsU.foldLeft(List.empty[String], IndexedSeq.empty[Transaction]) { case ((usedInputsIds, acc), tx) => if (tx.inputs.forall(input => !usedInputsIds.contains(Algos.encode(input.boxId)))) { @@ -224,7 +219,6 @@ class Miner(dataHolder: ActorRef, logger.info(s"Sending candidate block with ${candidate.transactions.length - 1} transactions " + s"and 1 coinbase for height $height.") - transactionsPool = IndexedSeq.empty[Transaction] candidate } @@ -248,7 +242,7 @@ class Miner(dataHolder: ActorRef, logger.info("Going to calculate last block:") val envelope: CandidateEnvelope = CandidateEnvelope - .fromCandidate(createCandidate(nodeView, bestHeaderOpt)) + .fromCandidate(createCandidate(nodeView, txs, bestHeaderOpt)) envelope } else CandidateEnvelope.empty candidate From 2b1c425f786e0e8a931fd7bbc36a2c42bfd404e7 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 13 Mar 2020 12:30:00 +0300 Subject: [PATCH 085/177] mempool keeps txs till ssm --- .../scala/encry/mpg/IntermediaryMempool.scala | 22 ++++---- src/main/scala/encry/mpg/MemoryPool.scala | 23 ++------ .../scala/encry/mpg/MemoryPoolStorage.scala | 10 ++-- .../scala/encry/nvg/IntermediaryNVH.scala | 53 ++++++++----------- src/main/scala/encry/nvg/NodeViewHolder.scala | 28 +++++----- .../encry/view/mempool/MemoryPoolTests.scala | 3 +- 6 files changed, 59 insertions(+), 80 deletions(-) diff --git a/src/main/scala/encry/mpg/IntermediaryMempool.scala b/src/main/scala/encry/mpg/IntermediaryMempool.scala index 51d4c450a0..c7ed461a0b 100644 --- a/src/main/scala/encry/mpg/IntermediaryMempool.scala +++ b/src/main/scala/encry/mpg/IntermediaryMempool.scala @@ -11,6 +11,7 @@ import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.mpg.MemoryPool._ import encry.mpg.TransactionsValidator.{ InvalidTransaction, ModifiersForValidating } +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier class IntermediaryMempool( settings: EncryAppSettings, @@ -36,16 +37,17 @@ class IntermediaryMempool( override def preStart(): Unit = networkRouter ! RegisterForTxHandling override def receive(): Receive = { - case msg: InvalidTransaction => networkRouter ! msg - case msg: BanPeer => networkRouter ! msg - case msg: RolledBackTransactions => memoryPool ! msg - case msg: ModifiersForValidating => memoryPool ! msg - case msg: DataFromPeer => mempoolProcessor ! msg - case msg: RequestFromLocal => networkRouter ! msg - case msg: ModifierFromNetwork => txValidator ! msg - case msg: TransactionProcessing => mempoolProcessor ! msg - case msg @ SendTransactionsToMiner => memoryPool.forward(msg) - case msg @ FullBlockChainIsSynced => mempoolProcessor ! msg + case msg: InvalidTransaction => networkRouter ! msg + case msg: BanPeer => networkRouter ! msg + case msg: RolledBackTransactions => memoryPool ! msg + case msg: ModifiersForValidating => memoryPool ! msg + case msg: DataFromPeer => mempoolProcessor ! msg + case msg: RequestFromLocal => networkRouter ! msg + case msg: ModifierFromNetwork => txValidator ! msg + case msg: TransactionProcessing => mempoolProcessor ! msg + case msg @ SendTransactionsToMiner => memoryPool.forward(msg) + case msg @ FullBlockChainIsSynced => mempoolProcessor ! msg + case msg @ SemanticallySuccessfulModifier(_) => memoryPool ! msg } } diff --git a/src/main/scala/encry/mpg/MemoryPool.scala b/src/main/scala/encry/mpg/MemoryPool.scala index 5bf935e1db..06d645ab0d 100644 --- a/src/main/scala/encry/mpg/MemoryPool.scala +++ b/src/main/scala/encry/mpg/MemoryPool.scala @@ -2,12 +2,11 @@ package encry.mpg import akka.actor.{ Actor, ActorRef, ActorSystem, Props } import akka.dispatch.{ PriorityGenerator, UnboundedStablePriorityMailbox } -import cats.syntax.either._ import com.google.common.base.Charsets import com.google.common.hash.{ BloomFilter, Funnels } import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging -import encry.mpg.MemoryPool.MemoryPoolStateType.NotProcessingNewTransactions +import encry.mpg.MemoryPool._ import encry.network.Messages.MessageToNetwork.{ RequestFromLocal, ResponseFromLocal } import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, SuccessfulTransaction } @@ -18,7 +17,6 @@ import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestModifiersNetworkMessage } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId -import encry.mpg.MemoryPool._ import scala.collection.IndexedSeq @@ -48,7 +46,6 @@ class MemoryPool( self, RemoveExpiredFromPool ) - context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) } override def receive: Receive = continueProcessing(currentNumberOfProcessedTransactions = 0) @@ -153,14 +150,13 @@ class MemoryPool( } def auxiliaryReceive(state: MemoryPoolStateType): Receive = { - case SemanticallySuccessfulModifier(modifier) if modifier.modifierTypeId == Block.modifierTypeId => + case SemanticallySuccessfulModifier(modifier: Block) => logger.debug( s"MemoryPool got SemanticallySuccessfulModifier with new block while $state." + s"Transit to a transactionsProcessor state." ) - if (state == NotProcessingNewTransactions) - Either.catchNonFatal(context.system.actorSelection("/user/nodeViewSynchronizer") ! StartTransactionsValidation) canProcessTransactions = true + memoryPool = memoryPool.compareWithMod(modifier) context.parent ! TransactionProcessing(canProcessTransactions) context.become(continueProcessing(currentNumberOfProcessedTransactions = 0)) @@ -171,9 +167,7 @@ class MemoryPool( ) case SendTransactionsToMiner => - val (newMemoryPool: MemoryPoolStorage, transactionsForMiner: Seq[Transaction]) = - memoryPool.getTransactionsForMiner - memoryPool = newMemoryPool + val transactionsForMiner: Seq[Transaction] = memoryPool.getTransactionsForMiner mempoolProcessor ! UpdateMempoolReader(MemoryPoolReader.apply(memoryPool)) sender() ! TransactionsForMiner(transactionsForMiner) logger.debug( @@ -185,15 +179,6 @@ class MemoryPool( memoryPool = memoryPool.filter(memoryPool.isExpired) logger.debug(s"MemoryPool got RemoveExpiredFromPool message. After cleaning pool size is: ${memoryPool.size}.") -// case RequestModifiersForTransactions(remote, ids) => -// val modifiersIds: Seq[Transaction] = ids -// .map(Algos.encode) -// .collect { case if memorascyPool.contains(id) => memoryPool.get(id) } -// .flatten -// sender() ! RequestedModifiersForRemote(remote, modifiersIds) -// logger.debug(s"MemoryPool got request modifiers message. Number of requested ids is ${ids.size}." + -// s" Number of sent transactions is ${modifiersIds.size}. Request was from $remote.") - case message => logger.debug(s"MemoryPool got unhandled message $message.") } diff --git a/src/main/scala/encry/mpg/MemoryPoolStorage.scala b/src/main/scala/encry/mpg/MemoryPoolStorage.scala index bcf425c729..5c561f879a 100644 --- a/src/main/scala/encry/mpg/MemoryPoolStorage.scala +++ b/src/main/scala/encry/mpg/MemoryPoolStorage.scala @@ -2,6 +2,7 @@ package encry.mpg import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider +import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.utils.Algos @@ -58,8 +59,8 @@ final case class MemoryPoolStorage private ( } } - def getTransactionsForMiner: (MemoryPoolStorage, Seq[Transaction]) = { - val (transactionsForMiner: Seq[Transaction], _) = transactions.toIndexedSeq.sortBy { case (_, tx) => tx.fee } + def getTransactionsForMiner: Seq[Transaction] = + transactions.toIndexedSeq.sortBy { case (_, tx) => tx.fee } .foldLeft(Seq.empty[Transaction], Set.empty[String]) { case ((validated, inputs), (_, transaction)) => val transactionInputsIds: Set[String] = transaction.inputs.map(input => Algos.encode(input.boxId)).toSet @@ -69,8 +70,9 @@ final case class MemoryPoolStorage private ( (validated :+ transaction, inputs ++ transactionInputsIds) else (validated, inputs) } - (removeSeveral(transactionsForMiner.map(_.encodedId)), transactionsForMiner) - } + ._1 + + def compareWithMod(block: Block): MemoryPoolStorage = removeSeveral(block.payload.txs.map(_.encodedId)) def isValid: Transaction => Boolean = tx => tx.semanticValidity.isSuccess && !contains(tx.encodedId) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index b6b6b65a1f..4bcf66def2 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -5,7 +5,8 @@ import akka.routing.BalancingPool import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo -import encry.local.miner.Miner.{ DisableMining, StartMining } +import encry.local.miner.Miner.{ DisableMining, EnableMining, StartMining } +import encry.mpg.MemoryPool._ import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } @@ -13,37 +14,20 @@ import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandlin import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation } +import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier +import encry.nvg.NodeViewHolder._ import encry.nvg.fast.sync.SnapshotProcessor -import encry.nvg.NodeViewHolder.{ - GetDataFromCurrentView, - RollbackFailed, - RollbackSucceed, - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification, - UpdateHistoryReader -} import encry.nvg.fast.sync.SnapshotProcessor.{ FastSyncDone, HeaderChainIsSynced, RequiredManifestHeightAndId, TreeChunks } -import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader -import encry.mpg.MemoryPool._ -import org.encryfoundation.common.network.BasicMessagesRepo.{ - InvNetworkMessage, - RequestChunkMessage, - RequestManifestMessage, - RequestModifiersNetworkMessage, - ResponseChunkMessage, - ResponseManifestMessage, - SyncInfoNetworkMessage -} +import org.encryfoundation.common.network.BasicMessagesRepo._ import org.encryfoundation.common.utils.Algos class IntermediaryNVH( @@ -124,18 +108,24 @@ class IntermediaryNVH( case msg @ TreeChunks(_, _) => //+ to fast sync case msg @ FastSyncDone => case msg @ HeaderChainIsSynced => - case msg @ FullBlockChainIsSynced => mempoolRef ! msg - case msg @ RolledBackTransactions(_) => mempoolRef ! msg - case msg @ StartMining => //+ to miner - case msg @ BlockAndHeaderInfo(_, _) => dataHolderRef ! msg - case msg: UpdateHistoryReader => dataHolderRef ! msg - case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) - case msg @ GetDataFromCurrentView(_) => nodeViewHolder.forward(msg) - case msg @ RollbackSucceed(_) => - case msg @ RollbackFailed(_) => + case msg @ DisableMining => context.system.eventStream.publish(msg) + case msg @ EnableMining => context.system.eventStream.publish(msg) + case msg @ FullBlockChainIsSynced => + context.system.eventStream.publish(msg) + mempoolRef ! msg + case msg @ RolledBackTransactions(_) => mempoolRef ! msg + case msg @ StartMining => //+ to miner + case msg @ BlockAndHeaderInfo(_, _) => dataHolderRef ! msg + case msg: UpdateHistoryReader => dataHolderRef ! msg + case msg: StatsSenderMessage => influxRef.foreach(_ ! msg) + case msg @ GetDataFromCurrentView(_) => nodeViewHolder.forward(msg) + case msg @ RollbackSucceed(_) => + case msg @ RollbackFailed(_) => case msg @ SemanticallySuccessfulModifier(_) => + context.system.eventStream.publish(msg) intermediaryNetwork ! msg networkMessagesProcessor ! msg + mempoolRef ! msg case msg @ SemanticallyFailedModification(_, _) => intermediaryNetwork ! msg } } @@ -148,5 +138,6 @@ object IntermediaryNVH { influxRef: Option[ActorRef], mempoolRef: ActorRef, dataHolderRef: ActorRef - ): Props = Props(new IntermediaryNVH(settings, intermediaryNetwork, timeProvider, influxRef, mempoolRef, dataHolderRef)) + ): Props = + Props(new IntermediaryNVH(settings, intermediaryNetwork, timeProvider, influxRef, mempoolRef, dataHolderRef)) } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 17b6488ca8..7d1d758d3e 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -2,19 +2,19 @@ package encry.nvg import java.io.File -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import akka.pattern._ import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{DisableMining, EnableMining, StartMining} +import encry.local.miner.Miner.{ DisableMining, EnableMining, StartMining } import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.ReceivableMessages.{CreateAccountManagerFromSeed, LocallyGeneratedModifier} +import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } import encry.nvg.NodeViewHolder._ import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId import encry.nvg.fast.sync.SnapshotProcessor._ @@ -26,23 +26,23 @@ import encry.view.NodeViewErrors.ModifierApplyError import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.NodeViewHolder.CurrentView import encry.view.history.storage.HistoryStorage -import encry.view.history.{History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader} +import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } import encry.mpg.MemoryPool._ import encry.view.state.UtxoState import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet import io.iohk.iodb.ByteArrayWrapper import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.modifiers.{PersistentModifier, PersistentNodeViewModifier} +import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } -import scala.collection.{IndexedSeq, Seq, mutable} +import scala.collection.{ mutable, IndexedSeq, Seq } import scala.concurrent.Future import scala.concurrent.duration._ -import scala.util.{Failure, Success, Try} +import scala.util.{ Failure, Success, Try } class NodeViewHolder( settings: EncryAppSettings, @@ -224,7 +224,7 @@ class NodeViewHolder( val blockAtHeight: Block = history.getBlockByHeader(headerAtHeight).get blocks :+ blockAtHeight } - context.system.eventStream.publish(DisableMining) + context.parent ! DisableMining state.rollbackTo(branchPoint, additionalBlocks) -> trimChainSuffix(suffixApplied, ModifierId !@@ branchPoint) } else Success(state) -> IndexedSeq.empty @@ -233,7 +233,8 @@ class NodeViewHolder( stateToApplyTry match { case Success(stateToApply: UtxoState) => context.parent ! RollbackSucceed(branchingPointOpt) - if (settings.node.mining && nodeView.history.isFullChainSynced && progressInfo.chainSwitchingNeeded) context.system.eventStream.publish(EnableMining) + if (settings.node.mining && nodeView.history.isFullChainSynced && progressInfo.chainSwitchingNeeded) + context.parent ! EnableMining val u0: UpdateInformation = UpdateInformation(history, stateToApply, none, none, suffixTrimmed) val uf: UpdateInformation = progressInfo.toApply.foldLeft(u0) { case (u: UpdateInformation, modToApply: PersistentModifier) => @@ -283,7 +284,7 @@ class NodeViewHolder( } if (settings.node.mining && progressInfo.chainSwitchingNeeded) context.parent ! StartMining - context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) + context.parent ! SemanticallySuccessfulModifier(modToApply) if (newHis.getBestHeaderId.exists( bestHeaderId => newHis.getBestBlockId.exists(bId => ByteArrayWrapper(bId) == ByteArrayWrapper(bestHeaderId)) @@ -355,14 +356,13 @@ class NodeViewHolder( logger.info(s"BlockChain is synced on nvh at the height ${newHistory.getBestHeaderHeight}.") ModifiersCache.setChainSynced() context.parent ! FullBlockChainIsSynced - context.system.eventStream.publish(FullBlockChainIsSynced) } updateNodeView(newHistory.some, newState.some, nodeView.wallet.some) } else { logger.info(s"Progress info is empty.") context.parent ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height) if (!isLocallyGenerated) requestDownloads(progressInfo, modifier.id.some) - context.system.eventStream.publish(SemanticallySuccessfulModifier(modifier)) + context.parent ! SemanticallySuccessfulModifier(modifier) updateNodeView(updatedHistory = historyBeforeStUpdate.some) } case Left(e: Throwable) => diff --git a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala index 76fc47ff40..a28c915c3c 100644 --- a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala +++ b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala @@ -54,8 +54,7 @@ class MemoryPoolTests val mempool = MemoryPoolStorage.empty(testNetSettings, timeProvider) val transactions = (0 until 10).map(k => coinbaseAt(k)) val (newMempool, _) = mempool.validateTransactions(transactions) - val (uPool, txs) = newMempool.getTransactionsForMiner - uPool.size shouldBe 0 + val txs = newMempool.getTransactionsForMiner txs.map(_.encodedId).forall(transactions.map(_.encodedId).contains) shouldBe true transactions.map(_.encodedId).forall(txs.map(_.encodedId).contains) shouldBe true } From 96e90f9e903b6036d0113be73df0dce44d3db55b Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Fri, 13 Mar 2020 15:33:07 +0300 Subject: [PATCH 086/177] request for headers fixed --- src/main/scala/encry/nvg/NetworkMessagesProcessor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 8415387d83..200f5a008c 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -99,7 +99,7 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St sender() ! RequestFromLocal(remote.some, invData._1, ids.toList) logger.info(s"Time of processing inv message is: ${(System.currentTimeMillis() - startTime) / 1000}s.") - case RequestModifiersNetworkMessage((typeId, requestedIds)) if typeId == Payload.modifierTypeId => + case RequestModifiersNetworkMessage((typeId, requestedIds)) => val modifiersFromCache: Map[ModifierId, Array[Byte]] = requestedIds .flatMap( (id: ModifierId) => From 96b6c170986573df2e8c76c53a29e82a7f1d99ea Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 16 Mar 2020 11:39:33 +0300 Subject: [PATCH 087/177] change messageBuilder naming --- src/main/scala/encry/network/NetworkRouter.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 419ab5bab3..261c6dc881 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -84,7 +84,7 @@ class NetworkRouter(settings: NetworkSettings, case msg: MessageToNetwork => context.actorOf( MessageBuilder.props(peersKeeper, deliveryManager), - s"messageBuilder${Random.nextInt()}" + s"messageBuilder${System.currentTimeMillis()}" ) ! msg } From b3eae66aff0d3ae558b1da795bc6e4a27d73eda5 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 16 Mar 2020 11:58:53 +0300 Subject: [PATCH 088/177] ec fix --- src/main/scala/encry/network/NetworkRouter.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/network/NetworkRouter.scala b/src/main/scala/encry/network/NetworkRouter.scala index 261c6dc881..2fd32463a6 100644 --- a/src/main/scala/encry/network/NetworkRouter.scala +++ b/src/main/scala/encry/network/NetworkRouter.scala @@ -23,10 +23,12 @@ import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, NetworkMessage} +import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} +import scorex.utils.Random import scala.concurrent.duration._ -import scala.util.Random +import scala.util.{Random => SRand} class NetworkRouter(settings: NetworkSettings, blackListSettings: BlackListSettings, @@ -84,7 +86,7 @@ class NetworkRouter(settings: NetworkSettings, case msg: MessageToNetwork => context.actorOf( MessageBuilder.props(peersKeeper, deliveryManager), - s"messageBuilder${System.currentTimeMillis()}" + s"messageBuilder${Algos.encode(Random.randomBytes()) ++ SRand.nextLong().toString}" ) ! msg } From 3a97b20ed4684249c316fc7d6b2328419d7588db Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 16 Mar 2020 12:09:59 +0300 Subject: [PATCH 089/177] todo added --- src/main/scala/encry/nvg/NodeViewHolder.scala | 2 +- .../scala/encry/view/history/HistoryPayloadsProcessor.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 7d1d758d3e..7ecff66a98 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -123,7 +123,7 @@ class NodeViewHolder( FileUtils.deleteDirectory(new File(s"${settings.directory}/keysTmp")) FileUtils.deleteDirectory(new File(s"${settings.directory}/walletTmp")) val newHistory = new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { - override val settings: EncryAppSettings = settings + override val settings: EncryAppSettings = settings //todo incorrect override var isFullChainSynced: Boolean = settings.node.offlineGeneration override val timeProvider: NetworkTimeProvider = ntp override val historyStorage: HistoryStorage = nodeView.history.historyStorage diff --git a/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala b/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala index ecdc4be798..071e547cf2 100644 --- a/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala +++ b/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala @@ -64,7 +64,7 @@ trait HistoryPayloadsProcessor extends HistoryApi { (fullBlock.header.height > bestHeaderHeight) || ( (fullBlock.header.height == bestHeaderHeight) && scoreOf(fullBlock.id) - .flatMap(fbScore => getBestHeaderId.flatMap(id => scoreOf(id).map(_ < fbScore))) + .flatMap(fbScore => getBestHeaderId.flatMap(scoreOf(_).map(_ < fbScore))) .getOrElse(false) ) val updatedHeadersAtHeightIds = From fb00a55b0baaa29dd8fc39dc5f6a73840663012d Mon Sep 17 00:00:00 2001 From: Lior Date: Mon, 16 Mar 2020 14:20:03 +0300 Subject: [PATCH 090/177] test removed --- .../encry/view/mempool/MemoryPoolTests.scala | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala index a28c915c3c..04e5b67301 100644 --- a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala +++ b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala @@ -59,19 +59,4 @@ class MemoryPoolTests transactions.map(_.encodedId).forall(txs.map(_.encodedId).contains) shouldBe true } } - "Mempool actor" should { - "send transactions to miner" in { - val miner1 = TestProbe() - val mempool1: TestActorRef[MemoryPool] = - TestActorRef[MemoryPool](MemoryPool.props(testNetSettings, timeProvider, Some(TestProbe().ref), miner1.ref)) - val transactions1 = (0 until 4).map { k => - val a = coinbaseAt(k) - a - } - transactions1.foreach(mempool1 ! NewTransaction(_)) - mempool1.underlyingActor.memoryPool.size shouldBe 4 - logger.info(s"generated: ${transactions1.map(_.encodedId)}") - miner1.expectMsg(20.seconds, TransactionsForMiner(transactions1)) - } - } } From 27e624072ad07e6104bb0c720d4530c3a2dc1c2a Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 16 Mar 2020 14:41:31 +0300 Subject: [PATCH 091/177] fixed modifiers propagation --- src/main/scala/encry/nvg/NetworkMessagesProcessor.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 200f5a008c..8577e16d38 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -1,9 +1,9 @@ package encry.nvg import akka.actor.{ Actor, Cancellable, Props } +import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } -import cats.syntax.option._ import encry.network.DeliveryManager.CheckPayloadsToDownload import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } import encry.network.ModifiersToNetworkUtils.toProto @@ -17,7 +17,6 @@ import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, - ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage } @@ -108,7 +107,7 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St .map(id -> _) ) .toMap - if (modifiersFromCache.nonEmpty) context.parent ! ModifiersNetworkMessage(typeId -> modifiersFromCache) + if (modifiersFromCache.nonEmpty) context.parent ! ResponseFromLocal(remote, typeId, modifiersFromCache) val unrequestedModifiers: List[ModifierId] = requestedIds.filterNot(modifiersFromCache.contains).toList typeId match { From 944ec15416e3314208f6eb0240e95390e7511b8c Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 16 Mar 2020 14:42:24 +0300 Subject: [PATCH 092/177] fixed modifiers propagation --- src/main/scala/encry/nvg/NetworkMessagesProcessor.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 8577e16d38..13761962ca 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -23,7 +23,6 @@ import org.encryfoundation.common.network.BasicMessagesRepo.{ import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId - import scala.concurrent.duration._ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { From fc96df1119d11ac531356121e2e2b56ed6ec1ed2 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 16 Mar 2020 16:02:39 +0300 Subject: [PATCH 093/177] add tests --- src/main/scala/encry/network/DM.scala | 2 +- .../DeliveryManagerTests/DMUtils.scala | 78 +-- .../DeliveryManagerPriorityTests.scala | 9 +- .../DeliveryManagerRequestModifiesSpec.scala | 587 ++++++++---------- 4 files changed, 313 insertions(+), 363 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 841b48cf5e..52adcc35b8 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -40,7 +40,7 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging receivedModifier += toKey(id) } else logger.info(s"Receive spam. ModId: ${Algos.encode(id)}!") } - case RequestSent(peer, modTypeId, modId) if !expectedModifiers.contains(toKey(modId))=> + case RequestSent(peer, modTypeId, modId) if !(expectedModifiers.contains(toKey(modId)) || receivedModifier.contains(toKey(modId))) => expectedModifiers += toKey(modId) context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)( self ! AwaitingRequest(peer, modTypeId, modId, 1) diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala b/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala index 03d54c59cb..0f62964c0d 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala @@ -1,11 +1,12 @@ package encry.network.DeliveryManagerTests import java.net.InetSocketAddress + import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestProbe} import encry.local.miner.Miner.{DisableMining, StartMining} import encry.modifiers.InstanceFactory -import encry.network.DeliveryManager +import encry.network.{DM, DeliveryManager} import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.NodeViewSynchronizer.ReceivableMessages.UpdatedHistory import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} @@ -14,44 +15,43 @@ import encry.view.history.History import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.network.BasicMessagesRepo.Handshake import org.encryfoundation.common.utils.TaggedTypes.ModifierId + import scala.collection.mutable import scala.collection.mutable.WrappedArray -//object DMUtils extends InstanceFactory { -// -// def initialiseDeliveryManager(isBlockChainSynced: Boolean, -// isMining: Boolean, -// settings: EncryAppSettings) -// (implicit actorSystem: ActorSystem): (TestActorRef[DeliveryManager], History) = { -// val history: History = generateDummyHistory(settings) -// val deliveryManager: TestActorRef[DeliveryManager] = -// TestActorRef[DeliveryManager](DeliveryManager -// .props(None, TestProbe().ref, TestProbe().ref, TestProbe().ref, TestProbe().ref, TestProbe().ref, settings)) -// deliveryManager ! UpdatedHistory(history) -// if (isMining) deliveryManager ! StartMining -// else deliveryManager ! DisableMining -// if (isBlockChainSynced) deliveryManager ! FullBlockChainIsSynced -// (deliveryManager, history) -// } -// -// def generateBlocks(qty: Int, history: History): (History, List[Block]) = -// (0 until qty).foldLeft(history, List.empty[Block]) { -// case ((prevHistory, blocks), _) => -// val block: Block = generateNextBlock(prevHistory) -// prevHistory.append(block.header) -// prevHistory.append(block.payload) -// val a = prevHistory.reportModifierIsValid(block) -// (a, blocks :+ block) -// } -// -// def toKey(id: ModifierId): WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) -// -// def createPeer(port: Int, -// host: String, -// settings: EncryAppSettings)(implicit system: ActorSystem): (InetSocketAddress, ConnectedPeer) = { -// val address = new InetSocketAddress(host, port) -// val peer: ConnectedPeer = ConnectedPeer(address, TestProbe().ref, Incoming, -// Handshake(protocolToBytes(settings.network.appVersion), host, Some(address), System.currentTimeMillis())) -// (address, peer) -// } -//} \ No newline at end of file +object DMUtils extends InstanceFactory { + + def initialiseDeliveryManager(isBlockChainSynced: Boolean, + isMining: Boolean, + settings: EncryAppSettings) + (implicit actorSystem: ActorSystem): (TestActorRef[DM], History) = { + val history: History = generateDummyHistory(settings) + val deliveryManager: TestActorRef[DM] = TestActorRef[DM](DM.props(settings.network)) + deliveryManager ! UpdatedHistory(history) + if (isMining) deliveryManager ! StartMining + else deliveryManager ! DisableMining + if (isBlockChainSynced) deliveryManager ! FullBlockChainIsSynced + (deliveryManager, history) + } + + def generateBlocks(qty: Int, history: History): (History, List[Block]) = + (0 until qty).foldLeft(history, List.empty[Block]) { + case ((prevHistory, blocks), _) => + val block: Block = generateNextBlock(prevHistory) + prevHistory.append(block.header) + prevHistory.append(block.payload) + val a = prevHistory.reportModifierIsValid(block) + (a, blocks :+ block) + } + + def toKey(id: ModifierId): WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) + + def createPeer(port: Int, + host: String, + settings: EncryAppSettings)(implicit system: ActorSystem): (InetSocketAddress, ConnectedPeer) = { + val address = new InetSocketAddress(host, port) + val peer: ConnectedPeer = ConnectedPeer(address, TestProbe().ref, Incoming, + Handshake(protocolToBytes(settings.network.appVersion), host, Some(address), System.currentTimeMillis())) + (address, peer) + } +} \ No newline at end of file diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala index fb358c98b8..a2a0752e80 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerPriorityTests.scala @@ -2,12 +2,14 @@ package encry.network.DeliveryManagerTests import java.net.InetSocketAddress -//import encry.network.DeliveryManagerTests.DMUtils.{createPeer, generateBlocks, initialiseDeliveryManager} +import encry.network.DM +import encry.network.DeliveryManagerTests.DMUtils.{createPeer, generateBlocks, initialiseDeliveryManager} import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestKit} import encry.consensus.HistoryConsensus import encry.consensus.HistoryConsensus.{Equal, Older, Younger} import encry.modifiers.InstanceFactory +import encry.network.DM.RequestStatus import encry.network.DeliveryManager import encry.network.NetworkController.ReceivableMessages.DataFromPeer //import encry.network.NodeViewSynchronizer.ReceivableMessages.RequestFromLocal @@ -32,7 +34,7 @@ import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecL // // override def afterAll: Unit = TestKit.shutdownActorSystem(system) // -// def initialiseState: (TestActorRef[DeliveryManager], ConnectedPeer, ConnectedPeer, ConnectedPeer, +// def initialiseState: (TestActorRef[DM], ConnectedPeer, ConnectedPeer, ConnectedPeer, // ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, ConnectedPeer, // List[Block], List[ModifierId]) = { // val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = true, isMining = true, testNetSettings) @@ -66,8 +68,7 @@ import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecL // val (deliveryManager, cp1, _, _, _, _, _, _, _, _, _, headersIds) = initialiseState // val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = // Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) +// deliveryManager ! RequestStatus(cp1, Header.modifierTypeId, headersIds) // val (result, _) = deliveryManager.underlyingActor.priorityCalculator.accumulatePeersStatistic // assert(result.contains(cp1.socketAddress)) // assert(result(cp1.socketAddress) == BadNode) diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala index 8a12695db0..f0e6f1215e 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala @@ -1,319 +1,268 @@ -//package encry.network.DeliveryManagerTests -// -//import java.net.InetSocketAddress -// -//import akka.actor.ActorSystem -//import akka.testkit.{TestActorRef, TestKit, TestProbe} -//import encry.consensus.HistoryConsensus -//import encry.consensus.HistoryConsensus.{Fork, Older, Younger} -//import encry.modifiers.InstanceFactory -//import encry.network.DeliveryManager -//import encry.network.NetworkController.ReceivableMessages.DataFromPeer -//import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} -//import encry.settings.TestNetSettings -//import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -//import encry.network.DeliveryManagerTests.DMUtils._ -//import encry.network.Messages.MessageToNetwork.RequestFromLocal -//import encry.network.PeersKeeper.UpdatedPeersCollection -//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus -//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ -//import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} -//import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -//import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} -//import org.encryfoundation.common.network.SyncInfo -//import org.encryfoundation.common.utils.Algos -//import org.encryfoundation.common.utils.TaggedTypes.ModifierId -// -//import scala.collection.mutable.WrappedArray -// -//class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll -// with Matchers -// with InstanceFactory -// with OneInstancePerTest -// with TestNetSettings { -// -// implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") -// -// override def afterAll(): Unit = TestKit.shutdownActorSystem(system) -// -// def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], -// ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { -// val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) -// val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) -// val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) -// val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) -// val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 -// val headersIds: List[ModifierId] = blocks.map(_.header.id) -// val headersAsKey = headersIds.map(toKey) -// (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) -// } -// -// "RequestModifies" should { -// "handle uniq modifiers from RequestFromLocal message correctly" in { -// val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.size == headersIds.size) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.forall(elem => headersAsKey.contains(elem))) -// deliveryManager.stop() -// } -// "not handle repeating modifiers from RequestFromLocal message" in { -// val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.size == headersIds.size) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.forall(elem => headersAsKey.contains(elem))) -// deliveryManager.stop() -// } -// "Delivery Manager should handle received modifier which were requested correctly" in { -// val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( -// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.isEmpty) -// assert(deliveryManager.underlyingActor.receivedModifiers.size == blocks.size) -// assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) -// deliveryManager.stop() -// } -// "Delivery manager should not handle repeating modifiers" in { -// val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( -// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( -// Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1) -// assert(deliveryManager.underlyingActor.receivedModifiers.size == headersIds.size) -// assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) -// deliveryManager.stop() -// } -// "handle priority request for payload correctly" in { -// val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, headersIds) -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage( -// Header.modifierTypeId -> blocks.map(k => k.header.id -> Array.emptyByteArray).toMap), cp1) -// headersIds.foreach(id => -// deliveryManager ! DownloadRequest(Payload.modifierTypeId, blocks.find(block => -// block.id.sameElements(id)).get.payload.id)) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .size == blocks.size) -// deliveryManager.stop() -// } -// "choose correct peer in priority request" in { -// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val address2 = new InetSocketAddress("123.123.123.124", 9001) -// val handler2: TestProbe = TestProbe() -// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.124", Some(address2), System.currentTimeMillis())) -// -// val address3 = new InetSocketAddress("123.123.123.125", 9001) -// val handler3: TestProbe = TestProbe() -// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.125", Some(address3), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map( -// address1 -> (cp1, Older, InitialPriority), -// address2 -> (cp2, Older, InitialPriority), -// address3 -> (cp3, Older, InitialPriority) -// ) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val header: Header = blocks.head.header -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) -// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) -// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) -// -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1) -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2) -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3) -// -// deliveryManager ! DownloadRequest(Payload.modifierTypeId, header.payloadId) -// -// handler1.expectMsgAnyOf( -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), -// RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), -// SyncInfoNetworkMessage(SyncInfo(List())) -// ) -// -// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) -// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) -// deliveryManager.stop() -// } -// "not ask modifiers while block chain is not synced from Younger nodes" in { -// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) -// -// val address2 = new InetSocketAddress("123.123.123.124", 9001) -// val handler2: TestProbe = TestProbe() -// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.124", Some(address2), System.currentTimeMillis())) -// -// val address3 = new InetSocketAddress("123.123.123.125", 9001) -// val handler3: TestProbe = TestProbe() -// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.125", Some(address3), System.currentTimeMillis())) -// -// val updatedPeersCollection = -// Map( -// address2 -> (cp2, Younger, InitialPriority), -// address3 -> (cp3, Fork, InitialPriority) -// ) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// -// val header: Header = blocks.head.header -// -// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) -// deliveryManager ! RequestFromLocal(cp3, Header.modifierTypeId, Seq(header.id)) -// -// handler2.expectNoMsg() -// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) -// deliveryManager.stop() -// } -// "not ask modifiers from peer which is not contained in status tracker" in { -// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val address2 = new InetSocketAddress("123.123.123.124", 9001) -// val handler2: TestProbe = TestProbe() -// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.124", Some(address2), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address2 -> (cp2, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val header: Header = blocks.head.header -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) -// deliveryManager ! RequestFromLocal(cp2, Header.modifierTypeId, Seq(header.id)) -// -// handler1.expectNoMsg() -// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) -// deliveryManager.stop() -// } -// "not ask transactions while block chain is not synced" in { -// val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) -// val txs: Seq[Transaction] = genInvalidPaymentTxs(1) -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) -// -// handler1.expectNoMsg() -// deliveryManager.stop() -// } -// "not ask transaction while node is not mining" in { -// val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) -// val txs: Seq[Transaction] = genInvalidPaymentTxs(1) -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, txs.map(_.id)) -// -// handler1.expectNoMsg() -// deliveryManager.stop() -// } -// "not re-ask modifiers which already have been received" in { -// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val header: Header = blocks.head.header -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) -// -// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(header).toByteArray -// -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> headerBytes)), cp1) -// -// handler1.expectMsgAllOf( -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)) -// ) -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header.id)) -// -// handler1.expectNoMsg() -// -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty) -// .keys.isEmpty) -// assert(deliveryManager.underlyingActor.receivedModifiers.size == 1) -// assert(deliveryManager.underlyingActor.receivedModifiers.contains(toKey(header.id))) -// deliveryManager.stop() -// } -// } -//} \ No newline at end of file +package encry.network.DeliveryManagerTests + +import java.net.InetSocketAddress + +import akka.actor.ActorSystem +import akka.testkit.{TestActorRef, TestKit, TestProbe} +import encry.consensus.HistoryConsensus +import encry.consensus.HistoryConsensus.{Fork, Older, Younger} +import encry.modifiers.InstanceFactory +import encry.network.DM.RequestSent +import encry.network.{DM, DeliveryManager} +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} +import encry.settings.TestNetSettings +import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +import encry.network.DeliveryManagerTests.DMUtils._ +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.PeersKeeper.UpdatedPeersCollection +import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus +import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus._ +import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} +import org.encryfoundation.common.network.SyncInfo +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.ModifierId + +import scala.collection.mutable.WrappedArray + +class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll + with Matchers + with InstanceFactory + with OneInstancePerTest + with TestNetSettings { + + implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") + + override def afterAll(): Unit = TestKit.shutdownActorSystem(system) + + def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DM], + ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { + val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) + val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) + val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) + val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) + val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 + val headersIds: List[ModifierId] = blocks.map(_.header.id) + val headersAsKey = headersIds.map(toKey) + (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) + } + + "RequestModifies" should { + "handle uniq modifiers from RequestFromLocal message correctly" in { + val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + headersIds.foreach(id => deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, id)) + assert(deliveryManager.underlyingActor.expectedModifiers.size == headersIds.size) + assert(deliveryManager.underlyingActor.expectedModifiers.forall(elem => headersAsKey.contains(elem))) + deliveryManager.stop() + } + "not handle repeating modifiers from RequestFromLocal message" in { + val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + headersIds.foreach(id => deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, id)) + headersIds.foreach(id => deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, id)) + assert(deliveryManager.underlyingActor.expectedModifiers.size == headersIds.size) + assert(deliveryManager.underlyingActor.expectedModifiers.forall(elem => headersAsKey.contains(elem))) + deliveryManager.stop() + } + "Delivery Manager should handle received modifier which were requested correctly" in { + val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + headersIds.foreach(id => deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, id)) + val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray + deliveryManager ! DataFromPeer(ModifiersNetworkMessage( + Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1.socketAddress) + assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) + assert(deliveryManager.underlyingActor.receivedModifier.size == blocks.size) + assert(deliveryManager.underlyingActor.receivedModifier.forall(elem => headersAsKey.contains(elem))) + deliveryManager.stop() + } + "Delivery manager should not handle repeating modifiers" in { + val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + headersIds.foreach(id => deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, id)) + val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray + deliveryManager ! DataFromPeer(ModifiersNetworkMessage( + Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1.socketAddress) + deliveryManager ! DataFromPeer(ModifiersNetworkMessage( + Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1.socketAddress) + assert(deliveryManager.underlyingActor.receivedModifier.size == headersIds.size) + assert(deliveryManager.underlyingActor.receivedModifier.forall(elem => headersAsKey.contains(elem))) + deliveryManager.stop() + } + "handle priority request for payload correctly" in { + val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) + + headersIds.foreach(id => deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, id)) + deliveryManager ! DataFromPeer(ModifiersNetworkMessage( + Header.modifierTypeId -> blocks.map(k => k.header.id -> Array.emptyByteArray).toMap), cp1.socketAddress) + headersIds.foreach(id => + deliveryManager ! RequestSent(cp1.socketAddress, Payload.modifierTypeId, blocks.find(block => + block.id.sameElements(id)).get.payload.id)) + assert(deliveryManager.underlyingActor.expectedModifiers.size == blocks.size) + deliveryManager.stop() + } + "choose correct peer in priority request" in { + val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val address2 = new InetSocketAddress("123.123.123.124", 9001) + val handler2: TestProbe = TestProbe() + val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.124", Some(address2), System.currentTimeMillis())) + + val address3 = new InetSocketAddress("123.123.123.125", 9001) + val handler3: TestProbe = TestProbe() + val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.125", Some(address3), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map( + address1 -> (cp1, Older, InitialPriority), + address2 -> (cp2, Older, InitialPriority), + address3 -> (cp3, Older, InitialPriority) + ) + + + + val header: Header = blocks.head.header + + deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) + deliveryManager ! RequestSent(cp2.socketAddress, Header.modifierTypeId, header.id) + deliveryManager ! RequestSent(cp3.socketAddress, Header.modifierTypeId, header.id) + + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1.socketAddress) + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2.socketAddress) + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3.socketAddress) + + deliveryManager ! RequestSent(cp1.socketAddress, Payload.modifierTypeId, header.payloadId) + + handler1.expectMsgAnyOf( + RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), + RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), + SyncInfoNetworkMessage(SyncInfo(List())) + ) + + handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) + handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) + deliveryManager.stop() + } + "not ask modifiers from peer which is not contained in status tracker" in { + val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val address2 = new InetSocketAddress("123.123.123.124", 9001) + val handler2: TestProbe = TestProbe() + val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.124", Some(address2), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address2 -> (cp2, Older, InitialPriority)) + + + + val header: Header = blocks.head.header + + deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) + deliveryManager ! RequestSent(cp2.socketAddress, Header.modifierTypeId, header.id) + + handler1.expectNoMsg() + handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) + deliveryManager.stop() + } + "not ask transactions while block chain is not synced" in { + val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) + val txs: Seq[Transaction] = genInvalidPaymentTxs(1) + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address1 -> (cp1, Older, InitialPriority)) + + + + txs.foreach(tx => deliveryManager ! RequestSent(cp1.socketAddress, Transaction.modifierTypeId, tx.id)) + + handler1.expectNoMsg() + deliveryManager.stop() + } + "not ask transaction while node is not mining" in { + val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) + val txs: Seq[Transaction] = genInvalidPaymentTxs(1) + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address1 -> (cp1, Older, InitialPriority)) + + + + txs.foreach(tx => deliveryManager ! RequestSent(cp1.socketAddress, Transaction.modifierTypeId, tx.id)) + + handler1.expectNoMsg() + deliveryManager.stop() + } + "not re-ask modifiers which already have been received" in { + val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address1 -> (cp1, Older, InitialPriority)) + + + + val header: Header = blocks.head.header + + deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) + + val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(header).toByteArray + + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> headerBytes)), cp1.socketAddress) + + deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) + + assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) + assert(deliveryManager.underlyingActor.receivedModifier.size == 1) + assert(deliveryManager.underlyingActor.receivedModifier.contains(toKey(header.id))) + deliveryManager.stop() + } + } +} \ No newline at end of file From 35e32b5a7a5ae8fd3d59806d968d4a46e8eea45b Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 16 Mar 2020 16:21:03 +0300 Subject: [PATCH 094/177] more dm tests --- ...DeliveryManagerReRequestModifiesSpec.scala | 285 ++++++++---------- .../DeliveryManagerRequestModifiesSpec.scala | 12 +- 2 files changed, 131 insertions(+), 166 deletions(-) diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala index 106005b9ff..e29d0daafd 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala @@ -1,55 +1,59 @@ -//package encry.network.DeliveryManagerTests -// -//import java.net.InetSocketAddress -//import akka.actor.ActorSystem -//import akka.testkit.{TestActorRef, TestProbe} -//import encry.consensus.HistoryConsensus -//import encry.consensus.HistoryConsensus.Older -//import encry.modifiers.InstanceFactory -//import encry.network.DeliveryManager -////import encry.network.DeliveryManagerTests.DMUtils._ -//import encry.network.NetworkController.ReceivableMessages.DataFromPeer -//import encry.network.NodeViewSynchronizer.ReceivableMessages._ -//import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} -//import encry.network.PeersKeeper.UpdatedPeersCollection -//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus.InitialPriority -//import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus -//import encry.settings.TestNetSettings -//import encry.view.history.History -//import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer} -//import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -//import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage} -//import org.encryfoundation.common.utils.TaggedTypes.ModifierId -//import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -//import scala.concurrent.duration._ -//import scala.collection.mutable.WrappedArray -// -//class DeliveryManagerReRequestModifiesSpec extends WordSpecLike -// with BeforeAndAfterAll -// with Matchers -// with InstanceFactory -// with OneInstancePerTest -// with TestNetSettings { -// -// implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") -// -// override def afterAll(): Unit = system.terminate() -// -// def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DeliveryManager], -// ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte], History) = { -// val (deliveryManager, history) = -// initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) -// val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) -// val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) -// val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) -// val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 -// val headersIds: List[ModifierId] = blocks.map(_.header.id) -// val headersAsKey = headersIds.map(toKey) -// (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey, history) -// } -// -// "ReRequestModifies" should { -// "re-ask necessary modifier several times (number of attempts from testNetSettings) and remove modifier from " + +package encry.network.DeliveryManagerTests + +import java.net.InetSocketAddress + +import akka.actor.ActorSystem +import akka.testkit.{TestActorRef, TestProbe} +import encry.consensus.HistoryConsensus +import encry.consensus.HistoryConsensus.Older +import encry.modifiers.InstanceFactory +import encry.network.DM.RequestSent +import encry.network.{DM, DeliveryManager} +import encry.network.DeliveryManagerTests.DMUtils._ +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.NodeViewSynchronizer.ReceivableMessages._ +import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} +import encry.network.PeersKeeper.UpdatedPeersCollection +import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus.InitialPriority +import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.settings.TestNetSettings +import encry.view.history.History +import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer} +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} + +import scala.concurrent.duration._ +import scala.collection.mutable.WrappedArray + +class DeliveryManagerReRequestModifiesSpec extends WordSpecLike + with BeforeAndAfterAll + with Matchers + with InstanceFactory + with OneInstancePerTest + with TestNetSettings { + + implicit val system: ActorSystem = ActorSystem("SynchronousTestingSpec") + + override def afterAll(): Unit = system.terminate() + + def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DM], + ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte], History) = { + val (deliveryManager, history) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) + val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) + val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) + val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) + val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 + val headersIds: List[ModifierId] = blocks.map(_.header.id) + val headersAsKey = headersIds.map(toKey) + (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey, history) + } + + "ReRequestModifies" should { + "re-ask necessary modifier several times (number of attempts from testNetSettings) and remove modifier from " + + //todo: move to message builder tests // "expectedModifiers collection after all attempts will expire" in { // val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() // @@ -66,7 +70,7 @@ // // val header: ModifierId = headersIds.head // -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header)) +// deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header) // handler1.expectMsgAllOf( // testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), // RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), @@ -76,112 +80,73 @@ // //this thread sleep is using for expecting modifier removal // Thread.sleep(6000) // -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty).isEmpty) -// deliveryManager.stop() -// } -// "not re-ask unnecessary modifiers" in { -// val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val header: ModifierId = headersIds.head -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(header)) -// -// //await one re-ask -// handler1.expectMsgAllOf( -// testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)) -// ) -// -// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray -// -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, -// Map(header -> headerBytes)), cp1) -// deliveryManager.stop() -// } -// "not re-ask modifiers which were applied to the history" in { -// val (deliveryManager, _, _, _, blocks, headerIds, _, history) = initialiseState() -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) -// -// handler1.expectMsg(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(headerIds.head))) -// -// val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray -// -// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, -// Map(headerIds.head -> headerBytes)), cp1) -// -// history.append(blocks.head.header) -// val uHistory: History = history.reportModifierIsValid(blocks.head.header) -// -// deliveryManager ! UpdatedHistory(uHistory) -// -// deliveryManager ! SemanticallySuccessfulModifier(blocks.head.header) -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) -// -// assert(deliveryManager.underlyingActor.expectedModifiers -// .getOrElse(cp1.socketAddress, Map.empty).isEmpty) -// deliveryManager.stop() -// } -// "remove peer from expectedModifiers if expected modifiers collection from this peer is empty" in { -// val (deliveryManager, cp1, _, _, _, headerIds, _, _) = initialiseState() -// -// deliveryManager ! RequestFromLocal(cp1, Header.modifierTypeId, Seq(headerIds.head)) -// //this thread sleep is using for expecting modifier removal -// Thread.sleep((testNetSettings.network.maxDeliveryChecks * testNetSettings.network.deliveryTimeout._1) * 1000) -// assert(deliveryManager.underlyingActor.expectedModifiers.getOrElse(cp1.socketAddress, Map.empty).isEmpty) -// assert(deliveryManager.underlyingActor.expectedModifiers -// .getOrElse(cp1.socketAddress, Map.empty) == Map.empty) -// deliveryManager.stop() -// } -// "not re-ask transactions" in { -// val (deliveryManager, _, _, _, _, _, _, _) = initialiseState() -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val transactions: Seq[ModifierId] = genValidPaymentTxs(1).map(_.id) -// -// deliveryManager ! RequestFromLocal(cp1, Transaction.modifierTypeId, transactions) -// -// handler1.expectMsgAllOf( -// RequestModifiersNetworkMessage(Transaction.modifierTypeId -> transactions) -// ) -// handler1.expectNoMsg(10.seconds) -// assert(deliveryManager.underlyingActor.expectedModifiers -// .getOrElse(cp1.socketAddress, Map.empty) == Map.empty) +// assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) // deliveryManager.stop() // } -// } -//} \ No newline at end of file + "not re-ask unnecessary modifiers" in { + val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address1 -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + val header: ModifierId = headersIds.head + + deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header) + + //await one re-ask + handler1.expectMsgAllOf( + testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), + RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), + RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)) + ) + + val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray + + deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, + Map(header -> headerBytes)), cp1.socketAddress) + deliveryManager.stop() + } + "remove peer from expectedModifiers if expected modifiers collection from this peer is empty" in { + val (deliveryManager, cp1, _, _, _, headerIds, _, _) = initialiseState() + + deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, headerIds.head) + //this thread sleep is using for expecting modifier removal + Thread.sleep((testNetSettings.network.maxDeliveryChecks * testNetSettings.network.deliveryTimeout._1) * 2000) + assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) + deliveryManager.stop() + } + "not re-ask transactions" in { + val (deliveryManager, _, _, _, _, _, _, _) = initialiseState() + + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = + Map(address1 -> (cp1, Older, InitialPriority)) + + deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) + + val transactions: Seq[ModifierId] = genValidPaymentTxs(1).map(_.id) + + transactions.foreach(txId => deliveryManager ! RequestSent(cp1.socketAddress, Transaction.modifierTypeId, txId)) + + handler1.expectMsgAllOf( + RequestModifiersNetworkMessage(Transaction.modifierTypeId -> transactions) + ) + handler1.expectNoMsg(10.seconds) + assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) + deliveryManager.stop() + } + } +} \ No newline at end of file diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala index f0e6f1215e..3e2d241362 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala @@ -81,8 +81,8 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte deliveryManager ! DataFromPeer(ModifiersNetworkMessage( Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1.socketAddress) assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) - assert(deliveryManager.underlyingActor.receivedModifier.size == blocks.size) - assert(deliveryManager.underlyingActor.receivedModifier.forall(elem => headersAsKey.contains(elem))) + assert(deliveryManager.underlyingActor.receivedModifiers.size == blocks.size) + assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) deliveryManager.stop() } "Delivery manager should not handle repeating modifiers" in { @@ -96,8 +96,8 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1.socketAddress) deliveryManager ! DataFromPeer(ModifiersNetworkMessage( Header.modifierTypeId -> blocks.map(k => k.header.id -> headerBytes).toMap), cp1.socketAddress) - assert(deliveryManager.underlyingActor.receivedModifier.size == headersIds.size) - assert(deliveryManager.underlyingActor.receivedModifier.forall(elem => headersAsKey.contains(elem))) + assert(deliveryManager.underlyingActor.receivedModifiers.size == headersIds.size) + assert(deliveryManager.underlyingActor.receivedModifiers.forall(elem => headersAsKey.contains(elem))) deliveryManager.stop() } "handle priority request for payload correctly" in { @@ -260,8 +260,8 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) - assert(deliveryManager.underlyingActor.receivedModifier.size == 1) - assert(deliveryManager.underlyingActor.receivedModifier.contains(toKey(header.id))) + assert(deliveryManager.underlyingActor.receivedModifiers.size == 1) + assert(deliveryManager.underlyingActor.receivedModifiers.contains(toKey(header.id))) deliveryManager.stop() } } From e08d76b3914c94adfe839d5b6d86c35ede1aad80 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 17 Mar 2020 11:52:39 +0300 Subject: [PATCH 095/177] fix some bugs in dm and pk --- src/main/scala/encry/network/DM.scala | 18 +- src/main/scala/encry/network/PK.scala | 1 + .../scala/encry/network/BlackListTests.scala | 193 +++++++++--------- .../history/ModifiersValidationTest.scala | 90 ++++---- 4 files changed, 151 insertions(+), 151 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 52adcc35b8..dbd2a2fc1f 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -26,7 +26,7 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging type ModifierIdAsKey = scala.collection.mutable.WrappedArray.ofByte var expectedModifiers: Set[ModifierIdAsKey] = Set.empty - var receivedModifier: Set[ModifierIdAsKey] = Set.empty + var receivedModifiers: Set[ModifierIdAsKey] = Set.empty override def preStart(): Unit = context.parent ! RegisterMessagesHandler(Seq(ModifiersNetworkMessage.NetworkMessageTypeID -> "ModifiersNetworkMessage"), self) @@ -37,16 +37,16 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging if (expectedModifiers.contains(toKey(id))) { context.parent ! ModifierFromNetwork(source, data._1, id, bytes) expectedModifiers -= toKey(id) - receivedModifier += toKey(id) + receivedModifiers += toKey(id) } else logger.info(s"Receive spam. ModId: ${Algos.encode(id)}!") } - case RequestSent(peer, modTypeId, modId) if !(expectedModifiers.contains(toKey(modId)) || receivedModifier.contains(toKey(modId))) => + case RequestSent(peer, modTypeId, modId) if !(expectedModifiers.contains(toKey(modId)) || receivedModifiers.contains(toKey(modId))) => expectedModifiers += toKey(modId) context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)( self ! AwaitingRequest(peer, modTypeId, modId, 1) ) case RequestSent(_, _, _) => //do nothing - case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts <= networkSettings.maxDeliveryChecks && expectedModifiers.contains(toKey(modId))=> + case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts < networkSettings.maxDeliveryChecks && expectedModifiers.contains(toKey(modId))=> context.parent ! RequestFromLocal(peer.some, modTypeId, List(modId)) logger.info(s"Re-request modifier ${Algos.encode(modId)}") context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! @@ -59,16 +59,16 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging case ModifierFromNetwork(source, modTypeId, modId, modBytes) => if (expectedModifiers.contains(toKey(modId))) { expectedModifiers -= toKey(modId) - receivedModifier += toKey(modId) + receivedModifiers += toKey(modId) context.parent ! ModifierFromNetwork(source, modTypeId, modId, modBytes) } else logger.info(s"Peer $source sent spam mod of type $modTypeId and id ${Algos.encode(modId)}") - case SemanticallySuccessfulModifier(mod) => receivedModifier -= toKey(mod.id) - case SemanticallyFailedModification(mod, _) => receivedModifier -= toKey(mod.id) + case SemanticallySuccessfulModifier(mod) => receivedModifiers -= toKey(mod.id) + case SemanticallyFailedModification(mod, _) => receivedModifiers -= toKey(mod.id) case IsRequested(modIds) => //logger.info(s"Going to check if ${Algos.encode(modId)} has been requested. Res: ${receivedModifier.contains(toKey(modId))}") sender ! RequestStatus( - requested = modIds.filter(id => receivedModifier.contains(toKey(id)) || expectedModifiers.contains(toKey(id))), - notRequested = modIds.filter(id => !receivedModifier.contains(toKey(id)) && !expectedModifiers.contains(toKey(id))) + requested = modIds.filter(id => receivedModifiers.contains(toKey(id)) || expectedModifiers.contains(toKey(id))), + notRequested = modIds.filter(id => !receivedModifiers.contains(toKey(id)) && !expectedModifiers.contains(toKey(id))) ) } diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 8b175cfcd2..ed3a9cdcf6 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -57,6 +57,7 @@ class PK(networkSettings: NetworkSettings, blackList.getAll.toList ) } + context.system.scheduler.schedule(600.millis, blacklistSettings.cleanupTime){blackList = blackList.cleanupBlackList} } override def receive: Receive = banPeersLogic orElse networkMessagesProcessingLogic orElse { diff --git a/src/test/scala/encry/network/BlackListTests.scala b/src/test/scala/encry/network/BlackListTests.scala index d71faf7d77..ada46f14a7 100644 --- a/src/test/scala/encry/network/BlackListTests.scala +++ b/src/test/scala/encry/network/BlackListTests.scala @@ -14,100 +14,99 @@ import org.encryfoundation.common.network.BasicMessagesRepo.Handshake import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} import scala.concurrent.duration._ -//class BlackListTests extends WordSpecLike -// with Matchers -// with BeforeAndAfterAll -// with InstanceFactory -// with OneInstancePerTest -// with TestNetSettings { -// -// implicit val system: ActorSystem = ActorSystem() -// -// override def afterAll(): Unit = system.terminate() -// -// val knowPeersSettings = testNetSettings.copy( -// network = settings.network.copy( -// knownPeers = List(new InetSocketAddress("172.16.11.11", 9001)), -// connectOnlyWithKnownPeers = Some(true) -// ), -// blackList = settings.blackList.copy( -// banTime = 2 seconds, -// cleanupTime = 3 seconds -// )) -// -// /* -// Unit tests -// */ -// "Black list" should { -// "temporary ban requested peer correctly" in { -// val blackList: BlackList = BlackList(settings.blackList) -// val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress -// val newBL = blackList.banPeer(SemanticallyInvalidPersistentModifier, peer) -// newBL.contains(peer) shouldBe true -// } -// "clean black list from peers with expired ban time which were banned by temporary ban" in { -// val blackList: BlackList = BlackList(settings.blackList) -// val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress -// val newBL = blackList.banPeer(SyntacticallyInvalidPersistentModifier, peer) -// Thread.sleep(2000) -// val newBL1 = newBL.cleanupBlackList -// newBL1.contains(peer) shouldBe false -// } -// "don't remove peer from black list before ban time expired" in { -// val blackList: BlackList = BlackList(settings.blackList) -// val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress -// val newBL = blackList.banPeer(SentInvForPayload, peer) -// val newBL1 = newBL.cleanupBlackList -// newBL1.contains(peer) shouldBe true -// } -// } -// -// /* -// Akka tests -// */ -// "Peers keeper" should { -// "handle ban peer message correctly" in { -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) -// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) -// val peerHandler: TestProbe = TestProbe() -// val connectedPeer: ConnectedPeer = ConnectedPeer( -// address, -// peerHandler.ref, -// Outgoing, -// Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) -// ) -// peersKeeper ! BanPeer(connectedPeer, SpamSender) -// peerHandler.expectMsg(CloseConnection) -// peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe true -// } -// "cleanup black list by scheduler correctly" in { -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) -// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) -// val peerHandler: TestProbe = TestProbe() -// val connectedPeer: ConnectedPeer = ConnectedPeer( -// address, -// peerHandler.ref, -// Outgoing, -// Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) -// ) -// peersKeeper ! BanPeer(connectedPeer, SentPeersMessageWithoutRequest) -// Thread.sleep(6000) -// peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe false -// } -// "don't remove peer from black list before ban time expired" in { -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) -// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) -// val peerHandler: TestProbe = TestProbe() -// val connectedPeer: ConnectedPeer = ConnectedPeer( -// address, -// peerHandler.ref, -// Outgoing, -// Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) -// ) -// Thread.sleep(4000) -// peersKeeper ! BanPeer(connectedPeer, CorruptedSerializedBytes) -// Thread.sleep(2000) -// peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe true -// } -// } -//} \ No newline at end of file +class BlackListTests extends WordSpecLike + with Matchers + with BeforeAndAfterAll + with InstanceFactory + with OneInstancePerTest + with TestNetSettings { + + implicit val system: ActorSystem = ActorSystem() + + override def afterAll(): Unit = system.terminate() + + val knowPeersSettings = testNetSettings.copy( + network = settings.network.copy( + knownPeers = List(new InetSocketAddress("172.16.11.11", 9001)), + connectOnlyWithKnownPeers = Some(true) + ), + blackList = settings.blackList.copy( + banTime = 2 seconds, + cleanupTime = 3 seconds + )) + + /* + Unit tests + */ + "Black list" should { + "temporary ban requested peer correctly" in { + val blackList: BlackList = BlackList(settings.blackList.copy(banTime = 1 millisecond)) + val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress + val newBL = blackList.banPeer(SemanticallyInvalidPersistentModifier, peer) + newBL.contains(peer) shouldBe true + } + "clean black list from peers with expired ban time which were banned by temporary ban" in { + val blackList: BlackList = BlackList(settings.blackList.copy(banTime = 1 millisecond)) + val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress + val newBL = blackList.banPeer(SyntacticallyInvalidPersistentModifier, peer) + Thread.sleep(2000) + val newBL1 = newBL.cleanupBlackList + newBL1.contains(peer) shouldBe false + } + "don't remove peer from black list before ban time expired" in { + val blackList: BlackList = BlackList(settings.blackList.copy(banTime = 1 millisecond)) + val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress + val newBL = blackList.banPeer(SentInvForPayload, peer) + val newBL1 = newBL.cleanupBlackList + newBL1.contains(peer) shouldBe true + } + } + + /* + Akka tests + */ + "Peers keeper" should { + "handle ban peer message correctly" in { + val peersKeeper: TestActorRef[PK] = TestActorRef[PK](PK.props(settings.network, settings.blackList)) + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) + val peerHandler: TestProbe = TestProbe() + val connectedPeer: ConnectedPeer = ConnectedPeer( + address, + peerHandler.ref, + Outgoing, + Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) + ) + peersKeeper ! BanPeer(connectedPeer.socketAddress, SpamSender) + peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe true + } + "cleanup black list by scheduler correctly" in { + val peersKeeper: TestActorRef[PK] = TestActorRef[PK](PK.props(settings.network, settings.blackList.copy(banTime = 1 millisecond, cleanupTime = 1 millisecond))) + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) + val peerHandler: TestProbe = TestProbe() + val connectedPeer: ConnectedPeer = ConnectedPeer( + address, + peerHandler.ref, + Outgoing, + Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) + ) + peersKeeper ! BanPeer(connectedPeer.socketAddress, SentPeersMessageWithoutRequest) + Thread.sleep(6000) + peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe false + } + "don't remove peer from black list before ban time expired" in { + val peersKeeper: TestActorRef[PK] = TestActorRef[PK](PK.props(settings.network, settings.blackList)) + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) + val peerHandler: TestProbe = TestProbe() + val connectedPeer: ConnectedPeer = ConnectedPeer( + address, + peerHandler.ref, + Outgoing, + Handshake(protocolToBytes(knowPeersSettings.network.appVersion), "test node", Some(address), System.currentTimeMillis()) + ) + Thread.sleep(4000) + peersKeeper ! BanPeer(connectedPeer.socketAddress, CorruptedSerializedBytes) + Thread.sleep(2000) + peersKeeper.underlyingActor.blackList.contains(address.getAddress) shouldBe true + } + } +} \ No newline at end of file diff --git a/src/test/scala/encry/view/history/ModifiersValidationTest.scala b/src/test/scala/encry/view/history/ModifiersValidationTest.scala index 722258dc54..e879864ea8 100644 --- a/src/test/scala/encry/view/history/ModifiersValidationTest.scala +++ b/src/test/scala/encry/view/history/ModifiersValidationTest.scala @@ -1,45 +1,45 @@ -//package encry.view.history -// -//import encry.modifiers.InstanceFactory -//import encry.network.DeliveryManagerTests.DMUtils.generateBlocks -//import encry.settings.{EncryAppSettings, TestNetSettings} -//import org.encryfoundation.common.modifiers.history.Block -//import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} -// -//class ModifiersValidationTest extends WordSpecLike -// with Matchers -// with InstanceFactory -// with OneInstancePerTest -// with TestNetSettings { -// -// "Modifiers validator" should { -// "validate genesis block" in { -// val newHistory: History = generateDummyHistory(testNetSettings) -// val genesisBlock: Block = generateGenesisBlock(testNetSettings.constants.GenesisHeight) -// newHistory.testApplicable(genesisBlock.header).isRight shouldBe true -// newHistory.append(genesisBlock.header) -// val updatedHistory: History = newHistory.reportModifierIsValid(genesisBlock.header) -// updatedHistory.testApplicable(genesisBlock.payload).isRight shouldBe true -// } -// "reject incorrect modifiers" in { -// val blocks: List[Block] = generateBlocks(2, generateDummyHistory(testNetSettings))._2 -// val newHistory: History = generateDummyHistory(testNetSettings) -// blocks.take(1).foldLeft(newHistory) { case (history, block) => -// history.testApplicable(block.header).isRight shouldBe true -// history.append(block.header) -// history.reportModifierIsValid(block.header) -// history.testApplicable(block.payload).isRight shouldBe true -// history.append(block.payload) -// history.reportModifierIsValid(block) -// } -// blocks.takeRight(1).foldLeft(newHistory) { case (history, block) => -// history.testApplicable(block.header).isRight shouldBe false -// history.append(block.header) -// history.reportModifierIsValid(block.header) -// history.testApplicable(block.payload).isRight shouldBe true -// history.append(block.payload) -// history.reportModifierIsValid(block) -// } -// } -// } -//} \ No newline at end of file +package encry.view.history + +import encry.modifiers.InstanceFactory +import encry.network.DeliveryManagerTests.DMUtils.generateBlocks +import encry.settings.{EncryAppSettings, TestNetSettings} +import org.encryfoundation.common.modifiers.history.Block +import org.scalatest.{Matchers, OneInstancePerTest, WordSpecLike} + +class ModifiersValidationTest extends WordSpecLike + with Matchers + with InstanceFactory + with OneInstancePerTest + with TestNetSettings { + + "Modifiers validator" should { + "validate genesis block" in { + val newHistory: History = generateDummyHistory(testNetSettings) + val genesisBlock: Block = generateGenesisBlock(testNetSettings.constants.GenesisHeight) + newHistory.testApplicable(genesisBlock.header).isRight shouldBe true + newHistory.append(genesisBlock.header) + val updatedHistory: History = newHistory.reportModifierIsValid(genesisBlock.header) + updatedHistory.testApplicable(genesisBlock.payload).isRight shouldBe true + } + "reject incorrect modifiers" in { + val blocks: List[Block] = generateBlocks(2, generateDummyHistory(testNetSettings))._2 + val newHistory: History = generateDummyHistory(testNetSettings) + blocks.take(1).foldLeft(newHistory) { case (history, block) => + history.testApplicable(block.header).isRight shouldBe true + history.append(block.header) + history.reportModifierIsValid(block.header) + history.testApplicable(block.payload).isRight shouldBe true + history.append(block.payload) + history.reportModifierIsValid(block) + } + blocks.takeRight(1).foldLeft(newHistory) { case (history, block) => + history.testApplicable(block.header).isRight shouldBe false + history.append(block.header) + history.reportModifierIsValid(block.header) + history.testApplicable(block.payload).isRight shouldBe true + history.append(block.payload) + history.reportModifierIsValid(block) + } + } + } +} \ No newline at end of file From 8f216f0c3c10281e28c437e7f8499352ad4629e9 Mon Sep 17 00:00:00 2001 From: lior Date: Tue, 17 Mar 2020 13:54:19 +0300 Subject: [PATCH 096/177] test added --- .../encry/view/mempool/MemoryPoolTests.scala | 43 ++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala index 04e5b67301..c805e925d3 100644 --- a/src/test/scala/encry/view/mempool/MemoryPoolTests.scala +++ b/src/test/scala/encry/view/mempool/MemoryPoolTests.scala @@ -1,16 +1,25 @@ package encry.view.mempool +import java.net.InetSocketAddress + +import scala.concurrent.duration._ import akka.actor.ActorSystem -import akka.testkit.{ TestActorRef, TestProbe } +import akka.testkit.{TestActorRef, TestProbe} import com.typesafe.scalalogging.StrictLogging import encry.modifiers.InstanceFactory -import encry.mpg.MemoryPool._ -import encry.mpg.{ MemoryPool, MemoryPoolStorage } +import encry.mpg.MemoryPool.{RolledBackTransactions, TransactionProcessing, UpdateMempoolReader} +import encry.mpg.{IntermediaryMempool, MemoryPool, MemoryPoolProcessor, MemoryPoolReader, MemoryPoolStorage, TransactionsValidator} +import encry.network.BlackList.BanReason.SemanticallyInvalidPersistentModifier +import encry.network.DeliveryManager.FullBlockChainIsSynced +import encry.network.DeliveryManagerTests.DMUtils.{createPeer, generateBlocks} +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.PeerConnectionHandler.ConnectedPeer +import encry.network.PeersKeeper.BanPeer import encry.settings.TestNetSettings import encry.utils.NetworkTimeProvider -import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } - -import scala.concurrent.duration._ +import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer, Payload} +import org.encryfoundation.common.network.BasicMessagesRepo.ModifiersNetworkMessage +import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} class MemoryPoolTests extends WordSpecLike @@ -58,5 +67,27 @@ class MemoryPoolTests txs.map(_.encodedId).forall(transactions.map(_.encodedId).contains) shouldBe true transactions.map(_.encodedId).forall(txs.map(_.encodedId).contains) shouldBe true } + "chainSynced is true on FullBlockChainIsSynced" in { + val mP = TestActorRef[MemoryPoolProcessor](MemoryPoolProcessor.props(settings, timeProvider)) + mP ! FullBlockChainIsSynced + mP.underlyingActor.chainSynced shouldBe true + } + "storage changes on RolledBackTransactions" in { + val fakeActor = TestProbe() + val storage = MemoryPoolStorage.empty(testNetSettings, timeProvider) + val memPool = TestActorRef[MemoryPool](MemoryPool.props(settings, timeProvider, Some(fakeActor.ref), fakeActor.ref)) + val txs = (0 until 10).map(k => coinbaseAt(k)) + memPool ! RolledBackTransactions(txs) + assert(memPool.underlyingActor.memoryPool != storage) + memPool.underlyingActor.memoryPool.size shouldBe txs.length + } + "TransactionProcessing" in { + val mP = TestActorRef[MemoryPoolProcessor](MemoryPoolProcessor.props(settings, timeProvider)) + mP ! TransactionProcessing(true) + mP.underlyingActor.canProcessTransactions shouldBe true + mP ! TransactionProcessing(false) + mP.underlyingActor.canProcessTransactions shouldBe false + } + } } From cb3a283110801306d5e0828dbe9fad387c4ba2d1 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 17 Mar 2020 14:29:29 +0300 Subject: [PATCH 097/177] sync info message tests added --- .../scala/encry/network/DeliveryManager.scala | 2 +- .../encry/network/NodeViewSynchronizer.scala | 4 +- src/main/scala/encry/network/PK.scala | 2 +- .../scala/encry/network/PeersKeeper.scala | 2 +- .../scala/encry/nvg/IntermediaryNVH.scala | 2 +- .../encry/nvg/NetworkMessagesProcessor.scala | 10 +- .../scala/encry/view/history/HistoryApi.scala | 2 +- .../history/HistoryPayloadsProcessor.scala | 28 +- src/test/scala/encry/nvg/NVHTests.scala | 5 + .../nvg/NetworkMessagesProcessorTests.scala | 284 ++++++++++++++++++ src/test/scala/encry/nvg/PipelinesTests.scala | 5 + src/test/scala/encry/nvg/Utils.scala | 24 ++ src/test/scala/encry/nvg/ValidatorTests.scala | 5 + 13 files changed, 355 insertions(+), 20 deletions(-) create mode 100644 src/test/scala/encry/nvg/NVHTests.scala create mode 100644 src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala create mode 100644 src/test/scala/encry/nvg/PipelinesTests.scala create mode 100644 src/test/scala/encry/nvg/Utils.scala create mode 100644 src/test/scala/encry/nvg/ValidatorTests.scala diff --git a/src/main/scala/encry/network/DeliveryManager.scala b/src/main/scala/encry/network/DeliveryManager.scala index 47c475f688..5e76c06349 100644 --- a/src/main/scala/encry/network/DeliveryManager.scala +++ b/src/main/scala/encry/network/DeliveryManager.scala @@ -538,7 +538,7 @@ object DeliveryManager { case StartTransactionsValidation => 2 - case OtherNodeSyncingStatus(_, _, _) => 1 + case OtherNodeSyncingStatus(_, _) => 1 case ConnectionStopped(_) => 1 diff --git a/src/main/scala/encry/network/NodeViewSynchronizer.scala b/src/main/scala/encry/network/NodeViewSynchronizer.scala index 45fd0cde77..4054bf3772 100644 --- a/src/main/scala/encry/network/NodeViewSynchronizer.scala +++ b/src/main/scala/encry/network/NodeViewSynchronizer.scala @@ -220,9 +220,7 @@ object NodeViewSynchronizer { case object SendLocalSyncInfo - final case class OtherNodeSyncingStatus(remote: InetSocketAddress, - status: encry.consensus.HistoryConsensus.HistoryComparisonResult, - extension: Option[Seq[(ModifierTypeId, ModifierId)]]) + final case class OtherNodeSyncingStatus(remote: InetSocketAddress, status: HistoryComparisonResult) sealed trait CLIPeer diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index ed3a9cdcf6..3c2efb1c5a 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -85,7 +85,7 @@ class PK(networkSettings: NetworkSettings, logger.info(s"Adding new peer: $peer to awaitingHandshakeConnections." + s" Current is: ${awaitingHandshakeConnections.mkString(",")}") } - case OtherNodeSyncingStatus(remote, comparison, _) => + case OtherNodeSyncingStatus(remote, comparison) => connectedPeers = connectedPeers.updateHistoryComparisonResult(Map(remote -> comparison)) case NewConnection(remote, remoteConnection) if connectedPeers.size < networkSettings.maxConnections && !isSelf(remote) => logger.info(s"Peers keeper got request for verifying the connection with remote: $remote. " + diff --git a/src/main/scala/encry/network/PeersKeeper.scala b/src/main/scala/encry/network/PeersKeeper.scala index dc92e76c20..72dc6e85bb 100644 --- a/src/main/scala/encry/network/PeersKeeper.scala +++ b/src/main/scala/encry/network/PeersKeeper.scala @@ -352,7 +352,7 @@ object PeersKeeper { class PeersKeeperPriorityQueue(settings: ActorSystem.Settings, config: Config) extends UnboundedStablePriorityMailbox( PriorityGenerator { - case OtherNodeSyncingStatus(_, _, _) => 0 + case OtherNodeSyncingStatus(_, _) => 0 case AccumulatedPeersStatistic(_) => 1 case BanPeer(_, _) => 1 case SendLocalSyncInfo => 1 diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 4bcf66def2..cd74130cbd 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -98,7 +98,7 @@ class IntermediaryNVH( case msg: LocallyGeneratedModifier => nodeViewHolder ! msg case msg @ BanPeer(_, _) => intermediaryNetwork ! msg case msg @ InvalidModifierBytes(_) => intermediaryNetwork ! msg - case msg @ OtherNodeSyncingStatus(_, _, _) => intermediaryNetwork ! msg + case msg @ OtherNodeSyncingStatus(_, _) => intermediaryNetwork ! msg case msg @ RequestFromLocal(_, _, _) => intermediaryNetwork ! msg case msg @ ResponseFromLocal(_, _, _) => intermediaryNetwork ! msg case msg @ BroadcastModifier(_, _) => intermediaryNetwork ! msg diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 13761962ca..58832e39eb 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -65,15 +65,13 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St case DataFromPeer(message, remote) => message match { case SyncInfoNetworkMessage(syncInfo: SyncInfo) => - val ext: Seq[ModifierId] = - historyReader.continuationIds(syncInfo, settings.network.syncPacketLength) + logger.info(s"Got sync info network message. This message includes ${syncInfo.lastHeaderIds.size} ids.") val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) logger.info( - s"Comparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}. " + - s"Comparison result is $comparison. Sending extension of length ${ext.length}." + s"\n\n Comparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}.\n " + + s"Comparison result is $comparison. \n " ) - if (!(ext.nonEmpty || comparison != Younger)) logger.info("Extension is empty while comparison is younger") - context.parent ! OtherNodeSyncingStatus(remote, comparison, Some(ext.map(h => Header.modifierTypeId -> h))) + context.parent ! OtherNodeSyncingStatus(remote, comparison) case InvNetworkMessage(invData) if invData._1 == Payload.modifierTypeId && !historyReader.isFullChainSynced => logger.info( diff --git a/src/main/scala/encry/view/history/HistoryApi.scala b/src/main/scala/encry/view/history/HistoryApi.scala index 4b8c6c7e92..91f5090cb8 100644 --- a/src/main/scala/encry/view/history/HistoryApi.scala +++ b/src/main/scala/encry/view/history/HistoryApi.scala @@ -216,7 +216,7 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore }.toList }.getOrElse(List.empty)) - def compare(si: SyncInfo): HistoryComparisonResult = lastSyncInfo.lastHeaderIds.lastOption match { + def compare(si: SyncInfo): HistoryComparisonResult = syncInfo.lastHeaderIds.lastOption match { //Our best header is the same as other history best header case Some(id) if si.lastHeaderIds.lastOption.exists(_ sameElements id) => Equal //Our best header is in other history best chain, but not at the last position diff --git a/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala b/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala index 071e547cf2..9732bd6af6 100644 --- a/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala +++ b/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala @@ -5,8 +5,9 @@ import cats.syntax.option._ import encry.consensus.HistoryConsensus.ProgressInfo import encry.modifiers.history.HeaderChain import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } -import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.{Height, ModifierId} trait HistoryPayloadsProcessor extends HistoryApi { @@ -27,7 +28,8 @@ trait HistoryPayloadsProcessor extends HistoryApi { processValidFirstBlock(blockToProcess, header, bestFullChain) case Some(header) if isBestBlockDefined && isBetterChain(header.id) => processBetterChain(blockToProcess, header, Seq.empty, settings.node.blocksToKeep) - case Some(_) => + case Some(header) => + logger.info(s"\n\nnonBestBlock. id: ${blockToProcess.header.encodedId}. cause: ${isBestBlockDefined} or ${ isBetterChain(header.id)}\n\n") nonBestBlock(blockToProcess) case None => logger.debug(s"Best full chain is empty. Returning empty progress info") @@ -54,7 +56,10 @@ trait HistoryPayloadsProcessor extends HistoryApi { val toApply: Seq[Block] = newChain.tail.headers .flatMap(h => if (h == fullBlock.header) fullBlock.some else getBlockByHeader(h)) toApply.foreach(addBlockToCacheIfNecessary) - if (toApply.lengthCompare(newChain.length - 1) != 0) nonBestBlock(fullBlock) + if (toApply.lengthCompare(newChain.length - 1) != 0) { + logger.info(s"To apply. processBetterChain. nonBestBlock.") + nonBestBlock(fullBlock) + } else { //application of this block leads to full chain with higher score logger.info(s"Appending ${fullBlock.encodedId}|${fullBlock.header.height} as a better chain") @@ -104,8 +109,19 @@ trait HistoryPayloadsProcessor extends HistoryApi { prevBestScore <- scoreOf(bestFullBlockId) score <- scoreOf(id) bestBlockHeight = getBestBlockHeight - } yield (bestBlockHeight < heightOfThisHeader) || (bestBlockHeight == heightOfThisHeader && score > prevBestScore)) - .getOrElse(false) + } yield { + logger.info(s"isBetterChain. id: ${Algos.encode(id)}. \n " + + s"bestBlockHeight: $bestBlockHeight.\n " + + s"heightOfThisHeader $heightOfThisHeader.\n " + + s"score: $score.\n " + + s"prevBestScore: $prevBestScore.\n " + + s"res is: ${(bestBlockHeight < heightOfThisHeader) || (bestBlockHeight == heightOfThisHeader && score > prevBestScore)}") + (bestBlockHeight < heightOfThisHeader) || (bestBlockHeight == heightOfThisHeader && score > prevBestScore) + }) + .getOrElse { + logger.info(s"isBetterChain. id: ${Algos.encode(id)}. getOrElse. false.") + false + } private def calculateBestFullChain(block: Block): Seq[Block] = { val continuations: Seq[Seq[Header]] = continuationHeaderChains(block.header, h => isBlockDefined(h)).map(_.tail) diff --git a/src/test/scala/encry/nvg/NVHTests.scala b/src/test/scala/encry/nvg/NVHTests.scala new file mode 100644 index 0000000000..3f7c4728ee --- /dev/null +++ b/src/test/scala/encry/nvg/NVHTests.scala @@ -0,0 +1,5 @@ +package encry.nvg + +class NVHTests { + +} diff --git a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala new file mode 100644 index 0000000000..896b4e857d --- /dev/null +++ b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala @@ -0,0 +1,284 @@ +package encry.nvg + +import java.net.InetSocketAddress + +import akka.actor.{ ActorRef, ActorSystem } +import akka.testkit.{ TestActorRef, TestKit, TestProbe } +import cats.syntax.eq._ +import encry.consensus.HistoryConsensus.{ Equal, Older, Unknown, Younger } +import encry.modifiers.InstanceFactory +import encry.network.DeliveryManager.CheckPayloadsToDownload +import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus +import encry.nvg.NodeViewHolder.UpdateHistoryReader +import encry.nvg.Utils.instances._ +import encry.settings.EncryAppSettings +import encry.view.history.{ History, HistoryReader } +import org.encryfoundation.common.modifiers.history.Block +import org.encryfoundation.common.network.BasicMessagesRepo.{ NetworkMessage, SyncInfoNetworkMessage } +import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } + +class NetworkMessagesProcessorTests + extends TestKit(ActorSystem("Tested-Akka-System")) + with WordSpecLike + with Matchers + with BeforeAndAfterAll + with InstanceFactory + with OneInstancePerTest { + + "Network messages processor" should { + "process sync info message correctly" should { + "determine older extension" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (historyMain: History, historyOlder: History) = + NetworkMessagesProcessorTests.formYoungerActorState(10, 10) + + val historyReader: HistoryReader = HistoryReader(historyMain) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(historyOlder.syncInfo) + + val (dataFromPeerMsg, address) = + NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + + val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Older) + + networkProcessor ! dataFromPeerMsg + + parentActor.expectMsgPF() { + case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case CheckPayloadsToDownload => + case _ => true shouldBe false + } + } + "determine younger extension" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (historyMain: History, historyYounger: History) = + NetworkMessagesProcessorTests.formOlderActorState(10, 10) + + val historyReader: HistoryReader = HistoryReader(historyMain) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(historyYounger.syncInfo) + + val (dataFromPeerMsg, address) = + NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + + val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Younger) + + networkProcessor ! dataFromPeerMsg + + parentActor.expectMsgPF() { + case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case CheckPayloadsToDownload => + case _ => true shouldBe false + } + } + "determine equals extension" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (history1: History, history2: History) = + NetworkMessagesProcessorTests.fromEqualActorState(10, 10) + + val historyReader: HistoryReader = HistoryReader(history1) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(history2.syncInfo) + + val (dataFromPeerMsg, address) = + NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + + val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Equal) + + networkProcessor ! dataFromPeerMsg + + parentActor.expectMsgPF() { + case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case CheckPayloadsToDownload => + case _ => true shouldBe false + } + } + "determine unknown extension" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (h1: History, h2: History) = NetworkMessagesProcessorTests.fromEUnknownActorState + + val historyReader: HistoryReader = HistoryReader(h1) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(h2.syncInfo) + + val (dataFromPeerMsg, address) = + NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + + val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Unknown) + + networkProcessor ! dataFromPeerMsg + + parentActor.expectMsgPF() { + case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case CheckPayloadsToDownload => + case _ => true shouldBe false + } + } + } + "process inv message correctly" in {} + "process request for modifier correctly" in {} + "have correct logic with check payloads to download" in {} + "have correct logic with semantically successful modifier" in {} + } +} + +object NetworkMessagesProcessorTests extends InstanceFactory { + + def initActorState(settings: EncryAppSettings)(implicit AS: ActorSystem): TestActorRef[NetworkMessagesProcessor] = { + val networkProcessor: TestActorRef[NetworkMessagesProcessor] = + TestActorRef[NetworkMessagesProcessor](NetworkMessagesProcessor.props(settings)) + networkProcessor + } + + def formDataFromPeerMessage(innerMessage: NetworkMessage, host: String, port: Int)( + implicit AS: ActorSystem + ): (DataFromPeer, InetSocketAddress) = { + val address = new InetSocketAddress(host, port) + DataFromPeer(innerMessage, address) -> address + } + + def formYoungerActorState(blocksQty: Int, olderBlocksQty: Int): (History, History) = { + val (hMain, hOlder, blocks) = (0 until blocksQty).foldLeft( + generateDummyHistory(settings), + generateDummyHistory(settings), + List.empty[Block] + ) { + case ((historyMain, historyOlder, blocks: List[Block]), _) => + val block: Block = generateNextBlock(historyMain) + val hMain: History = historyMain + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + + val hOlder = historyOlder + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + (hMain, hOlder, block +: blocks) + } + val (hOlderUpdated, _) = (0 until olderBlocksQty).foldLeft(hOlder, List.empty[Block]) { + case ((historyOlder, blocks: List[Block]), _) => + val block: Block = generateNextBlock(historyOlder) + val hOlder: History = historyOlder + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + (hOlder, block +: blocks) + } + (hMain, hOlderUpdated) + } + + def formOlderActorState(blocksQty: Int, olderBlocksQty: Int): (History, History) = { + val (historyYounger, historyOlder) = formYoungerActorState(blocksQty, olderBlocksQty) + (historyOlder, historyYounger) + } + + def fromEqualActorState(blocksQty: Int, olderBlocksQty: Int): (History, History) = + (0 until blocksQty).foldLeft( + generateDummyHistory(settings), + generateDummyHistory(settings) + ) { + case ((historyMain, historyOlder), _) => + val block: Block = generateNextBlock(historyMain) + val hEq1: History = historyMain + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + + val hEq2 = historyOlder + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + (hEq1, hEq2) + } + + def fromEUnknownActorState: (History, History) = { + val history1 = (0 until 100).foldLeft( + generateDummyHistory(settings) + ) { + case (h1, _) => + val block1: Block = generateNextBlock(h1) + val hEq1: History = h1 + .append(block1.header) + .right + .get + ._1 + .append(block1.payload) + .right + .get + ._1 + .reportModifierIsValid(block1) + hEq1 + } + val history2 = (0 until 2).foldLeft( + generateDummyHistory(settings) + ) { + case (h2, _) => + val block1: Block = generateNextBlock(h2) + val hEq2: History = h2 + .append(block1.header) + .right + .get + ._1 + .append(block1.payload) + .right + .get + ._1 + .reportModifierIsValid(block1) + hEq2 + } + history1 -> history2 + } + +} diff --git a/src/test/scala/encry/nvg/PipelinesTests.scala b/src/test/scala/encry/nvg/PipelinesTests.scala new file mode 100644 index 0000000000..05d227e223 --- /dev/null +++ b/src/test/scala/encry/nvg/PipelinesTests.scala @@ -0,0 +1,5 @@ +package encry.nvg + +class PipelinesTests { + +} diff --git a/src/test/scala/encry/nvg/Utils.scala b/src/test/scala/encry/nvg/Utils.scala new file mode 100644 index 0000000000..54c6ae0d34 --- /dev/null +++ b/src/test/scala/encry/nvg/Utils.scala @@ -0,0 +1,24 @@ +package encry.nvg + +import cats.Eq +import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus +import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import cats.instances.option._ +import cats.instances.int._ +import cats.instances.byte._ +import cats.syntax.eq._ + +object Utils { + object instances { + implicit object ModifierIdEq extends Eq[ModifierId] { + override def eqv(x: ModifierId, y: ModifierId): Boolean = + x.sameElements(y) + } + + implicit object OtherNodeSyncStatusEq extends Eq[OtherNodeSyncingStatus] { + override def eqv(x: OtherNodeSyncingStatus, y: OtherNodeSyncingStatus): Boolean = + x.status == y.status && x.remote == y.remote + } + } + +} diff --git a/src/test/scala/encry/nvg/ValidatorTests.scala b/src/test/scala/encry/nvg/ValidatorTests.scala new file mode 100644 index 0000000000..72bcac2eed --- /dev/null +++ b/src/test/scala/encry/nvg/ValidatorTests.scala @@ -0,0 +1,5 @@ +package encry.nvg + +class ValidatorTests { + +} From 204db4e048745d9e527e5c97099f38825ed6ce73 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 17 Mar 2020 14:30:20 +0300 Subject: [PATCH 098/177] fixed modifiers propagation --- src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala index 896b4e857d..09544360f9 100644 --- a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala +++ b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala @@ -1,7 +1,6 @@ package encry.nvg import java.net.InetSocketAddress - import akka.actor.{ ActorRef, ActorSystem } import akka.testkit.{ TestActorRef, TestKit, TestProbe } import cats.syntax.eq._ From 2d084cffb361589ae5874c8419636d52636736a8 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 17 Mar 2020 14:43:22 +0300 Subject: [PATCH 099/177] added test for fork in sync info --- .../encry/nvg/NetworkMessagesProcessor.scala | 18 +-- .../nvg/NetworkMessagesProcessorTests.scala | 113 ++++++++++++++++-- 2 files changed, 111 insertions(+), 20 deletions(-) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 58832e39eb..e3a819d6ea 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -62,17 +62,17 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St ) case _ => } + + case DataFromPeer(SyncInfoNetworkMessage(syncInfo: SyncInfo), remote) => + val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) + logger.info( + s"\n\n Comparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}.\n" + + s"Comparison result is $comparison. \n " + ) + context.parent ! OtherNodeSyncingStatus(remote, comparison) + case DataFromPeer(message, remote) => message match { - case SyncInfoNetworkMessage(syncInfo: SyncInfo) => - logger.info(s"Got sync info network message. This message includes ${syncInfo.lastHeaderIds.size} ids.") - val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) - logger.info( - s"\n\n Comparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}.\n " + - s"Comparison result is $comparison. \n " - ) - context.parent ! OtherNodeSyncingStatus(remote, comparison) - case InvNetworkMessage(invData) if invData._1 == Payload.modifierTypeId && !historyReader.isFullChainSynced => logger.info( s"Got inv message with payloads: ${invData._2.map(Algos.encode).mkString(",")}. " + diff --git a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala index 09544360f9..cbdb0f3c4d 100644 --- a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala +++ b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala @@ -1,10 +1,11 @@ package encry.nvg import java.net.InetSocketAddress -import akka.actor.{ ActorRef, ActorSystem } -import akka.testkit.{ TestActorRef, TestKit, TestProbe } + +import akka.actor.{ActorRef, ActorSystem} +import akka.testkit.{TestActorRef, TestKit, TestProbe} import cats.syntax.eq._ -import encry.consensus.HistoryConsensus.{ Equal, Older, Unknown, Younger } +import encry.consensus.HistoryConsensus.{Equal, Fork, Older, Unknown, Younger} import encry.modifiers.InstanceFactory import encry.network.DeliveryManager.CheckPayloadsToDownload import encry.network.NetworkController.ReceivableMessages.DataFromPeer @@ -12,10 +13,10 @@ import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingSta import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.nvg.Utils.instances._ import encry.settings.EncryAppSettings -import encry.view.history.{ History, HistoryReader } +import encry.view.history.{History, HistoryReader} import org.encryfoundation.common.modifiers.history.Block -import org.encryfoundation.common.network.BasicMessagesRepo.{ NetworkMessage, SyncInfoNetworkMessage } -import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } +import org.encryfoundation.common.network.BasicMessagesRepo.{NetworkMessage, SyncInfoNetworkMessage} +import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} class NetworkMessagesProcessorTests extends TestKit(ActorSystem("Tested-Akka-System")) @@ -87,7 +88,7 @@ class NetworkMessagesProcessorTests val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) val (history1: History, history2: History) = - NetworkMessagesProcessorTests.fromEqualActorState(10, 10) + NetworkMessagesProcessorTests.formEqualActorState(10, 10) val historyReader: HistoryReader = HistoryReader(history1) @@ -113,7 +114,7 @@ class NetworkMessagesProcessorTests val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) - val (h1: History, h2: History) = NetworkMessagesProcessorTests.fromEUnknownActorState + val (h1: History, h2: History) = NetworkMessagesProcessorTests.formUnknownActorState val historyReader: HistoryReader = HistoryReader(h1) @@ -128,6 +129,33 @@ class NetworkMessagesProcessorTests networkProcessor ! dataFromPeerMsg + parentActor.expectMsgPF() { + case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case CheckPayloadsToDownload => + case _ => true shouldBe false + } + } + "determine fork extension" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (h1: History, h2: History) = + NetworkMessagesProcessorTests.formForkActorState(10, 20, 5) + + val historyReader: HistoryReader = HistoryReader(h1) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(h2.syncInfo) + + val (dataFromPeerMsg, address) = + NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + + val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Fork) + + networkProcessor ! dataFromPeerMsg + parentActor.expectMsgPF() { case msg: OtherNodeSyncingStatus => msg eqv expectedResult case CheckPayloadsToDownload => @@ -211,7 +239,7 @@ object NetworkMessagesProcessorTests extends InstanceFactory { (historyOlder, historyYounger) } - def fromEqualActorState(blocksQty: Int, olderBlocksQty: Int): (History, History) = + def formEqualActorState(blocksQty: Int, olderBlocksQty: Int): (History, History) = (0 until blocksQty).foldLeft( generateDummyHistory(settings), generateDummyHistory(settings) @@ -242,8 +270,8 @@ object NetworkMessagesProcessorTests extends InstanceFactory { (hEq1, hEq2) } - def fromEUnknownActorState: (History, History) = { - val history1 = (0 until 100).foldLeft( + def formUnknownActorState: (History, History) = { + val history1 = (0 until 10).foldLeft( generateDummyHistory(settings) ) { case (h1, _) => @@ -280,4 +308,67 @@ object NetworkMessagesProcessorTests extends InstanceFactory { history1 -> history2 } + def formForkActorState(forkOn: Int, forkSize: Int, mainChainSize: Int): (History, History) = { + val (h1, h2) = (0 until forkOn).foldLeft( + generateDummyHistory(settings), + generateDummyHistory(settings) + ) { + case ((historyMain, historyOlder), _) => + val block: Block = generateNextBlock(historyMain) + val hEq1: History = historyMain + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + + val hEq2 = historyOlder + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + (hEq1, hEq2) + } + val fork = (0 until forkSize).foldLeft(h1) { + case (historyMain, _) => + val block: Block = generateNextBlock(historyMain) + val hEq1: History = historyMain + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + hEq1 + } + val mch = (0 until mainChainSize).foldLeft(h2) { + case (historyMain, _) => + val block: Block = generateNextBlock(historyMain) + val hEq1: History = historyMain + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + hEq1 + } + mch -> fork + } + } From 61ab55b6c4e624bdb17f85de49e897636d459fbe Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 17 Mar 2020 17:17:30 +0300 Subject: [PATCH 100/177] added tests for processing inv message on nvh messages processor --- .../encry/nvg/NetworkMessagesProcessor.scala | 47 ++++--- .../nvg/NetworkMessagesProcessorTests.scala | 131 ++++++++++++++++-- src/test/scala/encry/nvg/Utils.scala | 13 +- 3 files changed, 156 insertions(+), 35 deletions(-) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index e3a819d6ea..672154b22e 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -15,6 +15,7 @@ import encry.utils.Utils.idsToString import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.network.BasicMessagesRepo.BasicMsgDataTypes.InvData import org.encryfoundation.common.network.BasicMessagesRepo.{ InvNetworkMessage, RequestModifiersNetworkMessage, @@ -22,7 +23,8 @@ import org.encryfoundation.common.network.BasicMessagesRepo.{ } import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } + import scala.concurrent.duration._ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { @@ -67,34 +69,33 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) logger.info( s"\n\n Comparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}.\n" + - s"Comparison result is $comparison. \n " + s"Comparison result is $comparison.\n " ) context.parent ! OtherNodeSyncingStatus(remote, comparison) - case DataFromPeer(message, remote) => - message match { - case InvNetworkMessage(invData) if invData._1 == Payload.modifierTypeId && !historyReader.isFullChainSynced => - logger.info( - s"Got inv message with payloads: ${invData._2.map(Algos.encode).mkString(",")}. " + - s"But full chain is not synced. Ignore them." - ) - - case InvNetworkMessage(invData) => - val startTime: Long = System.currentTimeMillis() - val ids: Seq[ModifierId] = invData._2.filterNot( - (mid: ModifierId) => - historyReader.isModifierDefined(mid) || ModifiersCache.contains(NodeViewHolder.toKey(mid)) - ) + case DataFromPeer(InvNetworkMessage(invData: InvData), remote) => + if (invData._1 == Payload.modifierTypeId && !historyReader.isFullChainSynced) + logger.info(s"Got inv message from $remote with ${invData._2.size} ids but full chain is not synced.") + else { + val isHeader: Boolean = invData._1 == Header.modifierTypeId + val isPayloadAvailable: Boolean = + historyReader.isHeadersChainSyncedVar && invData._1 == Payload.modifierTypeId + val isRequestAvailable: Boolean = isHeader || isPayloadAvailable + if (isRequestAvailable) { + val ids: Seq[ModifierId] = invData._2.filterNot { mid: ModifierId => + historyReader.isModifierDefined(mid) || ModifiersCache.contains(NodeViewHolder.toKey(mid)) + } + logger.info(s"Sending request from local to $remote with ${ids.size} ids of type ${invData._1}.") + if (ids.nonEmpty) context.parent ! RequestFromLocal(remote.some, invData._1, ids.toList) + } else logger.info( - s"Got inv message from $remote. Type of nested ids is: ${invData._1}. " + - s"Ids for request are: ${ids.map(Algos.encode).mkString(",")}." + s"Got inv message from $remote but response is unavailable cause:" + + s" is header - $isHeader || is payload available - $isPayloadAvailable." ) - if (ids.nonEmpty && - (invData._1 == Header.modifierTypeId || - (historyReader.isHeadersChainSyncedVar && invData._1 == Payload.modifierTypeId))) - sender() ! RequestFromLocal(remote.some, invData._1, ids.toList) - logger.info(s"Time of processing inv message is: ${(System.currentTimeMillis() - startTime) / 1000}s.") + } + case DataFromPeer(message, remote) => + message match { case RequestModifiersNetworkMessage((typeId, requestedIds)) => val modifiersFromCache: Map[ModifierId, Array[Byte]] = requestedIds .flatMap( diff --git a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala index cbdb0f3c4d..48f6f37492 100644 --- a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala +++ b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala @@ -5,18 +5,26 @@ import java.net.InetSocketAddress import akka.actor.{ActorRef, ActorSystem} import akka.testkit.{TestActorRef, TestKit, TestProbe} import cats.syntax.eq._ +import cats.syntax.option._ import encry.consensus.HistoryConsensus.{Equal, Fork, Older, Unknown, Younger} import encry.modifiers.InstanceFactory import encry.network.DeliveryManager.CheckPayloadsToDownload +import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus import encry.nvg.NodeViewHolder.UpdateHistoryReader import encry.nvg.Utils.instances._ import encry.settings.EncryAppSettings import encry.view.history.{History, HistoryReader} -import org.encryfoundation.common.modifiers.history.Block -import org.encryfoundation.common.network.BasicMessagesRepo.{NetworkMessage, SyncInfoNetworkMessage} +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, NetworkMessage, SyncInfoNetworkMessage} +import org.encryfoundation.common.utils.TaggedTypes +import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +import scorex.utils.Random +import supertagged.@@ + +import scala.collection.immutable class NetworkMessagesProcessorTests extends TestKit(ActorSystem("Tested-Akka-System")) @@ -26,6 +34,8 @@ class NetworkMessagesProcessorTests with InstanceFactory with OneInstancePerTest { + override def afterAll(): Unit = system.terminate() + "Network messages processor" should { "process sync info message correctly" should { "determine older extension" in { @@ -50,7 +60,7 @@ class NetworkMessagesProcessorTests networkProcessor ! dataFromPeerMsg parentActor.expectMsgPF() { - case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => case _ => true shouldBe false } @@ -77,7 +87,7 @@ class NetworkMessagesProcessorTests networkProcessor ! dataFromPeerMsg parentActor.expectMsgPF() { - case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => case _ => true shouldBe false } @@ -104,7 +114,7 @@ class NetworkMessagesProcessorTests networkProcessor ! dataFromPeerMsg parentActor.expectMsgPF() { - case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => case _ => true shouldBe false } @@ -130,7 +140,7 @@ class NetworkMessagesProcessorTests networkProcessor ! dataFromPeerMsg parentActor.expectMsgPF() { - case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => case _ => true shouldBe false } @@ -157,13 +167,102 @@ class NetworkMessagesProcessorTests networkProcessor ! dataFromPeerMsg parentActor.expectMsgPF() { - case msg: OtherNodeSyncingStatus => msg eqv expectedResult + case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => case _ => true shouldBe false } } } - "process inv message correctly" in {} + "process inv message correctly" should { + "not process inv for payload while full chain is not synced" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) + + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) + + networkProcessor ! DataFromPeer(InvNetworkMessage(Payload.modifierTypeId -> ids), address) + + parentActor.expectNoMsg() + } + "not create response from local for payloads if header's chain is not synced" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) + + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) + + val historyReader = HistoryReader.empty + + historyReader.isHeadersChainSyncedVar = false + + historyReader.isFullChainSynced = true + + networkProcessor ! UpdateHistoryReader(historyReader) + + networkProcessor ! DataFromPeer(InvNetworkMessage(Payload.modifierTypeId -> ids), address) + + parentActor.expectNoMsg() + } + "create response from local for payloads if header's chain is synced" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) + + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) + + val historyReader = HistoryReader.empty + + historyReader.isHeadersChainSyncedVar = true + + historyReader.isFullChainSynced = true + + networkProcessor ! UpdateHistoryReader(historyReader) + + networkProcessor ! DataFromPeer(InvNetworkMessage(Header.modifierTypeId -> ids), address) + + val requiredRequestFromLocal = RequestFromLocal(address.some, Header.modifierTypeId, ids.toList) + + parentActor.expectMsgPF() { + case CheckPayloadsToDownload => + case msg: RequestFromLocal => (msg eqv requiredRequestFromLocal) shouldBe true + } + } + "request only unique new modifiers" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) + + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) + + val (history, blocks) = NetworkMessagesProcessorTests.formHistory + + val historyReader = HistoryReader(history) + + historyReader.isHeadersChainSyncedVar = true + + historyReader.isFullChainSynced = true + + networkProcessor ! UpdateHistoryReader(historyReader) + + networkProcessor ! DataFromPeer(InvNetworkMessage(Header.modifierTypeId -> (ids ++ blocks.map(_.id))), address) + + val requiredRequestFromLocal = RequestFromLocal(address.some, Header.modifierTypeId, ids.toList) + + parentActor.expectMsgPF() { + case CheckPayloadsToDownload => + case msg: RequestFromLocal => (msg eqv requiredRequestFromLocal) shouldBe true + } + } + } "process request for modifier correctly" in {} "have correct logic with check payloads to download" in {} "have correct logic with semantically successful modifier" in {} @@ -371,4 +470,20 @@ object NetworkMessagesProcessorTests extends InstanceFactory { mch -> fork } + def formHistory: (History, List[Block]) = + (0 to 10).foldLeft(generateDummyHistory(settings), List.empty[Block]) { + case ((historyMain, blocks), _) => + val block: Block = generateNextBlock(historyMain) + val hEq1: History = historyMain + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + hEq1 -> (blocks :+ block) + } } diff --git a/src/test/scala/encry/nvg/Utils.scala b/src/test/scala/encry/nvg/Utils.scala index 54c6ae0d34..9df1ead263 100644 --- a/src/test/scala/encry/nvg/Utils.scala +++ b/src/test/scala/encry/nvg/Utils.scala @@ -1,12 +1,10 @@ package encry.nvg import cats.Eq +import cats.syntax.eq._ +import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus import org.encryfoundation.common.utils.TaggedTypes.ModifierId -import cats.instances.option._ -import cats.instances.int._ -import cats.instances.byte._ -import cats.syntax.eq._ object Utils { object instances { @@ -19,6 +17,13 @@ object Utils { override def eqv(x: OtherNodeSyncingStatus, y: OtherNodeSyncingStatus): Boolean = x.status == y.status && x.remote == y.remote } + + implicit object RequestFromLocalEq extends Eq[RequestFromLocal] { + override def eqv(x: RequestFromLocal, y: RequestFromLocal): Boolean = + x.modifierIds.size == y.modifierIds.size && + x.modifierIds.zip(y.modifierIds).forall { case (id, id1) => id === id1 } && + x.source.zip(y.source).forall { case (is1, is2) => is1 == is2 } + } } } From 9e19dd42d38465b73d818dd542eea73072d96792 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 17 Mar 2020 18:19:31 +0300 Subject: [PATCH 101/177] added tests for request modifier, for local cache, for semantically successful modifier correctness --- .../encry/nvg/NetworkMessagesProcessor.scala | 74 +++---- .../nvg/NetworkMessagesProcessorTests.scala | 205 ++++++++++++++++-- 2 files changed, 221 insertions(+), 58 deletions(-) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 672154b22e..80c07af43d 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -13,7 +13,7 @@ import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, UpdateHistoryR import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.history.HistoryReader -import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.network.BasicMessagesRepo.BasicMsgDataTypes.InvData import org.encryfoundation.common.network.BasicMessagesRepo.{ @@ -48,23 +48,20 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St ) def workingCycle(modifiersRequester: Option[Cancellable]): Receive = { - case UpdateHistoryReader(newReader: HistoryReader) => - historyReader = newReader - case SemanticallySuccessfulModifier(mod) => - //todo possible way to call CheckPayloadsToDownload - mod match { - case block: Block if historyReader.isFullChainSynced => - List(block.header, block.payload).foreach { mod: PersistentModifier => - logger.info(s"Going to broadcast inv for modifier of type ${mod.modifierTypeId} with id: ${mod.encodedId}.") - context.parent ! BroadcastModifier(mod.modifierTypeId, mod.id) - } - modifiersRequestCache = Map( - block.encodedId -> toProto(block.header), - block.payload.encodedId -> toProto(block.payload) - ) - case _ => + case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader + + //todo possible way to call CheckPayloadsToDownload + case SemanticallySuccessfulModifier(block: Block) if historyReader.isFullChainSynced => + List(block.header, block.payload).foreach { mod: PersistentModifier => + logger.info(s"Going to broadcast inv for modifier of type ${mod.modifierTypeId} with id: ${mod.encodedId}.") + context.parent ! BroadcastModifier(mod.modifierTypeId, mod.id) } + modifiersRequestCache = Map( + block.encodedId -> toProto(block.header), + block.payload.encodedId -> toProto(block.payload) + ) + case SemanticallySuccessfulModifier(_) => case DataFromPeer(SyncInfoNetworkMessage(syncInfo: SyncInfo), remote) => val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) logger.info( @@ -94,30 +91,31 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St ) } - case DataFromPeer(message, remote) => - message match { - case RequestModifiersNetworkMessage((typeId, requestedIds)) => - val modifiersFromCache: Map[ModifierId, Array[Byte]] = requestedIds - .flatMap( - (id: ModifierId) => - modifiersRequestCache - .get(Algos.encode(id)) - .map(id -> _) - ) - .toMap - if (modifiersFromCache.nonEmpty) context.parent ! ResponseFromLocal(remote, typeId, modifiersFromCache) - val unrequestedModifiers: List[ModifierId] = requestedIds.filterNot(modifiersFromCache.contains).toList - - typeId match { - case h if h == Header.modifierTypeId => - context.parent ! ResponseFromLocal(remote, typeId, getModsForRemote(unrequestedModifiers, historyReader)) - case _ => - getModsForRemote(unrequestedModifiers, historyReader).foreach { - case (id: ModifierId, bytes: Array[Byte]) => - context.parent ! ResponseFromLocal(remote, typeId, Map(id -> bytes)) - } + case DataFromPeer(RequestModifiersNetworkMessage((typeId, requestedIds)), remote) => + val modifiersFromCache: Map[ModifierId, Array[Byte]] = requestedIds.flatMap { id: ModifierId => + modifiersRequestCache + .get(Algos.encode(id)) + .map(id -> _) + }.toMap + if (modifiersFromCache.nonEmpty) { + logger.info( + s"Send response from local with mods from cache: " + + s" ${modifiersFromCache.keys.toList.map(Algos.encode).mkString(",")} to $remote." + ) + context.parent ! ResponseFromLocal(remote, typeId, modifiersFromCache) + } + val unrequestedModifiers: List[ModifierId] = requestedIds.filterNot(modifiersFromCache.contains).toList + + typeId match { + case h if h == Header.modifierTypeId => + context.parent ! ResponseFromLocal(remote, typeId, getModsForRemote(unrequestedModifiers, historyReader)) + case _ => + getModsForRemote(unrequestedModifiers, historyReader).foreach { + case (id: ModifierId, bytes: Array[Byte]) => + context.parent ! ResponseFromLocal(remote, typeId, Map(id -> bytes)) } } + case CheckPayloadsToDownload => val newIds: Seq[ModifierId] = historyReader.payloadsIdsToDownload(settings.network.networkChunkSize) logger.debug(s"newIds: ${newIds.map(elem => Algos.encode(elem)).mkString(",")}") diff --git a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala index 48f6f37492..94bbd949c5 100644 --- a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala +++ b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala @@ -2,25 +2,30 @@ package encry.nvg import java.net.InetSocketAddress -import akka.actor.{ActorRef, ActorSystem} -import akka.testkit.{TestActorRef, TestKit, TestProbe} +import akka.actor.{ ActorRef, ActorSystem } +import akka.testkit.{ TestActorRef, TestKit, TestProbe } import cats.syntax.eq._ import cats.syntax.option._ -import encry.consensus.HistoryConsensus.{Equal, Fork, Older, Unknown, Younger} +import encry.consensus.HistoryConsensus.{ Equal, Fork, Older, Unknown, Younger } import encry.modifiers.InstanceFactory import encry.network.DeliveryManager.CheckPayloadsToDownload -import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal } import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus -import encry.nvg.NodeViewHolder.UpdateHistoryReader +import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, UpdateHistoryReader } import encry.nvg.Utils.instances._ import encry.settings.EncryAppSettings -import encry.view.history.{History, HistoryReader} -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} -import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, NetworkMessage, SyncInfoNetworkMessage} +import encry.view.history.{ History, HistoryReader } +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.network.BasicMessagesRepo.{ + InvNetworkMessage, + NetworkMessage, + RequestModifiersNetworkMessage, + SyncInfoNetworkMessage +} import org.encryfoundation.common.utils.TaggedTypes -import org.encryfoundation.common.utils.TaggedTypes.ModifierId -import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } +import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } import scorex.utils.Random import supertagged.@@ @@ -62,7 +67,6 @@ class NetworkMessagesProcessorTests parentActor.expectMsgPF() { case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => - case _ => true shouldBe false } } "determine younger extension" in { @@ -89,7 +93,6 @@ class NetworkMessagesProcessorTests parentActor.expectMsgPF() { case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => - case _ => true shouldBe false } } "determine equals extension" in { @@ -116,7 +119,6 @@ class NetworkMessagesProcessorTests parentActor.expectMsgPF() { case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => - case _ => true shouldBe false } } "determine unknown extension" in { @@ -142,7 +144,6 @@ class NetworkMessagesProcessorTests parentActor.expectMsgPF() { case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => - case _ => true shouldBe false } } "determine fork extension" in { @@ -169,7 +170,6 @@ class NetworkMessagesProcessorTests parentActor.expectMsgPF() { case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true case CheckPayloadsToDownload => - case _ => true shouldBe false } } } @@ -231,7 +231,7 @@ class NetworkMessagesProcessorTests parentActor.expectMsgPF() { case CheckPayloadsToDownload => - case msg: RequestFromLocal => (msg eqv requiredRequestFromLocal) shouldBe true + case msg: RequestFromLocal => (msg eqv requiredRequestFromLocal) shouldBe true } } "request only unique new modifiers" in { @@ -259,13 +259,178 @@ class NetworkMessagesProcessorTests parentActor.expectMsgPF() { case CheckPayloadsToDownload => - case msg: RequestFromLocal => (msg eqv requiredRequestFromLocal) shouldBe true + case msg: RequestFromLocal => (msg eqv requiredRequestFromLocal) shouldBe true + } + } + } + "process semantically successful modifier correctly" should { + "update local cache with last semantically successful modifier" in { + val networkMessagesProcessor: TestActorRef[NetworkMessagesProcessor] = + NetworkMessagesProcessorTests.initActorState(settings) + + val reader = HistoryReader.empty + + reader.isFullChainSynced = true + + networkMessagesProcessor ! UpdateHistoryReader(reader) + + val block = generateGenesisBlock(Height @@ 1) + + networkMessagesProcessor ! SemanticallySuccessfulModifier(block) + + networkMessagesProcessor.underlyingActor.modifiersRequestCache.size shouldBe 2 + networkMessagesProcessor.underlyingActor.modifiersRequestCache.get(block.encodedId).nonEmpty shouldBe true + networkMessagesProcessor.underlyingActor.modifiersRequestCache + .get(block.payload.encodedId) + .nonEmpty shouldBe true + + val block2 = generateGenesisBlock(Height @@ 2) + + networkMessagesProcessor ! SemanticallySuccessfulModifier(block2) + + networkMessagesProcessor.underlyingActor.modifiersRequestCache.size shouldBe 2 + networkMessagesProcessor.underlyingActor.modifiersRequestCache.get(block.encodedId).nonEmpty shouldBe false + networkMessagesProcessor.underlyingActor.modifiersRequestCache + .get(block.payload.encodedId) + .nonEmpty shouldBe false + + networkMessagesProcessor.underlyingActor.modifiersRequestCache.get(block2.encodedId).nonEmpty shouldBe true + networkMessagesProcessor.underlyingActor.modifiersRequestCache + .get(block2.payload.encodedId) + .nonEmpty shouldBe true + } + "send broadcast message for new modifier" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val reader = HistoryReader.empty + + reader.isFullChainSynced = true + + networkProcessor ! UpdateHistoryReader(reader) + + val block = generateGenesisBlock(Height @@ 1) + + networkProcessor ! SemanticallySuccessfulModifier(block) + + parentActor.expectMsgPF() { + case CheckPayloadsToDownload => + case BroadcastModifier(modType, id) if modType == Header.modifierTypeId => + id.sameElements(block.id) shouldBe true + case BroadcastModifier(modType, id) if modType == Payload.modifierTypeId => + id.sameElements(block.payload.id) shouldBe true + } + } + } + "process request for modifier correctly" should { + "response for modifiers which are in cache by using this cache" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (history, blocks) = NetworkMessagesProcessorTests.formHistory + + val historyReader = HistoryReader(history) + + historyReader.isFullChainSynced = true + + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val block = generateGenesisBlock(Height @@ 1) + + networkProcessor ! SemanticallySuccessfulModifier(block) + + networkProcessor ! DataFromPeer( + RequestModifiersNetworkMessage(Header.modifierTypeId -> (blocks.headOption.get.id :: block.id :: Nil)), + address + ) + + parentActor.expectMsgPF() { + case ResponseFromLocal(_, _, ids) if ids.size == 1 => + ids.forall { + case (id, _) => + id.sameElements(block.id) + } shouldBe true + case ResponseFromLocal(_, _, ids) => + ids.forall { + case (id, _) => + id.sameElements(blocks.headOption.get.id) + } shouldBe true + case CheckPayloadsToDownload => + case _: BroadcastModifier => + } + } + "response for headers in 1 message for all headers" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (history, blocks) = NetworkMessagesProcessorTests.formHistory + + val historyReader = HistoryReader(history) + + historyReader.isFullChainSynced = true + + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val block = generateGenesisBlock(Height @@ 1) + + networkProcessor ! SemanticallySuccessfulModifier(block) + + networkProcessor ! DataFromPeer( + RequestModifiersNetworkMessage(Header.modifierTypeId -> blocks.take(2).map(_.id)), + address + ) + + parentActor.expectMsgPF() { + case ResponseFromLocal(_, _, ids) if ids.size == 2 => + ids.keys.toList.zip(blocks.take(2).map(_.id)).forall { case (id, id1) => id.sameElements(id1) } shouldBe true + case CheckPayloadsToDownload => + case _: BroadcastModifier => } } + "response for payloads in 1 message for 1 payload" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (history, blocks) = NetworkMessagesProcessorTests.formHistory + + val historyReader = HistoryReader(history) + + historyReader.isFullChainSynced = true + + val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val block = generateGenesisBlock(Height @@ 1) + + networkProcessor ! SemanticallySuccessfulModifier(block) + + networkProcessor ! DataFromPeer( + RequestModifiersNetworkMessage(Payload.modifierTypeId -> blocks.take(2).map(_.payload.id)), + address + ) + + parentActor.expectMsgPF() { + case ResponseFromLocal(_, _, ids) if ids.size == 1 => + ids.keys.toList.forall { id => + id.sameElements(blocks(1).payload.id) || id.sameElements(blocks.head.payload.id) + } shouldBe true + case CheckPayloadsToDownload => + case _: BroadcastModifier => + } + } + } + "have correct logic with check payloads to download" should { + } - "process request for modifier correctly" in {} - "have correct logic with check payloads to download" in {} - "have correct logic with semantically successful modifier" in {} } } From 532c23c7ff2f84e586de91d2f58d9d35af0abdac Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 17 Mar 2020 18:55:20 +0300 Subject: [PATCH 102/177] added tests for check payloads to download --- .../encry/nvg/NetworkMessagesProcessor.scala | 8 +- .../nvg/NetworkMessagesProcessorTests.scala | 79 ++++++++++++++++++- 2 files changed, 79 insertions(+), 8 deletions(-) diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala index 80c07af43d..109dd8f860 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala @@ -3,7 +3,7 @@ package encry.nvg import akka.actor.{ Actor, Cancellable, Props } import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging -import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Younger } +import encry.consensus.HistoryConsensus.HistoryComparisonResult import encry.network.DeliveryManager.CheckPayloadsToDownload import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } import encry.network.ModifiersToNetworkUtils.toProto @@ -13,7 +13,7 @@ import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, UpdateHistoryR import encry.settings.EncryAppSettings import encry.utils.Utils.idsToString import encry.view.history.HistoryReader -import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } +import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.network.BasicMessagesRepo.BasicMsgDataTypes.InvData import org.encryfoundation.common.network.BasicMessagesRepo.{ @@ -23,7 +23,7 @@ import org.encryfoundation.common.network.BasicMessagesRepo.{ } import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } +import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.concurrent.duration._ @@ -118,7 +118,7 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St case CheckPayloadsToDownload => val newIds: Seq[ModifierId] = historyReader.payloadsIdsToDownload(settings.network.networkChunkSize) - logger.debug(s"newIds: ${newIds.map(elem => Algos.encode(elem)).mkString(",")}") + logger.debug(s"newIds: ${newIds.map(Algos.encode).mkString(",")}") if (newIds.nonEmpty) context.parent ! RequestFromLocal(none, Payload.modifierTypeId, newIds.toList) val nextCheckModsScheduler: Cancellable = context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)(self ! CheckPayloadsToDownload) diff --git a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala index 94bbd949c5..fe281a3a7e 100644 --- a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala +++ b/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala @@ -6,10 +6,10 @@ import akka.actor.{ ActorRef, ActorSystem } import akka.testkit.{ TestActorRef, TestKit, TestProbe } import cats.syntax.eq._ import cats.syntax.option._ -import encry.consensus.HistoryConsensus.{ Equal, Fork, Older, Unknown, Younger } +import encry.consensus.HistoryConsensus._ import encry.modifiers.InstanceFactory import encry.network.DeliveryManager.CheckPayloadsToDownload -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal } +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus import encry.nvg.NodeViewHolder.{ SemanticallySuccessfulModifier, UpdateHistoryReader } @@ -23,13 +23,12 @@ import org.encryfoundation.common.network.BasicMessagesRepo.{ RequestModifiersNetworkMessage, SyncInfoNetworkMessage } -import org.encryfoundation.common.utils.TaggedTypes import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } import scorex.utils.Random -import supertagged.@@ import scala.collection.immutable +import scala.concurrent.duration._ class NetworkMessagesProcessorTests extends TestKit(ActorSystem("Tested-Akka-System")) @@ -429,7 +428,79 @@ class NetworkMessagesProcessorTests } } "have correct logic with check payloads to download" should { + "request required modifiers" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (_, blocks) = NetworkMessagesProcessorTests.formHistory + + val history = blocks.take(5).foldLeft(generateDummyHistory(settings)) { + case (h, block) => + h.append(block.header).right.get._1.reportModifierIsValid(block.header) + } + + history.isFullChainSynced = true + + val historyReader = HistoryReader(history) + + networkProcessor ! UpdateHistoryReader(historyReader) + + val requiredResponse = RequestFromLocal(none, Payload.modifierTypeId, blocks.take(5).map(_.payload.id)) + + parentActor.expectMsgPF(settings.network.syncInterval + 1.seconds) { + case _: SendSyncInfo => + case msg: RequestFromLocal => msg.eqv(requiredResponse) shouldBe true + } + + parentActor.expectMsgPF(settings.network.syncInterval + 1.seconds) { + case _: SendSyncInfo => + case msg: RequestFromLocal => msg.eqv(requiredResponse) shouldBe true + } + + val updHistory = blocks.drop(5).foldLeft(history) { + case (h, block) => + h.append(block.header).right.get._1.reportModifierIsValid(block.header) + } + + val updHistory2 = blocks.take(3).foldLeft(updHistory) { + case (h, block) => + h.append(block.payload).right.get._1.reportModifierIsValid(block.payload).reportModifierIsValid(block) + } + val historyReader1 = HistoryReader(updHistory2) + + networkProcessor ! UpdateHistoryReader(historyReader1) + + val requiredResponse1 = RequestFromLocal(none, Payload.modifierTypeId, blocks.drop(3).map(_.payload.id)) + + parentActor.expectMsgPF(settings.network.syncInterval + 1.seconds) { + case _: SendSyncInfo => + case msg: RequestFromLocal => msg.eqv(requiredResponse1) shouldBe true + } + } + "not request if headers chain is not synced" in { + val parentActor = TestProbe() + + val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + + val (_, blocks) = NetworkMessagesProcessorTests.formHistory + + val history = blocks.foldLeft(generateDummyHistory(settings)) { + case (h, block) => + h.append(block.header).right.get._1.reportModifierIsValid(block.header) + } + + history.isFullChainSynced = false + + val historyReader = HistoryReader(history) + + networkProcessor ! UpdateHistoryReader(historyReader) + + parentActor.expectMsgPF(settings.network.syncInterval + 2.seconds) { + case _: SendSyncInfo => + } + } } } } From dc805387f6643f974c008d98224b0797389a5d20 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 17 Mar 2020 18:58:14 +0300 Subject: [PATCH 103/177] NodeViewNMProcessor tests finished --- .../scala/encry/nvg/IntermediaryNVH.scala | 2 +- ...cessor.scala => NodeViewNMProcessor.scala} | 6 +- ...s.scala => NodeViewNMProcessorTests.scala} | 76 +++++++++---------- 3 files changed, 42 insertions(+), 42 deletions(-) rename src/main/scala/encry/nvg/{NetworkMessagesProcessor.scala => NodeViewNMProcessor.scala} (96%) rename src/test/scala/encry/nvg/{NetworkMessagesProcessorTests.scala => NodeViewNMProcessorTests.scala} (90%) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index cd74130cbd..0cecfa527a 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -52,7 +52,7 @@ class IntermediaryNVH( intermediaryNetwork ! RegisterForModsHandling val networkMessagesProcessor: ActorRef = - context.actorOf(NetworkMessagesProcessor.props(settings), name = "Network-messages-processor") + context.actorOf(NodeViewNMProcessor.props(settings), name = "Network-messages-processor") val nodeViewHolder: ActorRef = context.actorOf(NodeViewHolder.props(settings, timeProvider, influxRef), name = "Node-view-holder") val modifiersValidatorRouter: ActorRef = diff --git a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala similarity index 96% rename from src/main/scala/encry/nvg/NetworkMessagesProcessor.scala rename to src/main/scala/encry/nvg/NodeViewNMProcessor.scala index 109dd8f860..77b63d1e04 100644 --- a/src/main/scala/encry/nvg/NetworkMessagesProcessor.scala +++ b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala @@ -27,7 +27,7 @@ import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.concurrent.duration._ -class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { +class NodeViewNMProcessor(settings: EncryAppSettings) extends Actor with StrictLogging { import context.dispatcher @@ -132,6 +132,6 @@ class NetworkMessagesProcessor(settings: EncryAppSettings) extends Actor with St .toMap } -object NetworkMessagesProcessor { - def props(settings: EncryAppSettings): Props = Props(new NetworkMessagesProcessor(settings)) +object NodeViewNMProcessor { + def props(settings: EncryAppSettings): Props = Props(new NodeViewNMProcessor(settings)) } diff --git a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala b/src/test/scala/encry/nvg/NodeViewNMProcessorTests.scala similarity index 90% rename from src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala rename to src/test/scala/encry/nvg/NodeViewNMProcessorTests.scala index fe281a3a7e..ddaa238e6e 100644 --- a/src/test/scala/encry/nvg/NetworkMessagesProcessorTests.scala +++ b/src/test/scala/encry/nvg/NodeViewNMProcessorTests.scala @@ -30,7 +30,7 @@ import scorex.utils.Random import scala.collection.immutable import scala.concurrent.duration._ -class NetworkMessagesProcessorTests +class NodeViewNMProcessorTests extends TestKit(ActorSystem("Tested-Akka-System")) with WordSpecLike with Matchers @@ -45,10 +45,10 @@ class NetworkMessagesProcessorTests "determine older extension" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val (historyMain: History, historyOlder: History) = - NetworkMessagesProcessorTests.formYoungerActorState(10, 10) + NodeViewNMProcessorTests.formYoungerActorState(10, 10) val historyReader: HistoryReader = HistoryReader(historyMain) @@ -57,7 +57,7 @@ class NetworkMessagesProcessorTests val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(historyOlder.syncInfo) val (dataFromPeerMsg, address) = - NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Older) @@ -71,10 +71,10 @@ class NetworkMessagesProcessorTests "determine younger extension" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val (historyMain: History, historyYounger: History) = - NetworkMessagesProcessorTests.formOlderActorState(10, 10) + NodeViewNMProcessorTests.formOlderActorState(10, 10) val historyReader: HistoryReader = HistoryReader(historyMain) @@ -83,7 +83,7 @@ class NetworkMessagesProcessorTests val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(historyYounger.syncInfo) val (dataFromPeerMsg, address) = - NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Younger) @@ -97,10 +97,10 @@ class NetworkMessagesProcessorTests "determine equals extension" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val (history1: History, history2: History) = - NetworkMessagesProcessorTests.formEqualActorState(10, 10) + NodeViewNMProcessorTests.formEqualActorState(10, 10) val historyReader: HistoryReader = HistoryReader(history1) @@ -109,7 +109,7 @@ class NetworkMessagesProcessorTests val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(history2.syncInfo) val (dataFromPeerMsg, address) = - NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Equal) @@ -123,9 +123,9 @@ class NetworkMessagesProcessorTests "determine unknown extension" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - val (h1: History, h2: History) = NetworkMessagesProcessorTests.formUnknownActorState + val (h1: History, h2: History) = NodeViewNMProcessorTests.formUnknownActorState val historyReader: HistoryReader = HistoryReader(h1) @@ -134,7 +134,7 @@ class NetworkMessagesProcessorTests val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(h2.syncInfo) val (dataFromPeerMsg, address) = - NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Unknown) @@ -148,10 +148,10 @@ class NetworkMessagesProcessorTests "determine fork extension" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val (h1: History, h2: History) = - NetworkMessagesProcessorTests.formForkActorState(10, 20, 5) + NodeViewNMProcessorTests.formForkActorState(10, 20, 5) val historyReader: HistoryReader = HistoryReader(h1) @@ -160,7 +160,7 @@ class NetworkMessagesProcessorTests val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(h2.syncInfo) val (dataFromPeerMsg, address) = - NetworkMessagesProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) + NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Fork) @@ -176,7 +176,7 @@ class NetworkMessagesProcessorTests "not process inv for payload while full chain is not synced" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) @@ -189,7 +189,7 @@ class NetworkMessagesProcessorTests "not create response from local for payloads if header's chain is not synced" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) @@ -210,7 +210,7 @@ class NetworkMessagesProcessorTests "create response from local for payloads if header's chain is synced" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) @@ -236,13 +236,13 @@ class NetworkMessagesProcessorTests "request only unique new modifiers" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) - val (history, blocks) = NetworkMessagesProcessorTests.formHistory + val (history, blocks) = NodeViewNMProcessorTests.formHistory val historyReader = HistoryReader(history) @@ -264,8 +264,8 @@ class NetworkMessagesProcessorTests } "process semantically successful modifier correctly" should { "update local cache with last semantically successful modifier" in { - val networkMessagesProcessor: TestActorRef[NetworkMessagesProcessor] = - NetworkMessagesProcessorTests.initActorState(settings) + val networkMessagesProcessor: TestActorRef[NodeViewNMProcessor] = + NodeViewNMProcessorTests.initActorState(settings) val reader = HistoryReader.empty @@ -301,7 +301,7 @@ class NetworkMessagesProcessorTests "send broadcast message for new modifier" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) val reader = HistoryReader.empty @@ -326,9 +326,9 @@ class NetworkMessagesProcessorTests "response for modifiers which are in cache by using this cache" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - val (history, blocks) = NetworkMessagesProcessorTests.formHistory + val (history, blocks) = NodeViewNMProcessorTests.formHistory val historyReader = HistoryReader(history) @@ -365,9 +365,9 @@ class NetworkMessagesProcessorTests "response for headers in 1 message for all headers" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - val (history, blocks) = NetworkMessagesProcessorTests.formHistory + val (history, blocks) = NodeViewNMProcessorTests.formHistory val historyReader = HistoryReader(history) @@ -396,9 +396,9 @@ class NetworkMessagesProcessorTests "response for payloads in 1 message for 1 payload" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - val (history, blocks) = NetworkMessagesProcessorTests.formHistory + val (history, blocks) = NodeViewNMProcessorTests.formHistory val historyReader = HistoryReader(history) @@ -431,9 +431,9 @@ class NetworkMessagesProcessorTests "request required modifiers" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - val (_, blocks) = NetworkMessagesProcessorTests.formHistory + val (_, blocks) = NodeViewNMProcessorTests.formHistory val history = blocks.take(5).foldLeft(generateDummyHistory(settings)) { case (h, block) => @@ -482,9 +482,9 @@ class NetworkMessagesProcessorTests "not request if headers chain is not synced" in { val parentActor = TestProbe() - val networkProcessor: ActorRef = parentActor.childActorOf(NetworkMessagesProcessor.props(settings)) + val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - val (_, blocks) = NetworkMessagesProcessorTests.formHistory + val (_, blocks) = NodeViewNMProcessorTests.formHistory val history = blocks.foldLeft(generateDummyHistory(settings)) { case (h, block) => @@ -505,11 +505,11 @@ class NetworkMessagesProcessorTests } } -object NetworkMessagesProcessorTests extends InstanceFactory { +object NodeViewNMProcessorTests extends InstanceFactory { - def initActorState(settings: EncryAppSettings)(implicit AS: ActorSystem): TestActorRef[NetworkMessagesProcessor] = { - val networkProcessor: TestActorRef[NetworkMessagesProcessor] = - TestActorRef[NetworkMessagesProcessor](NetworkMessagesProcessor.props(settings)) + def initActorState(settings: EncryAppSettings)(implicit AS: ActorSystem): TestActorRef[NodeViewNMProcessor] = { + val networkProcessor: TestActorRef[NodeViewNMProcessor] = + TestActorRef[NodeViewNMProcessor](NodeViewNMProcessor.props(settings)) networkProcessor } From 0fa0a3da1254727d4459bfebd7caf2fec382bacd Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 17 Mar 2020 20:11:41 +0300 Subject: [PATCH 104/177] fixed proto parsing error finished tests for modifiers validator --- src/main/scala/encry/network/BlackList.scala | 1 + .../scala/encry/nvg/IntermediaryNVH.scala | 4 +- .../scala/encry/nvg/ModifiersValidator.scala | 83 ++++--- .../encry/view/history/HistoryReader.scala | 4 +- .../DownloadedModifiersValidatorTests.scala | 231 ------------------ src/test/scala/encry/nvg/ValidatorTests.scala | 181 +++++++++++++- 6 files changed, 235 insertions(+), 269 deletions(-) delete mode 100644 src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala diff --git a/src/main/scala/encry/network/BlackList.scala b/src/main/scala/encry/network/BlackList.scala index a360757969..0d84bb4f02 100644 --- a/src/main/scala/encry/network/BlackList.scala +++ b/src/main/scala/encry/network/BlackList.scala @@ -38,6 +38,7 @@ object BlackList { case object SyntacticallyInvalidPersistentModifier extends BanReason case object SyntacticallyInvalidTransaction extends BanReason case object CorruptedSerializedBytes extends BanReason + case object ModifierIdInTheNetworkMessageIsNotTheSameAsIdOfModifierInThisMessage extends BanReason case object SpamSender extends BanReason case object SentPeersMessageWithoutRequest extends BanReason case object SentInvForPayload extends BanReason diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 0cecfa527a..a1585f5e7f 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -58,7 +58,7 @@ class IntermediaryNVH( val modifiersValidatorRouter: ActorRef = context.actorOf( BalancingPool(5) - .props(ModifiersValidator.props(nodeViewHolder, settings)), + .props(ModifiersValidator.props(nodeViewHolder, self, settings)), name = "Modifiers-validator-router" ) val snapshotProcessor: Option[ActorRef] = @@ -98,7 +98,7 @@ class IntermediaryNVH( case msg: LocallyGeneratedModifier => nodeViewHolder ! msg case msg @ BanPeer(_, _) => intermediaryNetwork ! msg case msg @ InvalidModifierBytes(_) => intermediaryNetwork ! msg - case msg @ OtherNodeSyncingStatus(_, _) => intermediaryNetwork ! msg + case msg @ OtherNodeSyncingStatus(_, _) => intermediaryNetwork ! msg case msg @ RequestFromLocal(_, _, _) => intermediaryNetwork ! msg case msg @ ResponseFromLocal(_, _, _) => intermediaryNetwork ! msg case msg @ BroadcastModifier(_, _) => intermediaryNetwork ! msg diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index e3990ec576..4814b7736f 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -11,6 +11,7 @@ import encry.modifiers.history.HeaderUtils.{ IllegalHeight, PreSemanticValidatio import encry.modifiers.history.{ HeaderUtils, PayloadUtils } import encry.network.BlackList.BanReason.{ CorruptedSerializedBytes, + ModifierIdInTheNetworkMessageIsNotTheSameAsIdOfModifierInThisMessage, PreSemanticInvalidModifier, SyntacticallyInvalidPersistentModifier } @@ -24,37 +25,68 @@ import org.encryfoundation.common.modifiers.history.{ Header, HeaderProtoSeriali import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } -class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings) extends Actor with StrictLogging { +import scala.util.Try + +class ModifiersValidator( + nodeViewHolderRef: ActorRef, + intermediaryNVH: ActorRef, + settings: EncryAppSettings +) extends Actor + with StrictLogging { override def receive: Receive = { - case ModifierForValidation(reader, modifierId, modifierTypeId, modifierBytes, remote) => - fromProto(modifierTypeId, modifierBytes) match { + case ModifierForValidation(reader, id, modifierTypeId, modifierBytes, remote) if !reader.isModifierDefined(id) => + ModifiersValidator.fromProto(modifierTypeId, modifierBytes) match { case Left(error) => - logger.info(s"Modifier ${Algos.encode(modifierId)} is incorrect cause: ${error.getMessage}.") - context.parent ! BanPeer(remote, CorruptedSerializedBytes) - context.parent ! InvalidModifierBytes(modifierId) + logger.info(s"Modifier ${Algos.encode(id)} is incorrect cause: ${error.getMessage}.") + intermediaryNVH ! BanPeer(remote, CorruptedSerializedBytes) + intermediaryNVH ! InvalidModifierBytes(id) case Right(modifier) => val preSemanticValidation: Either[PreSemanticValidationException, Unit] = - isPreSemanticValid(modifier, reader, settings) + ModifiersValidator.isPreSemanticValid(modifier, reader, settings) val syntacticValidation: Boolean = - isSyntacticallyValid(modifier, settings.constants.ModifierIdSize) + ModifiersValidator.isSyntacticallyValid(modifier, settings.constants.ModifierIdSize) if (preSemanticValidation.isRight && syntacticValidation) { - logger.debug(s"Modifier ${modifier.encodedId} is valid.") - nodeViewHolderRef ! ValidatedModifier(modifier) + if (modifier.id.sameElements(id)) { + logger.debug(s"Modifier ${modifier.encodedId} is valid.") + nodeViewHolderRef ! ValidatedModifier(modifier) + } else { + logger.info(s"Modifier ${modifier.encodedId} should have ${Algos.encode(id)} id!") + intermediaryNVH ! BanPeer(remote, ModifierIdInTheNetworkMessageIsNotTheSameAsIdOfModifierInThisMessage) + intermediaryNVH ! SyntacticallyFailedModification(modifier, List.empty) + } } else if (!syntacticValidation) { logger.info(s"Modifier ${modifier.encodedId} is syntactically invalid.") - context.parent ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) - context.parent ! SyntacticallyFailedModification(modifier, List.empty) + intermediaryNVH ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) + intermediaryNVH ! SyntacticallyFailedModification(modifier, List.empty) } else preSemanticValidation.leftMap { case IllegalHeight(error) => logger.info(s"Modifier ${modifier.encodedId} is invalid cause: $error.") - context.parent ! BanPeer(remote, PreSemanticInvalidModifier(error)) - context.parent ! SyntacticallyFailedModification(modifier, List.empty) + intermediaryNVH ! BanPeer(remote, PreSemanticInvalidModifier(error)) + intermediaryNVH ! SyntacticallyFailedModification(modifier, List.empty) } } + case m: ModifierForValidation => + logger.info(s"Got modifier ${Algos.encode(m.modifierId)} but this mod is already in history.") } +} + +object ModifiersValidator { + + final case class ModifierForValidation( + historyReader: HistoryReader, + modifierId: ModifierId, + modifierTypeId: ModifierTypeId, + modifierBytes: Array[Byte], + remote: InetSocketAddress + ) + + final case class ValidatedModifier(modifier: PersistentModifier) extends AnyVal + + final case class InvalidModifierBytes(id: ModifierId) extends AnyVal + private def isPreSemanticValid( modifier: PersistentModifier, historyReader: HistoryReader, @@ -77,8 +109,8 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings bytes: Array[Byte] ): Either[Throwable, PersistentModifier] = Either.fromTry(modType match { - case Header.modifierTypeId => HeaderProtoSerializer.fromProto(HeaderProtoMessage.parseFrom(bytes)) - case Payload.modifierTypeId => PayloadProtoSerializer.fromProto(PayloadProtoMessage.parseFrom(bytes)) + case Header.modifierTypeId => Try(HeaderProtoSerializer.fromProto(HeaderProtoMessage.parseFrom(bytes))).flatten + case Payload.modifierTypeId => Try(PayloadProtoSerializer.fromProto(PayloadProtoMessage.parseFrom(bytes))).flatten }) def isSyntacticallyValid( @@ -88,22 +120,7 @@ class ModifiersValidator(nodeViewHolderRef: ActorRef, settings: EncryAppSettings case h: Header => HeaderUtils.syntacticallyValidity(h, modifierIdSize).isSuccess case p: Payload => PayloadUtils.syntacticallyValidity(p, modifierIdSize).isSuccess } -} - -object ModifiersValidator { - - final case class ModifierForValidation( - historyReader: HistoryReader, - modifierId: ModifierId, - modifierTypeId: ModifierTypeId, - modifierBytes: Array[Byte], - remote: InetSocketAddress - ) - - final case class ValidatedModifier(modifier: PersistentModifier) extends AnyVal - - final case class InvalidModifierBytes(id: ModifierId) extends AnyVal - def props(nodeViewHolderRef: ActorRef, settings: EncryAppSettings): Props = - Props(new ModifiersValidator(nodeViewHolderRef, settings)) + def props(nodeViewHolderRef: ActorRef, intermediaryNVH: ActorRef, settings: EncryAppSettings): Props = + Props(new ModifiersValidator(nodeViewHolderRef, intermediaryNVH, settings)) } diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 64f9c9edb7..7d781b3f00 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -1,9 +1,9 @@ package encry.view.history -import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Older} +import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Older } import encry.modifiers.history.HeaderChain import io.iohk.iodb.ByteArrayWrapper -import org.encryfoundation.common.modifiers.history.{Block, Header} +import org.encryfoundation.common.modifiers.history.{ Block, Header } import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.ModifierId diff --git a/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala b/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala deleted file mode 100644 index 0b05c5bfb7..0000000000 --- a/src/test/scala/encry/network/DownloadedModifiersValidatorTests.scala +++ /dev/null @@ -1,231 +0,0 @@ -package encry.network - -import java.net.InetSocketAddress - -import akka.actor.ActorSystem -import akka.testkit.{ TestActorRef, TestProbe } -import encry.modifiers.InstanceFactory -import encry.network.BlackList.BanReason._ -//import encry.network.DownloadedModifiersValidator.{ InvalidModifier, ModifiersForValidating } -import encry.network.NodeViewSynchronizer.ReceivableMessages.{ ChangedHistory, UpdatedHistory } -import encry.network.PeerConnectionHandler.{ ConnectedPeer, Outgoing } -import encry.network.PeersKeeper.BanPeer -import encry.settings.TestNetSettings -//import encry.view.NodeViewHolder.ReceivableMessages.ModifierFromRemote -import encry.view.history.History -import org.encryfoundation.common.crypto.equihash.EquihashSolution -import org.encryfoundation.common.modifiers.history.{ - Block, - Header, - HeaderProtoSerializer, - Payload, - PayloadProtoSerializer -} -import org.encryfoundation.common.network.BasicMessagesRepo.Handshake -import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } -import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } -import scorex.crypto.hash.Digest32 -import scorex.utils.Random - -//class DownloadedModifiersValidatorTests -// extends WordSpecLike -// with Matchers -// with BeforeAndAfterAll -// with InstanceFactory -// with OneInstancePerTest -// with TestNetSettings { -// -// implicit val system: ActorSystem = ActorSystem() -// -// override def afterAll(): Unit = system.terminate() -// -// "DownloadedModifiersValidatorTests" should { -// "find too old header by height" in { -// val nodeViewHolder = TestProbe() -// val peersKeeper = TestProbe() -// val nodeViewSync = TestProbe() -// val mempool = TestProbe() -// -// val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( -// DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, -// nodeViewHolder.ref, -// peersKeeper.ref, -// nodeViewSync.ref, -// mempool.ref, -// None, -// settings) -// ) -// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) -// val peerHandler: TestProbe = TestProbe() -// val connectedPeer: ConnectedPeer = ConnectedPeer( -// address, -// peerHandler.ref, -// Outgoing, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test node", -// Some(address), -// System.currentTimeMillis()) -// ) -// -// val (history, _) = generateBlocks(200, generateDummyHistory(testNetSettings)) -// downloadedModifiersValidator ! UpdatedHistory(history) -// val invalidHeader = generateGenesisBlock(Height @@ 1) -// -// val mods: Map[ModifierId, Array[Byte]] = List(invalidHeader) -// .map( -// b => b.header.id -> HeaderProtoSerializer.toProto(b.header).toByteArray -// ) -// .toMap -// import scala.concurrent.duration._ -// downloadedModifiersValidator ! ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods) -// peersKeeper.expectMsgPF(10.seconds) { -// case BanPeer(connected, _) => connected == connectedPeer -// } -// val validHeightHeader = generateGenesisBlock(Height @@ 200) -// -// val mods1: Map[ModifierId, Array[Byte]] = List(validHeightHeader) -// .map( -// b => b.header.id -> HeaderProtoSerializer.toProto(b.header).toByteArray -// ) -// .toMap -// downloadedModifiersValidator ! ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods1) -// nodeViewHolder.expectMsgPF(10.seconds) { -// case ModifierFromRemote(mod) => mod == validHeightHeader.header -// } -// } -// "find corrupted header" in { -// val nodeViewHolder = TestProbe() -// val peersKeeper = TestProbe() -// val deliveryManager = TestProbe() -// val nodeViewSync = TestProbe() -// val mempool = TestProbe() -// -// val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( -// DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, -// nodeViewHolder.ref, -// peersKeeper.ref, -// nodeViewSync.ref, -// mempool.ref, -// None, -// settings) -// ) -// val history: History = generateDummyHistory(testNetSettings) -// -// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) -// val peerHandler: TestProbe = TestProbe() -// val connectedPeer: ConnectedPeer = ConnectedPeer( -// address, -// peerHandler.ref, -// Outgoing, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test node", -// Some(address), -// System.currentTimeMillis()) -// ) -// -// val timestamp1 = System.currentTimeMillis() -// Thread.sleep(1000) -// val timestamp2 = System.currentTimeMillis() -// -// val header_first: Header = Header( -// 1.toByte, -// ModifierId @@ Random.randomBytes(), -// Digest32 @@ Random.randomBytes(), -// timestamp2, -// 2, -// scala.util.Random.nextLong(), -// testNetSettings.constants.InitialDifficulty, -// EquihashSolution(Seq(1, 3)), -// Random.randomBytes() -// ) -// val header_second: Header = Header( -// 1.toByte, -// header_first.id, -// Digest32 @@ Random.randomBytes(), -// timestamp1, -// 1, -// scala.util.Random.nextLong(), -// testNetSettings.constants.InitialDifficulty, -// EquihashSolution(Seq(1, 3)), -// Random.randomBytes() -// ) -// -// history.append(header_first) -// -// nodeViewSync.send(downloadedModifiersValidator, UpdatedHistory(history)) -// -// /* Header */ -// val mods = Seq(header_second).map(x => x.id -> HeaderProtoSerializer.toProto(x).toByteArray.reverse).toMap -// val msg = ModifiersForValidating(connectedPeer, Header.modifierTypeId, mods) -// -// deliveryManager.send(downloadedModifiersValidator, msg) -// peersKeeper.expectMsg(BanPeer(connectedPeer, CorruptedSerializedBytes)) -// nodeViewHolder.expectNoMsg() -// nodeViewSync.expectMsg(InvalidModifier(header_second.id)) -// } -// "find corrupted payload" in { -// val nodeViewHolder = TestProbe() -// val peersKeeper = TestProbe() -// val deliveryManager = TestProbe() -// val nodeViewSync = TestProbe() -// val mempool = TestProbe() -// -// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9000) -// val peerHandler: TestProbe = TestProbe() -// val connectedPeer: ConnectedPeer = ConnectedPeer( -// address, -// peerHandler.ref, -// Outgoing, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test node", -// Some(address), -// System.currentTimeMillis()) -// ) -// -// val downloadedModifiersValidator = TestActorRef[DownloadedModifiersValidator]( -// DownloadedModifiersValidator.props(testNetSettings.constants.ModifierIdSize, -// nodeViewHolder.ref, -// peersKeeper.ref, -// nodeViewSync.ref, -// mempool.ref, -// None, -// settings) -// ) -// val history: History = generateDummyHistory(testNetSettings) -// -// val historyWith10Blocks = (0 until 10).foldLeft(history, Seq.empty[Block]) { -// case ((prevHistory, blocks), _) => -// val block: Block = generateNextBlock(prevHistory) -// prevHistory.append(block.header) -// prevHistory.append(block.payload) -// (prevHistory.reportModifierIsValid(block), blocks :+ block) -// } -// -// val payload = Payload(ModifierId @@ scorex.utils.Random.randomBytes(), Seq(coinbaseTransaction)) -// -// nodeViewSync.send(downloadedModifiersValidator, UpdatedHistory(historyWith10Blocks._1)) -// -// val bytes = PayloadProtoSerializer.toProto(payload).toByteArray -// -// val mods: Map[ModifierId, Array[Byte]] = (historyWith10Blocks._2.map( -// b => b.payload.id -> PayloadProtoSerializer.toProto(b.payload).toByteArray.reverse -// ) :+ (payload.id -> bytes)).toMap -// -// deliveryManager -// .send(downloadedModifiersValidator, ModifiersForValidating(connectedPeer, Payload.modifierTypeId, mods)) -// -// peersKeeper.expectMsg(BanPeer(connectedPeer, CorruptedSerializedBytes)) -// nodeViewHolder.expectMsg(ModifierFromRemote(payload)) -// } -// } -// -// def generateBlocks(qty: Int, history: History): (History, List[Block]) = -// (0 until qty).foldLeft(history, List.empty[Block]) { -// case ((prevHistory, blocks), _) => -// val block: Block = generateNextBlock(prevHistory) -// prevHistory.append(block.header) -// prevHistory.append(block.payload) -// val a = prevHistory.reportModifierIsValid(block) -// (a, blocks :+ block) -// } -//} diff --git a/src/test/scala/encry/nvg/ValidatorTests.scala b/src/test/scala/encry/nvg/ValidatorTests.scala index 72bcac2eed..83af8da7f7 100644 --- a/src/test/scala/encry/nvg/ValidatorTests.scala +++ b/src/test/scala/encry/nvg/ValidatorTests.scala @@ -1,5 +1,184 @@ package encry.nvg -class ValidatorTests { +import java.net.InetSocketAddress + +import akka.actor.{ ActorRef, ActorSystem } +import akka.testkit.{ TestKit, TestProbe } +import encry.modifiers.InstanceFactory +import encry.network.BlackList.BanReason.{ + CorruptedSerializedBytes, + ModifierIdInTheNetworkMessageIsNotTheSameAsIdOfModifierInThisMessage, + PreSemanticInvalidModifier, + SyntacticallyInvalidPersistentModifier +} +import encry.network.PeersKeeper.BanPeer +import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation, ValidatedModifier } +import encry.nvg.NodeViewHolder.SyntacticallyFailedModification +import encry.view.history.HistoryReader +import org.encryfoundation.common.modifiers.history.{ Block, Header, HeaderProtoSerializer } +import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } +import scorex.utils.Random + +class ValidatorTests + extends TestKit(ActorSystem("Tested-Akka-System")) + with WordSpecLike + with Matchers + with BeforeAndAfterAll + with InstanceFactory + with OneInstancePerTest { + + override def afterAll(): Unit = system.terminate() + + "Modifiers validator" should { + "notify intermediary actor about modifier with invalid raw bytes" in { + val nvh = TestProbe() + + val intermediary = TestProbe() + + val parentActor = TestProbe() + + val networkProcessor: ActorRef = + parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) + + val (history, blocks) = NodeViewNMProcessorTests.formHistory + + val reader = HistoryReader(history) + + val corruptedBytes = Random.randomBytes(190) + + val remote = new InetSocketAddress("0.0.0.0", 9001) + + val randomId = ModifierId @@ Random.randomBytes() + + networkProcessor ! ModifierForValidation(reader, randomId, Header.modifierTypeId, corruptedBytes, remote) + + intermediary.expectMsgPF() { + case BanPeer(r, CorruptedSerializedBytes) => r shouldBe remote + case InvalidModifierBytes(id) => id.sameElements(blocks.head.id) shouldBe true + } + } + "notify intermediary actor about pre semantic invalid modifier" in { + val nvh = TestProbe() + + val intermediary = TestProbe() + + val parentActor = TestProbe() + + val networkProcessor: ActorRef = + parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) + + val (history, blocks) = NodeViewNMProcessorTests.formHistory + + val reader = HistoryReader(history) + + val corruptedBlock = blocks.head.copy( + header = blocks.head.header.copy(height = -1000) + ) + + val corruptedBytes = HeaderProtoSerializer.toProto(corruptedBlock.header).toByteArray + + val remote = new InetSocketAddress("0.0.0.0", 9001) + + networkProcessor ! ModifierForValidation(reader, + ModifierId @@ Random.randomBytes(), + Header.modifierTypeId, + corruptedBytes, + remote) + + intermediary.expectMsgPF() { + case BanPeer(r, PreSemanticInvalidModifier(_)) => r shouldBe remote + case SyntacticallyFailedModification(mod, _) => mod.id.sameElements(blocks.head.id) shouldBe true + } + } + "notify intermediary actor about syntactically invalid modifier" in { + val nvh = TestProbe() + + val intermediary = TestProbe() + + val parentActor = TestProbe() + + val networkProcessor: ActorRef = + parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) + + val (history, blocks) = NodeViewNMProcessorTests.formHistory + + val reader = HistoryReader(history) + + val corruptedBlock = blocks.head.copy( + header = blocks.head.header.copy(parentId = ModifierId @@ blocks.head.header.id.drop(2)) + ) + + val corruptedBytes = HeaderProtoSerializer.toProto(corruptedBlock.header).toByteArray + + val remote = new InetSocketAddress("0.0.0.0", 9001) + + networkProcessor ! ModifierForValidation(reader, + ModifierId @@ Random.randomBytes(), + Header.modifierTypeId, + corruptedBytes, + remote) + + intermediary.expectMsgPF() { + case BanPeer(r, SyntacticallyInvalidPersistentModifier) => r shouldBe remote + case SyntacticallyFailedModification(mod, _) => mod.id.sameElements(blocks.head.id) shouldBe true + } + } + "notify intermediary about incorrect modifier id" in { + val nvh = TestProbe() + + val intermediary = TestProbe() + + val parentActor = TestProbe() + + val networkProcessor: ActorRef = + parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) + + val (history, blocks) = NodeViewNMProcessorTests.formHistory + + val reader = HistoryReader(history) + + val corruptedBytes = HeaderProtoSerializer.toProto(blocks.head.header).toByteArray + + val remote = new InetSocketAddress("0.0.0.0", 9001) + + networkProcessor ! ModifierForValidation(reader, + ModifierId @@ Random.randomBytes(), + Header.modifierTypeId, + corruptedBytes, + remote) + + intermediary.expectMsgPF() { + case BanPeer(r, ModifierIdInTheNetworkMessageIsNotTheSameAsIdOfModifierInThisMessage) => r shouldBe remote + case SyntacticallyFailedModification(mod, _) => mod.id.sameElements(blocks.head.id) shouldBe true + } + } + "notify nvh actor about valid modifier" in { + val nvh = TestProbe() + + val intermediary = TestProbe() + + val parentActor = TestProbe() + + val networkProcessor: ActorRef = + parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) + + val (history, _) = NodeViewNMProcessorTests.formHistory + + val reader = HistoryReader(history) + + val correctBlock: Block = generateNextBlock(history) + + val correctBytes = HeaderProtoSerializer.toProto(correctBlock.header).toByteArray + + val remote = new InetSocketAddress("0.0.0.0", 9001) + + networkProcessor ! ModifierForValidation(reader, correctBlock.id, Header.modifierTypeId, correctBytes, remote) + + nvh.expectMsgPF() { + case ValidatedModifier(mod) => mod.id.sameElements(correctBlock.id) shouldBe true + } + } + } } From f793d13dbcfae0d7e782882fcec143c9d8fec8bc Mon Sep 17 00:00:00 2001 From: aleksandr Date: Wed, 18 Mar 2020 09:26:19 +0300 Subject: [PATCH 105/177] add test for pk pipeline --- src/main/scala/encry/network/PK.scala | 28 +- .../network/ConnectWithNewPeerTests.scala | 306 +++++++++--------- 2 files changed, 174 insertions(+), 160 deletions(-) diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 3c2efb1c5a..9d84c09143 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -33,8 +33,8 @@ class PK(networkSettings: NetworkSettings, var blackList: BlackList = BlackList(blacklistSettings) - var knownPeers: Set[InetAddress] = networkSettings.knownPeers - .collect { case peer: InetSocketAddress if !isSelf(peer) => peer.getAddress }.toSet + var knownPeers: Set[InetSocketAddress] = networkSettings.knownPeers + .collect { case peer: InetSocketAddress if !isSelf(peer) => peer }.toSet var outgoingConnections: Set[InetSocketAddress] = Set.empty @@ -48,15 +48,6 @@ class PK(networkSettings: NetworkSettings, PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" ), self) - context.system.scheduler.schedule(5.seconds, 5.seconds) { - context.parent ! UpdatingPeersInfo( - peersForConnection.keys.toList, - connectedPeers.collect[ConnectedPeer](getAllPeers, getConnectedPeers).map(peer => - (peer.socketAddress, peer.handshake.nodeName, peer.direction) - ).toList, - blackList.getAll.toList - ) - } context.system.scheduler.schedule(600.millis, blacklistSettings.cleanupTime){blackList = blackList.cleanupBlackList} } @@ -100,14 +91,16 @@ class PK(networkSettings: NetworkSettings, outgoingConnections -= remote sender() ! ConnectionVerified(remote, remoteConnection, Outgoing) } - else if (connectWithOnlyKnownPeers && knownPeers.contains(remote.getAddress)) { + else if (connectWithOnlyKnownPeers && knownPeers.contains(remote)) { logger.info(s"connectWithOnlyKnownPeers - true, but connected peer is contained in known peers collection.") + awaitingHandshakeConnections += remote sender() ! ConnectionVerified(remote, remoteConnection, Incoming) } else if (connectWithOnlyKnownPeers) logger.info(s"Got incoming connection but we can connect only with known peers.") else { logger.info(s"Got new incoming connection. Sending to network controller approvement for connect.") + awaitingHandshakeConnections += remote sender() ! ConnectionVerified(remote, remoteConnection, Incoming) } } else logger.info(s"Connection for requested peer: $remote is unavailable cause of:" + @@ -126,7 +119,7 @@ class PK(networkSettings: NetworkSettings, peersForConnection = peersForConnection.updated(connectedPeer.socketAddress, 0) logger.info(s"Adding new peer: ${connectedPeer.socketAddress} to available collection." + s" Current collection is: ${peersForConnection.keys.mkString(",")}.") - + updatePeersCollection() case ConnectionStopped(peer) => logger.info(s"Connection stopped for: $peer.") awaitingHandshakeConnections -= peer @@ -136,6 +129,7 @@ class PK(networkSettings: NetworkSettings, logger.info(s"Peer: $peer removed from availablePeers cause of it has been banned. " + s"Current is: ${peersForConnection.mkString(",")}.") } + updatePeersCollection() case predicate: GetPeerByPredicate => connectedPeers.getAll.find { case (_, info) => predicate.predicate(info) }.map { @@ -197,6 +191,14 @@ class PK(networkSettings: NetworkSettings, InetAddress.getLocalHost.getAddress.sameElements(address.getAddress.getAddress) || InetAddress.getLoopbackAddress.getAddress.sameElements(address.getAddress.getAddress)).getOrElse(true) + def updatePeersCollection(): Unit = context.parent ! UpdatingPeersInfo( + peersForConnection.keys.toList, + connectedPeers.collect[ConnectedPeer](getAllPeers, getConnectedPeers).map(peer => + (peer.socketAddress, peer.handshake.nodeName, peer.direction) + ).toList, + blackList.getAll.toList + ) + def getAllPeers: (InetSocketAddress, PeerInfo) => Boolean = (_, _) => true def getConnectedPeers(add: InetSocketAddress, info: PeerInfo): ConnectedPeer = info.connectedPeer } diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index 5f8160f949..3f40526557 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -2,164 +2,176 @@ package encry.network import java.net.InetSocketAddress -import akka.actor.ActorSystem +import akka.actor.{ActorRef, ActorSystem} import akka.testkit.{TestActorRef, TestProbe} +import encry.api.http.DataHolderForApi.UpdatingPeersInfo import encry.modifiers.InstanceFactory import encry.network.BlackList.BanReason._ -import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} +import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming, Outgoing} +import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} import encry.network.PeersKeeper._ import encry.settings.TestNetSettings -import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, PeersNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{GetPeersNetworkMessage, Handshake, PeersNetworkMessage} import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} + import scala.concurrent.duration._ -//class ConnectWithNewPeerTests extends WordSpecLike -// with Matchers -// with BeforeAndAfterAll -// with InstanceFactory -// with OneInstancePerTest -// with TestNetSettings { +class ConnectWithNewPeerTests extends WordSpecLike + with Matchers + with BeforeAndAfterAll + with InstanceFactory + with OneInstancePerTest + with TestNetSettings { + + implicit val system: ActorSystem = ActorSystem() + + override def afterAll(): Unit = system.terminate() + + val knowPeersSettings = testNetSettings.copy( + network = testNetSettings.network.copy( + knownPeers = List(new InetSocketAddress("172.16.11.11", 9001)), + connectOnlyWithKnownPeers = Some(true) + ), + blackList = testNetSettings.blackList.copy( + banTime = 2 seconds, + cleanupTime = 3 seconds + )) + + "Peers keeper" should { + "maintain outgoing connection process correctly" in { + /* Request first peer while current number of connections is 0 */ + val networkController: TestProbe = TestProbe() + val nodeViewSync = TestProbe() + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2), settings.blackList), + networkController.ref + ) + + val availablePeers: Set[InetSocketAddress] = peersKeeper.underlyingActor.knownPeers + + networkController.send(peersKeeper, RequestPeerForConnection) + println(peersKeeper.underlyingActor.knownPeers.map(PeerForConnection)) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + val msg = networkController.expectMsgType[PeerForConnection] + peersKeeper.underlyingActor.outgoingConnections.contains(msg.peer) shouldBe true + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(msg.peer) shouldBe true + + val remoteAkkaConnectionHandler = TestProbe() + + networkController.send(peersKeeper, NewConnection(msg.peer, remoteAkkaConnectionHandler.ref)) + networkController.expectMsg(ConnectionVerified(msg.peer, remoteAkkaConnectionHandler.ref, Outgoing)) + peersKeeper.underlyingActor.outgoingConnections.contains(msg.peer) shouldBe false + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(msg.peer) shouldBe true + + val peerHandler = TestProbe() + val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head, peerHandler.ref, Outgoing, + Handshake(protocolToBytes(settings.network.appVersion), + "test-peer", Some(availablePeers.head), System.currentTimeMillis())) + + networkController.send(peersKeeper, HandshakedDone(connectedPeer)) + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head) shouldBe false + peersKeeper.underlyingActor.knownPeers.contains(availablePeers.head) shouldBe true + networkController.expectMsgType[UpdatingPeersInfo] + + /* Request next peer after first connection setup */ + + val newAvailablePeers: Set[InetSocketAddress] = peersKeeper.underlyingActor.knownPeers.drop(1) + + networkController.send(peersKeeper, RequestPeerForConnection) + val nextPeerForConnectionMsg = networkController.expectMsgType[PeerForConnection] + peersKeeper.underlyingActor.outgoingConnections.contains(nextPeerForConnectionMsg.peer) shouldBe true + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(nextPeerForConnectionMsg.peer) shouldBe true + + networkController.send(peersKeeper, NewConnection(newAvailablePeers.filter(_ != nextPeerForConnectionMsg.peer).head, remoteAkkaConnectionHandler.ref)) + networkController.expectMsg(ConnectionVerified(newAvailablePeers.filter(_ != nextPeerForConnectionMsg.peer).head, remoteAkkaConnectionHandler.ref, Incoming)) + peersKeeper.underlyingActor.outgoingConnections.contains(newAvailablePeers.filter(_ != nextPeerForConnectionMsg.peer).head) shouldBe false + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(newAvailablePeers.filter(_ != nextPeerForConnectionMsg.peer).head) shouldBe true + + val newPeerHandler = TestProbe() + val newConnectedPeer: ConnectedPeer = ConnectedPeer(newAvailablePeers.filter(_ != nextPeerForConnectionMsg.peer).head, newPeerHandler.ref, Outgoing, + Handshake(protocolToBytes(settings.network.appVersion), + "test-peer_new", Some(newAvailablePeers.filter(_ != nextPeerForConnectionMsg.peer).head), System.currentTimeMillis())) + + networkController.send(peersKeeper, HandshakedDone(newConnectedPeer)) + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(newAvailablePeers.filter(_ != nextPeerForConnectionMsg.peer).head) shouldBe false + peersKeeper.underlyingActor.knownPeers.contains(newAvailablePeers.filter(_ != nextPeerForConnectionMsg.peer).head) shouldBe true + networkController.expectMsgType[UpdatingPeersInfo] + /* Try to ask one more peer while max number of connections has been expired */ + + networkController.send(peersKeeper, RequestPeerForConnection) + networkController.expectNoMsg() + + /* Now we will ban one peer */ + + val actorWhichSendBanMessage = TestProbe() + + actorWhichSendBanMessage.send(peersKeeper, BanPeer(newConnectedPeer.socketAddress, ExpiredNumberOfConnections)) + newPeerHandler.expectMsgAnyOf(CloseConnection) + peersKeeper.underlyingActor.blackList.contains(newConnectedPeer.socketAddress.getAddress) shouldBe true + networkController.send(peersKeeper, ConnectionStopped(newConnectedPeer.socketAddress)) + peersKeeper.underlyingActor.connectedPeers.contains(newConnectedPeer.socketAddress) shouldBe false + networkController.expectMsgType[UpdatingPeersInfo] + + /* Try to setup Incoming connection from banned peer */ + + networkController.send(peersKeeper, NewConnection(newConnectedPeer.socketAddress, remoteAkkaConnectionHandler.ref)) + networkController.expectNoMsg() + + /* Try to request new connection */ + + val updatedAvailablePeers: Set[InetSocketAddress] = peersKeeper.underlyingActor.knownPeers.filter(peer => + peer != msg.peer && peer != newConnectedPeer.socketAddress + ) + + networkController.send(peersKeeper, RequestPeerForConnection) + val oneMorePeer = networkController.expectMsgType[PeerForConnection] + peersKeeper.underlyingActor.outgoingConnections.contains(oneMorePeer.peer) shouldBe true + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(oneMorePeer.peer) shouldBe true + + val updatedRemoteAkkaConnectionHandler = TestProbe() + + networkController.send(peersKeeper, NewConnection(oneMorePeer.peer, updatedRemoteAkkaConnectionHandler.ref)) + networkController.expectMsg(ConnectionVerified(oneMorePeer.peer, updatedRemoteAkkaConnectionHandler.ref, Outgoing)) + peersKeeper.underlyingActor.outgoingConnections.contains(oneMorePeer.peer) shouldBe false + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(oneMorePeer.peer) shouldBe true + + val updatedConnectedPeer: ConnectedPeer = ConnectedPeer(oneMorePeer.peer, updatedRemoteAkkaConnectionHandler.ref, Outgoing, + Handshake(protocolToBytes(settings.network.appVersion), + "test-peer", Some(oneMorePeer.peer), System.currentTimeMillis())) + + networkController.send(peersKeeper, HandshakedDone(updatedConnectedPeer)) + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(oneMorePeer.peer) shouldBe false + peersKeeper.underlyingActor.knownPeers.contains(oneMorePeer.peer) shouldBe true + } +// "remove peer from available we can't connect to" in { +// val networkController = TestProbe() +// val nodeViewSync = TestProbe() +// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettingsWithAllPeers, nodeViewSync.ref, TestProbe().ref)) +// +// val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.knownPeers // -// implicit val system: ActorSystem = ActorSystem() +// networkController.send(peersKeeper, RequestPeerForConnection) +// networkController.expectMsg(PeerForConnection(availablePeers.head._1)) // -// override def afterAll(): Unit = system.terminate() +// networkController.send(peersKeeper, OutgoingConnectionFailed(availablePeers.head._1)) +// peersKeeper.underlyingActor.outgoingConnections.contains(availablePeers.head._1) shouldBe false +// peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe false +// peersKeeper.underlyingActor.knownPeers.get(availablePeers.head._1) shouldBe Some(1) // -// val knowPeersSettings = testNetSettings.copy( -// network = testNetSettings.network.copy( -// knownPeers = List(new InetSocketAddress("172.16.11.11", 9001)), -// connectOnlyWithKnownPeers = Some(true) -// ), -// blackList = testNetSettings.blackList.copy( -// banTime = 2 seconds, -// cleanupTime = 3 seconds -// )) +// networkController.send(peersKeeper, RequestPeerForConnection) +// networkController.expectMsg(PeerForConnection(availablePeers.head._1)) // -// "Peers keeper" should { - // "maintain outgoing connection process correctly" in { - // /* Request first peer while current number of connections is 0 */ - // val networkController = TestProbe() - // val nodeViewSync = TestProbe() - // val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettingsWithAllPeers, nodeViewSync.ref, TestProbe().ref)) - // - // val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.knownPeers - // - // networkController.send(peersKeeper, RequestPeerForConnection) - // networkController.expectMsg(PeerForConnection(availablePeers.head._1)) - // peersKeeper.underlyingActor.outgoingConnections.contains(availablePeers.head._1) shouldBe true - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe true - // - // val remoteAkkaConnectionHandler = TestProbe() - // - // networkController.send(peersKeeper, VerifyConnection(availablePeers.head._1, remoteAkkaConnectionHandler.ref)) - // networkController.expectMsg(ConnectionVerified(availablePeers.head._1, remoteAkkaConnectionHandler.ref, Outgoing)) - // peersKeeper.underlyingActor.outgoingConnections.contains(availablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe true - // - // val peerHandler = TestProbe() - // val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, - // Handshake(protocolToBytes(testNetSettingsWithAllPeers.network.appVersion), - // "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) - // - // networkController.send(peersKeeper, HandshakedDone(connectedPeer)) - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.knownPeers.contains(availablePeers.head._1) shouldBe true - // peersKeeper.underlyingActor.knownPeers.get(availablePeers.head._1) shouldBe Some(0) - // - // /* Request next peer after first connection setup */ - // - // val newAvailablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.knownPeers.drop(1) - // - // networkController.send(peersKeeper, RequestPeerForConnection) - // networkController.expectMsg(PeerForConnection(newAvailablePeers.head._1)) - // peersKeeper.underlyingActor.outgoingConnections.contains(newAvailablePeers.head._1) shouldBe true - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(newAvailablePeers.head._1) shouldBe true - // - // networkController.send(peersKeeper, VerifyConnection(newAvailablePeers.head._1, remoteAkkaConnectionHandler.ref)) - // networkController.expectMsg(ConnectionVerified(newAvailablePeers.head._1, remoteAkkaConnectionHandler.ref, Outgoing)) - // peersKeeper.underlyingActor.outgoingConnections.contains(newAvailablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(newAvailablePeers.head._1) shouldBe true - // - // val newPeerHandler = TestProbe() - // val newConnectedPeer: ConnectedPeer = ConnectedPeer(newAvailablePeers.head._1, newPeerHandler.ref, Outgoing, - // Handshake(protocolToBytes(testNetSettingsWithAllPeers.network.appVersion), - // "test-peer_new", Some(newAvailablePeers.head._1), System.currentTimeMillis())) - // - // networkController.send(peersKeeper, HandshakedDone(newConnectedPeer)) - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(newAvailablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.knownPeers.contains(newAvailablePeers.head._1) shouldBe true - // peersKeeper.underlyingActor.knownPeers.get(newAvailablePeers.head._1) shouldBe Some(0) - // - // /* Try to ask one more peer while max number of connections has been expired */ - // - // networkController.send(peersKeeper, RequestPeerForConnection) - // networkController.expectNoMsg() - // - // /* Now we will ban one peer */ - // - // val actorWhichSendBanMessage = TestProbe() - // - // actorWhichSendBanMessage.send(peersKeeper, BanPeer(newConnectedPeer, ExpiredNumberOfConnections)) - // newPeerHandler.expectMsgAnyOf(CloseConnection, GetPeersNetworkMessage) - // peersKeeper.underlyingActor.blackList.contains(newConnectedPeer.socketAddress.getAddress) shouldBe true - // networkController.send(peersKeeper, ConnectionStopped(newConnectedPeer.socketAddress)) - // peersKeeper.underlyingActor.knownPeers.contains(newConnectedPeer.socketAddress) shouldBe false - // peersKeeper.underlyingActor.connectedPeers.contains(newConnectedPeer.socketAddress) shouldBe false - // - // /* Try to setup Incoming connection from banned peer */ - // - // networkController.send(peersKeeper, VerifyConnection(newConnectedPeer.socketAddress, remoteAkkaConnectionHandler.ref)) - // networkController.expectNoMsg() - // - // /* Try to request new connection */ - // - // val updatedAvailablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.knownPeers.takeRight(1) - // - // networkController.send(peersKeeper, RequestPeerForConnection) - // networkController.expectMsg(PeerForConnection(updatedAvailablePeers.head._1)) - // peersKeeper.underlyingActor.outgoingConnections.contains(updatedAvailablePeers.head._1) shouldBe true - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(updatedAvailablePeers.head._1) shouldBe true - // - // val updatedRemoteAkkaConnectionHandler = TestProbe() - // - // networkController.send(peersKeeper, VerifyConnection(updatedAvailablePeers.head._1, updatedRemoteAkkaConnectionHandler.ref)) - // networkController.expectMsg(ConnectionVerified(updatedAvailablePeers.head._1, updatedRemoteAkkaConnectionHandler.ref, Outgoing)) - // peersKeeper.underlyingActor.outgoingConnections.contains(updatedAvailablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(updatedAvailablePeers.head._1) shouldBe true - // - // val updatedConnectedPeer: ConnectedPeer = ConnectedPeer(updatedAvailablePeers.head._1, updatedRemoteAkkaConnectionHandler.ref, Outgoing, - // Handshake(protocolToBytes(testNetSettingsWithAllPeers.network.appVersion), - // "test-peer", Some(updatedAvailablePeers.head._1), System.currentTimeMillis())) - // - // networkController.send(peersKeeper, HandshakedDone(updatedConnectedPeer)) - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(updatedAvailablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.knownPeers.contains(updatedAvailablePeers.head._1) shouldBe true - // peersKeeper.underlyingActor.knownPeers.get(updatedAvailablePeers.head._1) shouldBe Some(0) - // } - // "remove peer from available we can't connect to" in { - // val networkController = TestProbe() - // val nodeViewSync = TestProbe() - // val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettingsWithAllPeers, nodeViewSync.ref, TestProbe().ref)) - // - // val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.knownPeers - // - // networkController.send(peersKeeper, RequestPeerForConnection) - // networkController.expectMsg(PeerForConnection(availablePeers.head._1)) - // - // networkController.send(peersKeeper, OutgoingConnectionFailed(availablePeers.head._1)) - // peersKeeper.underlyingActor.outgoingConnections.contains(availablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.knownPeers.get(availablePeers.head._1) shouldBe Some(1) - // - // networkController.send(peersKeeper, RequestPeerForConnection) - // networkController.expectMsg(PeerForConnection(availablePeers.head._1)) - // - // networkController.send(peersKeeper, OutgoingConnectionFailed(availablePeers.head._1)) - // peersKeeper.underlyingActor.outgoingConnections.contains(availablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.knownPeers.contains(availablePeers.head._1) shouldBe false - // peersKeeper.underlyingActor.blackList.contains(availablePeers.head._1.getAddress) shouldBe true - // } +// networkController.send(peersKeeper, OutgoingConnectionFailed(availablePeers.head._1)) +// peersKeeper.underlyingActor.outgoingConnections.contains(availablePeers.head._1) shouldBe false +// peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe false +// peersKeeper.underlyingActor.knownPeers.contains(availablePeers.head._1) shouldBe false +// peersKeeper.underlyingActor.blackList.contains(availablePeers.head._1.getAddress) shouldBe true +// } // "remove peer from available if it has been banned" in { // val networkController = TestProbe() // val nodeViewSync = TestProbe() @@ -340,5 +352,5 @@ import scala.concurrent.duration._ // networkController.expectMsg( // ConnectionVerified(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref, Outgoing)) // } -// } -//} \ No newline at end of file + } +} \ No newline at end of file From 117dc271c2ac59fc26d2d94d2bac68f18eeb8f07 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 18 Mar 2020 10:12:25 +0300 Subject: [PATCH 106/177] stats added --- .../scala/encry/nvg/ModifiersValidator.scala | 2 + .../scala/encry/nvg/NodeViewNMProcessor.scala | 2 +- src/main/scala/encry/stats/StatsSender.scala | 149 ++++++++++-------- 3 files changed, 87 insertions(+), 66 deletions(-) diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index 4814b7736f..d91a97e66b 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -19,6 +19,7 @@ import encry.network.PeersKeeper.BanPeer import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation, ValidatedModifier } import encry.nvg.NodeViewHolder.SyntacticallyFailedModification import encry.settings.EncryAppSettings +import encry.stats.StatsSender.ValidatedModifierFromNetwork import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Header, HeaderProtoSerializer, Payload, PayloadProtoSerializer } @@ -49,6 +50,7 @@ class ModifiersValidator( if (preSemanticValidation.isRight && syntacticValidation) { if (modifier.id.sameElements(id)) { logger.debug(s"Modifier ${modifier.encodedId} is valid.") + intermediaryNVH ! ValidatedModifierFromNetwork(modifierTypeId) nodeViewHolderRef ! ValidatedModifier(modifier) } else { logger.info(s"Modifier ${modifier.encodedId} should have ${Algos.encode(id)} id!") diff --git a/src/main/scala/encry/nvg/NodeViewNMProcessor.scala b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala index 77b63d1e04..693cab0618 100644 --- a/src/main/scala/encry/nvg/NodeViewNMProcessor.scala +++ b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala @@ -65,7 +65,7 @@ class NodeViewNMProcessor(settings: EncryAppSettings) extends Actor with StrictL case DataFromPeer(SyncInfoNetworkMessage(syncInfo: SyncInfo), remote) => val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) logger.info( - s"\n\n Comparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}.\n" + + s"\n\nComparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}.\n" + s"Comparison result is $comparison.\n " ) context.parent ! OtherNodeSyncingStatus(remote, comparison) diff --git a/src/main/scala/encry/stats/StatsSender.scala b/src/main/scala/encry/stats/StatsSender.scala index deae07515f..3295f2c3c1 100644 --- a/src/main/scala/encry/stats/StatsSender.scala +++ b/src/main/scala/encry/stats/StatsSender.scala @@ -4,27 +4,29 @@ import java.io.File import java.net.InetAddress import java.util import java.text.SimpleDateFormat -import akka.actor.{Actor, Props} +import akka.actor.{ Actor, Props } import com.typesafe.scalalogging.StrictLogging import encry.EncryApp.timeProvider import encry.consensus.EncrySupplyController -import encry.settings.{InfluxDBSettings, NetworkSettings} +import encry.settings.{ InfluxDBSettings, NetworkSettings } import encry.stats.StatsSender._ import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{Height, ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId, ModifierTypeId } import org.encryfoundation.common.utils.constants.Constants -import org.influxdb.{InfluxDB, InfluxDBFactory} +import org.influxdb.{ InfluxDB, InfluxDBFactory } import scala.concurrent.ExecutionContext.Implicits.global -class StatsSender(influxDBSettings: InfluxDBSettings, networkSettings: NetworkSettings, constants: Constants) extends Actor with StrictLogging { +class StatsSender(influxDBSettings: InfluxDBSettings, networkSettings: NetworkSettings, constants: Constants) + extends Actor + with StrictLogging { var modifiersToDownload: Map[String, (ModifierTypeId, Long)] = Map.empty - var modifiersToApply: Map[String, (ModifierTypeId, Long)] = Map.empty + var modifiersToApply: Map[String, (ModifierTypeId, Long)] = Map.empty val nodeName: String = networkSettings.nodeName match { case Some(value) => value - case None => InetAddress.getLocalHost.getHostAddress + ":" + networkSettings.bindAddress.getPort + case None => InetAddress.getLocalHost.getHostAddress + ":" + networkSettings.bindAddress.getPort } val influxDB: InfluxDB = InfluxDBFactory .connect(influxDBSettings.url, influxDBSettings.login, influxDBSettings.password) @@ -47,7 +49,8 @@ class StatsSender(influxDBSettings: InfluxDBSettings, networkSettings: NetworkSe s"value=${new File("encry/data/history/").listFiles.foldLeft(0L)(_ + _.length())}", //++ s"supply,nodeName=$nodeName,height=${fb.height} " + s"value=${EncrySupplyController.supplyAt(fb.height.asInstanceOf[Height], constants)}" //++ - )) + ) + ) case HeightStatistics(bestHeaderHeight, bestBlockHeight) => influxDB.write( @@ -75,33 +78,41 @@ class StatsSender(influxDBSettings: InfluxDBSettings, networkSettings: NetworkSe case InfoAboutTransactionsFromMiner(qty) => influxDB.write(influxDBSettings.udpPort, s"infoAboutTxsFromMiner,nodeName=$nodeName value=$qty") - case GetModifiers(_, modifiers) => modifiers - .foreach(downloadedModifierId => - modifiersToDownload.get(Algos.encode(downloadedModifierId)).foreach { dowloadInfo => - influxDB.write( - influxDBSettings.udpPort, - s"modDownloadStat,nodeName=$nodeName,modId=${Algos.encode(downloadedModifierId)}," + - s"modType=${dowloadInfo._1} value=${System.currentTimeMillis() - dowloadInfo._2}" - ) - modifiersToDownload = modifiersToDownload - Algos.encode(downloadedModifierId) - } - ) + case GetModifiers(_, modifiers) => + modifiers + .foreach( + downloadedModifierId => + modifiersToDownload.get(Algos.encode(downloadedModifierId)).foreach { dowloadInfo => + influxDB.write( + influxDBSettings.udpPort, + s"modDownloadStat,nodeName=$nodeName,modId=${Algos.encode(downloadedModifierId)}," + + s"modType=${dowloadInfo._1} value=${System.currentTimeMillis() - dowloadInfo._2}" + ) + modifiersToDownload = modifiersToDownload - Algos.encode(downloadedModifierId) + } + ) - case MiningEnd(blockHeader, workerIdx, workersQty) => timeProvider - .time() - .map(time => influxDB.write( - influxDBSettings.udpPort, - util.Arrays.asList( - s"miningEnd,nodeName=$nodeName,block=${Algos.encode(blockHeader.id)}," + - s"height=${blockHeader.height},worker=$workerIdx value=${time - blockHeader.timestamp}", - s"minerIterCount,nodeName=$nodeName,block=${Algos.encode(blockHeader.id)}," + - s"height=${blockHeader.height} value=${blockHeader.nonce - Long.MaxValue / workersQty * workerIdx + 1}" - ))) + case MiningEnd(blockHeader, workerIdx, workersQty) => + timeProvider + .time() + .map( + time => + influxDB.write( + influxDBSettings.udpPort, + util.Arrays.asList( + s"miningEnd,nodeName=$nodeName,block=${Algos.encode(blockHeader.id)}," + + s"height=${blockHeader.height},worker=$workerIdx value=${time - blockHeader.timestamp}", + s"minerIterCount,nodeName=$nodeName,block=${Algos.encode(blockHeader.id)}," + + s"height=${blockHeader.height} value=${blockHeader.nonce - Long.MaxValue / workersQty * workerIdx + 1}" + ) + ) + ) case EndOfApplyingModifier(modifierId) => modifiersToApply.get(Algos.encode(modifierId)).foreach { modInfo => - influxDB.write(influxDBSettings.udpPort, s"modifApplying,nodeName=$nodeName," + - s"modType=${modInfo._1} value=${System.currentTimeMillis() - modInfo._2}") + influxDB.write(influxDBSettings.udpPort, + s"modifApplying,nodeName=$nodeName," + + s"modType=${modInfo._1} value=${System.currentTimeMillis() - modInfo._2}") modifiersToApply -= Algos.encode(modifierId) } @@ -112,65 +123,73 @@ class StatsSender(influxDBSettings: InfluxDBSettings, networkSettings: NetworkSe case SleepTime(time) => influxDB.write(influxDBSettings.udpPort, s"sleepTime,nodeName=$nodeName value=$time") - case StateUpdating(time) => influxDB.write(influxDBSettings.udpPort, s"stateUpdatingTime,nodeName=$nodeName value=$time") + case StateUpdating(time) => + influxDB.write(influxDBSettings.udpPort, s"stateUpdatingTime,nodeName=$nodeName value=$time") case AvlStat(storageInsert: Long, avlDeleteTime: Long, avlInsertTime: Long) => - influxDB.write(influxDBSettings.udpPort, + influxDB.write( + influxDBSettings.udpPort, s"avlStat,nodeName=$nodeName avlDelete=$avlDeleteTime,insertTime=$avlInsertTime,value=$storageInsert" ) case UtxoStat(txsNumber: Int, validationTime: Long) => - influxDB.write(influxDBSettings.udpPort, s"utxoStat,nodeName=$nodeName txsNumber=$txsNumber,value=$validationTime") - - case msg: ModifiersDownloadStatistic => msg match { - case _ if nodeName.exists(_.isDigit) => - val nodeNumber: Long = nodeName.filter(_.isDigit).toLong - val (isHeader: Boolean, tableName: String) = msg match { - case SerializedModifierFromNetwork(t) => - (t == Header.modifierTypeId) -> "serializedModifierFromNetwork" - case ValidatedModifierFromNetwork(t) => - (t == Header.modifierTypeId) -> "validatedModifierFromNetwork" - } - influxDB.write( - influxDBSettings.udpPort, - s"""$tableName,nodeName=$nodeNumber,isHeader=$isHeader value=$nodeNumber""" - ) - case _ => //do nothing - } + influxDB.write(influxDBSettings.udpPort, + s"utxoStat,nodeName=$nodeName txsNumber=$txsNumber,value=$validationTime") + + case msg: ModifiersDownloadStatistic => + msg match { + case _ if nodeName.exists(_.isDigit) => + val nodeNumber: Long = nodeName.filter(_.isDigit).toLong + val (isHeader: Boolean, tableName: String) = msg match { + case SerializedModifierFromNetwork(t) => + (t == Header.modifierTypeId) -> "serializedModifierFromNetwork" + case ValidatedModifierFromNetwork(t) => + (t == Header.modifierTypeId) -> "validatedModifierFromNetwork" + } + influxDB.write( + influxDBSettings.udpPort, + s"""$tableName,nodeName=$nodeNumber,isHeader=$isHeader value=$nodeNumber""" + ) + case _ => //do nothing + } case ModifierAppendedToHistory(_, _) => - case ModifierAppendedToState(_) => - + case ModifierAppendedToState(_) => case SendDownloadRequest(modifierTypeId: ModifierTypeId, modifiers: Seq[ModifierId]) => - modifiersToDownload = modifiersToDownload ++ modifiers.map(mod => (Algos.encode(mod), (modifierTypeId, System.currentTimeMillis()))) + modifiersToDownload = modifiersToDownload ++ modifiers.map( + mod => (Algos.encode(mod), (modifierTypeId, System.currentTimeMillis())) + ) } } object StatsSender { sealed trait StatsSenderMessage - final case class TransactionsInBlock(txsNum: Int) extends StatsSenderMessage - final case class BestHeaderInChain(bestHeader: Header) extends StatsSenderMessage - final case class HeightStatistics(bestHeaderHeight: Int, bestBlockHeight: Int) extends StatsSenderMessage + final case class TransactionsInBlock(txsNum: Int) extends StatsSenderMessage + final case class BestHeaderInChain(bestHeader: Header) extends StatsSenderMessage + final case class HeightStatistics(bestHeaderHeight: Int, bestBlockHeight: Int) extends StatsSenderMessage final case class ModifierAppendedToHistory(isHeader: Boolean, success: Boolean) extends StatsSenderMessage - final case class ModifierAppendedToState(success: Boolean) extends StatsSenderMessage - final case class InfoAboutTransactionsFromMiner(qty: Int) extends AnyVal - final case class EndOfApplyingModifier(modifierId: ModifierId) extends StatsSenderMessage - final case class StateUpdating(time: Long) extends StatsSenderMessage - final case class SleepTime(time: Long) extends AnyVal - final case class StartApplyingModifier(modifierId: ModifierId, modifierTypeId: ModifierTypeId, startTime: Long) extends StatsSenderMessage + final case class ModifierAppendedToState(success: Boolean) extends StatsSenderMessage + final case class InfoAboutTransactionsFromMiner(qty: Int) extends AnyVal + final case class EndOfApplyingModifier(modifierId: ModifierId) extends StatsSenderMessage + final case class StateUpdating(time: Long) extends StatsSenderMessage + final case class SleepTime(time: Long) extends AnyVal + final case class StartApplyingModifier(modifierId: ModifierId, modifierTypeId: ModifierTypeId, startTime: Long) + extends StatsSenderMessage final case class MiningEnd(blockHeader: Header, workerIdx: Int, workersQty: Int) final case class MiningTime(time: Long) extends AnyVal final case class SendDownloadRequest(modifierTypeId: ModifierTypeId, modifiers: Seq[ModifierId]) final case class GetModifiers(modifierTypeId: ModifierTypeId, modifiers: Seq[ModifierId]) sealed trait ModifiersDownloadStatistic final case class SerializedModifierFromNetwork(modifierTypeId: ModifierTypeId) extends ModifiersDownloadStatistic - final case class ValidatedModifierFromNetwork(modifierTypeId: ModifierTypeId) extends ModifiersDownloadStatistic + final case class ValidatedModifierFromNetwork(modifierTypeId: ModifierTypeId) + extends ModifiersDownloadStatistic + with StatsSenderMessage final case class AvlStat(storageInsert: Long, avlDeleteTime: Long, avlInsertTime: Long) final case class UtxoStat(txsNumber: Int, validationTime: Long) final case class NewHeightByHistory(height: Int) extends AnyVal - final case class NewHeightByState(height: Int) extends AnyVal + final case class NewHeightByState(height: Int) extends AnyVal def props(influxDBSettings: InfluxDBSettings, networkSettings: NetworkSettings, constants: Constants): Props = Props(new StatsSender(influxDBSettings, networkSettings, constants)) -} \ No newline at end of file +} From 73f3357dd19f68d640c508b869ae6e588f58cc3b Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 18 Mar 2020 10:12:51 +0300 Subject: [PATCH 107/177] fixed modifiers propagation --- src/main/scala/encry/nvg/ModifiersValidator.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index d91a97e66b..94ef506adf 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -25,7 +25,6 @@ import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Header, HeaderProtoSerializer, Payload, PayloadProtoSerializer } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } - import scala.util.Try class ModifiersValidator( From c427b7258e6d4108e81c7dd080f59325f4ac2a5c Mon Sep 17 00:00:00 2001 From: aleksandr Date: Wed, 18 Mar 2020 10:18:05 +0300 Subject: [PATCH 108/177] add NotifyNodeAboutModifier --- .../scala/encry/network/MessageBuilder.scala | 1 - src/main/scala/encry/network/Messages.scala | 3 + .../network/ConnectWithNewPeerTests.scala | 56 +++++++++---------- 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 3adeba3341..873628cfb4 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -77,7 +77,6 @@ case class MessageBuilder(peersKeeper: ActorRef, context.parent ! MsgSent(PeersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } } - } } diff --git a/src/main/scala/encry/network/Messages.scala b/src/main/scala/encry/network/Messages.scala index a9303937eb..2e47bb1646 100644 --- a/src/main/scala/encry/network/Messages.scala +++ b/src/main/scala/encry/network/Messages.scala @@ -19,5 +19,8 @@ object Messages { modifierId: ModifierId) extends MessageToNetwork final case class SendPeers(peers: List[InetSocketAddress], to: InetSocketAddress) extends MessageToNetwork final case class BroadcastManifestRequest(manifestId: Array[Byte]) extends MessageToNetwork + final case class NotifyNodeAboutModifier(source: InetSocketAddress, + modifierTypeId: ModifierTypeId, + modifierId: ModifierId) extends MessageToNetwork } } diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index 3f40526557..a76153a211 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -10,7 +10,7 @@ import encry.network.BlackList.BanReason._ import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming, Outgoing} -import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} +import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection, OutgoingConnectionFailed} import encry.network.PeersKeeper._ import encry.settings.TestNetSettings import org.encryfoundation.common.network.BasicMessagesRepo.{GetPeersNetworkMessage, Handshake, PeersNetworkMessage} @@ -43,7 +43,6 @@ class ConnectWithNewPeerTests extends WordSpecLike "maintain outgoing connection process correctly" in { /* Request first peer while current number of connections is 0 */ val networkController: TestProbe = TestProbe() - val nodeViewSync = TestProbe() val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( PK.props(settings.network.copy(maxConnections = 2), settings.blackList), networkController.ref @@ -124,10 +123,6 @@ class ConnectWithNewPeerTests extends WordSpecLike /* Try to request new connection */ - val updatedAvailablePeers: Set[InetSocketAddress] = peersKeeper.underlyingActor.knownPeers.filter(peer => - peer != msg.peer && peer != newConnectedPeer.socketAddress - ) - networkController.send(peersKeeper, RequestPeerForConnection) val oneMorePeer = networkController.expectMsgType[PeerForConnection] peersKeeper.underlyingActor.outgoingConnections.contains(oneMorePeer.peer) shouldBe true @@ -148,30 +143,31 @@ class ConnectWithNewPeerTests extends WordSpecLike peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(oneMorePeer.peer) shouldBe false peersKeeper.underlyingActor.knownPeers.contains(oneMorePeer.peer) shouldBe true } -// "remove peer from available we can't connect to" in { -// val networkController = TestProbe() -// val nodeViewSync = TestProbe() -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettingsWithAllPeers, nodeViewSync.ref, TestProbe().ref)) -// -// val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.knownPeers -// -// networkController.send(peersKeeper, RequestPeerForConnection) -// networkController.expectMsg(PeerForConnection(availablePeers.head._1)) -// -// networkController.send(peersKeeper, OutgoingConnectionFailed(availablePeers.head._1)) -// peersKeeper.underlyingActor.outgoingConnections.contains(availablePeers.head._1) shouldBe false -// peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe false -// peersKeeper.underlyingActor.knownPeers.get(availablePeers.head._1) shouldBe Some(1) -// -// networkController.send(peersKeeper, RequestPeerForConnection) -// networkController.expectMsg(PeerForConnection(availablePeers.head._1)) -// -// networkController.send(peersKeeper, OutgoingConnectionFailed(availablePeers.head._1)) -// peersKeeper.underlyingActor.outgoingConnections.contains(availablePeers.head._1) shouldBe false -// peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(availablePeers.head._1) shouldBe false -// peersKeeper.underlyingActor.knownPeers.contains(availablePeers.head._1) shouldBe false -// peersKeeper.underlyingActor.blackList.contains(availablePeers.head._1.getAddress) shouldBe true -// } + "remove peer from available we can't connect to" in { + val networkController: TestProbe = TestProbe() + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2), settings.blackList), + networkController.ref + ) + + val availablePeers: Set[InetSocketAddress] = peersKeeper.underlyingActor.knownPeers + + networkController.send(peersKeeper, RequestPeerForConnection) + val msg = networkController.expectMsgType[PeerForConnection] + + networkController.send(peersKeeper, OutgoingConnectionFailed(msg.peer)) + peersKeeper.underlyingActor.outgoingConnections.contains(msg.peer) shouldBe false + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(msg.peer) shouldBe false + + networkController.send(peersKeeper, RequestPeerForConnection) + val nextPeer = networkController.expectMsgType[PeerForConnection] + + networkController.send(peersKeeper, OutgoingConnectionFailed(nextPeer.peer)) + peersKeeper.underlyingActor.outgoingConnections.contains(nextPeer.peer) shouldBe false + peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(nextPeer.peer) shouldBe false + peersKeeper.underlyingActor.knownPeers.contains(nextPeer.peer) shouldBe false + peersKeeper.underlyingActor.blackList.contains(nextPeer.peer) shouldBe true + } // "remove peer from available if it has been banned" in { // val networkController = TestProbe() // val nodeViewSync = TestProbe() From 913230ee5dd177dc618ffe280468f468ffeea7da Mon Sep 17 00:00:00 2001 From: aleksandr Date: Wed, 18 Mar 2020 10:19:16 +0300 Subject: [PATCH 109/177] ec-fix --- src/main/scala/encry/network/Messages.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/encry/network/Messages.scala b/src/main/scala/encry/network/Messages.scala index 2e47bb1646..0f7c0f1641 100644 --- a/src/main/scala/encry/network/Messages.scala +++ b/src/main/scala/encry/network/Messages.scala @@ -21,6 +21,6 @@ object Messages { final case class BroadcastManifestRequest(manifestId: Array[Byte]) extends MessageToNetwork final case class NotifyNodeAboutModifier(source: InetSocketAddress, modifierTypeId: ModifierTypeId, - modifierId: ModifierId) extends MessageToNetwork + modifierId: List[ModifierId]) extends MessageToNetwork } } From 9cea1f81efad826022e29c104c6c1bc896bdcb4b Mon Sep 17 00:00:00 2001 From: aleksandr Date: Wed, 18 Mar 2020 10:21:15 +0300 Subject: [PATCH 110/177] add InvNetworkMessage on message builder --- src/main/scala/encry/network/MessageBuilder.scala | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 873628cfb4..8dc72f1897 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -10,7 +10,7 @@ import encry.consensus.HistoryConsensus.{Equal, Older, Younger} import encry.network.ConnectedPeersCollection.PeerInfo import encry.network.DM.{IsRequested, RequestSent, RequestStatus} import encry.network.MessageBuilder.{GetPeerInfo, GetPeers, MsgSent} -import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendPeers, SendSyncInfo} +import encry.network.Messages.MessageToNetwork.{BroadcastModifier, NotifyNodeAboutModifier, RequestFromLocal, ResponseFromLocal, SendPeers, SendSyncInfo} import encry.network.PeerConnectionHandler.ConnectedPeer import org.encryfoundation.common.network.BasicMessagesRepo.{InvNetworkMessage, ModifiersNetworkMessage, PeersNetworkMessage, RequestModifiersNetworkMessage, SyncInfoNetworkMessage} import org.encryfoundation.common.utils.Algos @@ -63,13 +63,18 @@ case class MessageBuilder(peersKeeper: ActorRef, context.parent ! MsgSent(ModifiersNetworkMessage.NetworkMessageTypeID, peer.socketAddress) } } - context.stop(self) + case NotifyNodeAboutModifier(peer, modTypeId, modsIds) => + Try { + (peersKeeper ? GetPeerInfo(peer)).mapTo[ConnectedPeer].map { peer => + peer.handlerRef ! InvNetworkMessage(modTypeId -> modsIds) + context.parent ! MsgSent(InvNetworkMessage.NetworkMessageTypeID, peer.socketAddress) + } + } case BroadcastModifier(modTypeId, modInfo) => (peersKeeper ? GetPeers).mapTo[List[ConnectedPeer]].map { peers => peers.foreach(_.handlerRef ! InvNetworkMessage(modTypeId -> List(modInfo))) context.parent ! MsgSent(InvNetworkMessage.NetworkMessageTypeID, peers.head.socketAddress) } - context.stop(self) case SendPeers(peers, remote) => Try { (peersKeeper ? GetPeerInfo(remote)).mapTo[ConnectedPeer].map { peer => From d5618cb6d438a94fa0a17bf10f67898d9bd60070 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 18 Mar 2020 10:24:36 +0300 Subject: [PATCH 111/177] added inv sending --- src/main/scala/encry/nvg/IntermediaryNVH.scala | 9 ++++++++- src/main/scala/encry/nvg/NodeViewNMProcessor.scala | 13 +++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index a1585f5e7f..f82d227e89 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -8,7 +8,13 @@ import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.local.miner.Miner.{ DisableMining, EnableMining, StartMining } import encry.mpg.MemoryPool._ import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } +import encry.network.Messages.MessageToNetwork.{ + BroadcastModifier, + NotifyNodeAboutModifier, + RequestFromLocal, + ResponseFromLocal, + SendSyncInfo +} import encry.network.NetworkController.ReceivableMessages.{ DataFromPeer, RegisterMessagesHandler } import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } import encry.network.NodeViewSynchronizer.ReceivableMessages._ @@ -104,6 +110,7 @@ class IntermediaryNVH( case msg @ BroadcastModifier(_, _) => intermediaryNetwork ! msg case msg @ SyntacticallyFailedModification(_, _) => intermediaryNetwork ! msg case msg @ SendSyncInfo(_) => intermediaryNetwork ! msg + case msg @ NotifyNodeAboutModifier(_, _, _) => intermediaryNetwork ! msg case msg @ RequiredManifestHeightAndId(_, _) => //+ to fast sync case msg @ TreeChunks(_, _) => //+ to fast sync case msg @ FastSyncDone => diff --git a/src/main/scala/encry/nvg/NodeViewNMProcessor.scala b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala index 693cab0618..824dee4acc 100644 --- a/src/main/scala/encry/nvg/NodeViewNMProcessor.scala +++ b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala @@ -5,7 +5,13 @@ import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.HistoryComparisonResult import encry.network.DeliveryManager.CheckPayloadsToDownload -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, ResponseFromLocal, SendSyncInfo } +import encry.network.Messages.MessageToNetwork.{ + BroadcastModifier, + NotifyNodeAboutModifier, + RequestFromLocal, + ResponseFromLocal, + SendSyncInfo +} import encry.network.ModifiersToNetworkUtils.toProto import encry.network.NetworkController.ReceivableMessages.DataFromPeer import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingStatus @@ -63,12 +69,15 @@ class NodeViewNMProcessor(settings: EncryAppSettings) extends Actor with StrictL case SemanticallySuccessfulModifier(_) => case DataFromPeer(SyncInfoNetworkMessage(syncInfo: SyncInfo), remote) => + val extension: Seq[ModifierId] = + historyReader.continuationIds(syncInfo, settings.network.syncPacketLength) val comparison: HistoryComparisonResult = historyReader.compare(syncInfo) logger.info( s"\n\nComparison with $remote has starting points ${idsToString(syncInfo.startingPoints)}.\n" + - s"Comparison result is $comparison.\n " + s"Comparison result is $comparison. Extension length is: ${extension.size}.\n " ) context.parent ! OtherNodeSyncingStatus(remote, comparison) + context.parent ! NotifyNodeAboutModifier(remote, Header.modifierTypeId, extension.toList) case DataFromPeer(InvNetworkMessage(invData: InvData), remote) => if (invData._1 == Payload.modifierTypeId && !historyReader.isFullChainSynced) From 477c481f5bb88ef3ee656507e1ba079a71028973 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Wed, 18 Mar 2020 10:54:01 +0300 Subject: [PATCH 112/177] one more test on pk --- src/main/scala/encry/network/PK.scala | 13 ++++++++++++- .../encry/network/ConnectWithNewPeerTests.scala | 8 +++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index 9d84c09143..fd0362a5e1 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -15,7 +15,7 @@ import encry.network.NodeViewSynchronizer.ReceivableMessages.OtherNodeSyncingSta import encry.network.PeerConnectionHandler.ReceivableMessages.CloseConnection import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming, Outgoing} import encry.network.PeersKeeper.{BanPeer, BanPeerFromAPI, PeerForConnection, RequestPeerForConnection} -import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection} +import encry.network.PeersKeeper.ConnectionStatusMessages.{ConnectionStopped, ConnectionVerified, HandshakedDone, NewConnection, OutgoingConnectionFailed} import encry.settings.{BlackListSettings, NetworkSettings} import org.encryfoundation.common.network.BasicMessagesRepo.{GetPeersNetworkMessage, PeersNetworkMessage} @@ -76,6 +76,17 @@ class PK(networkSettings: NetworkSettings, logger.info(s"Adding new peer: $peer to awaitingHandshakeConnections." + s" Current is: ${awaitingHandshakeConnections.mkString(",")}") } + case OutgoingConnectionFailed(peer) => + logger.info(s"Connection failed for: $peer.") + outgoingConnections -= peer + awaitingHandshakeConnections -= peer + val connectionAttempts: Int = peersForConnection.getOrElse(peer, 0) + 1 + if (connectionAttempts >= networkSettings.maxNumberOfReConnections) { + logger.info(s"Removing peer: $peer from available peers for ExpiredNumberOfConnections.") + //todo think about penalty for the less time than general ban + //blackList.banPeer(ExpiredNumberOfConnections, peer.getAddress) + peersForConnection -= peer + } else peersForConnection = peersForConnection.updated(peer, connectionAttempts) case OtherNodeSyncingStatus(remote, comparison) => connectedPeers = connectedPeers.updateHistoryComparisonResult(Map(remote -> comparison)) case NewConnection(remote, remoteConnection) if connectedPeers.size < networkSettings.maxConnections && !isSelf(remote) => diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index a76153a211..13251c3b4e 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -146,9 +146,13 @@ class ConnectWithNewPeerTests extends WordSpecLike "remove peer from available we can't connect to" in { val networkController: TestProbe = TestProbe() val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( - PK.props(settings.network.copy(maxConnections = 2), settings.blackList), + PK.props(settings.network.copy(maxConnections = 2, knownPeers = List(new InetSocketAddress("1.1.1.1", 1234))), settings.blackList), networkController.ref ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) val availablePeers: Set[InetSocketAddress] = peersKeeper.underlyingActor.knownPeers @@ -165,8 +169,6 @@ class ConnectWithNewPeerTests extends WordSpecLike networkController.send(peersKeeper, OutgoingConnectionFailed(nextPeer.peer)) peersKeeper.underlyingActor.outgoingConnections.contains(nextPeer.peer) shouldBe false peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(nextPeer.peer) shouldBe false - peersKeeper.underlyingActor.knownPeers.contains(nextPeer.peer) shouldBe false - peersKeeper.underlyingActor.blackList.contains(nextPeer.peer) shouldBe true } // "remove peer from available if it has been banned" in { // val networkController = TestProbe() From e2b35118849f3af375fa796c48fe75317a998996 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Wed, 18 Mar 2020 11:19:50 +0300 Subject: [PATCH 113/177] more tests --- .../network/ConnectWithNewPeerTests.scala | 208 ++++++++++-------- 1 file changed, 118 insertions(+), 90 deletions(-) diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index 13251c3b4e..da28e377e8 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -170,96 +170,124 @@ class ConnectWithNewPeerTests extends WordSpecLike peersKeeper.underlyingActor.outgoingConnections.contains(nextPeer.peer) shouldBe false peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(nextPeer.peer) shouldBe false } -// "remove peer from available if it has been banned" in { -// val networkController = TestProbe() -// val nodeViewSync = TestProbe() -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, nodeViewSync.ref, TestProbe().ref)) -// -// val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.peersForConnection -// -// val peerHandler = TestProbe() -// val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) -// -// networkController.send(peersKeeper, BanPeer(connectedPeer, ExpiredNumberOfConnections)) -// networkController.send(peersKeeper, ConnectionStopped(availablePeers.head._1)) -// -// peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false -// } -// "filter peers from network message" in { -// val networkController = TestProbe() -// val nodeViewSync = TestProbe() -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, nodeViewSync.ref, TestProbe().ref)) -// -// val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.peersForConnection -// -// val peerHandler = TestProbe() -// val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) -// -// val newPeerHandler = TestProbe() -// val newConnectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.last._1, newPeerHandler.ref, Outgoing, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test-peer", Some(availablePeers.last._1), System.currentTimeMillis())) -// -// networkController.send(peersKeeper, BanPeer(connectedPeer, ExpiredNumberOfConnections)) -// networkController.send(peersKeeper, ConnectionStopped(availablePeers.head._1)) -// networkController.send(peersKeeper, HandshakedDone(newConnectedPeer)) -// peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false -// -// val peer = new InetSocketAddress("172.16.28.98", 9023) -// val peers = Seq(availablePeers.last._1, availablePeers.head._1, peer) -// -// networkController.send(peersKeeper, DataFromPeer(PeersNetworkMessage(peers), newConnectedPeer)) -// peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false -// peersKeeper.underlyingActor.peersForConnection.contains(peer) shouldBe true -// } -// "handle successful connection process" in { -// val networkController = TestProbe() -// val peersSenderProbe = TestProbe() -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) -// val connectedPeer: ConnectedPeer = ConnectedPeer(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test-peer", Some(testNetSettings.network.knownPeers.head), System.currentTimeMillis())) -// -// networkController.send(peersKeeper, RequestPeerForConnection) -// networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) -// peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true -// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true -// -// networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) -// networkController.expectMsg( -// ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) -// -// networkController.send(peersKeeper, HandshakedDone(connectedPeer)) -// peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe true -// } -// "handle stop connection process" in { -// val networkController = TestProbe() -// val peersSenderProbe = TestProbe() -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) -// val connectedPeer: ConnectedPeer = ConnectedPeer(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test-peer", Some(testNetSettings.network.knownPeers.head), System.currentTimeMillis())) -// -// networkController.send(peersKeeper, RequestPeerForConnection) -// networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) -// peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true -// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true -// -// networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) -// networkController.expectMsg( -// ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) -// -// networkController.send(peersKeeper, HandshakedDone(connectedPeer)) -// peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe true -// -// peersKeeper ! ConnectionStopped(testNetSettings.network.knownPeers.head) -// peersKeeper.underlyingActor.connectedPeers.contains(testNetSettings.network.knownPeers.head) shouldBe false -// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true -// } + "remove peer from available if it has been banned" in { + val networkController: TestProbe = TestProbe() + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2, knownPeers = List(new InetSocketAddress("1.1.1.1", 1234))), settings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + + val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.peersForConnection + + val peerHandler = TestProbe() + val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) + + networkController.send(peersKeeper, BanPeer(connectedPeer.socketAddress, ExpiredNumberOfConnections)) + networkController.send(peersKeeper, ConnectionStopped(availablePeers.head._1)) + + peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false + } + "filter peers from network message" in { + val networkController: TestProbe = TestProbe() + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2), settings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + + val availablePeers: Map[InetSocketAddress, Int] = peersKeeper.underlyingActor.peersForConnection + + val peerHandler = TestProbe() + val connectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.head._1, peerHandler.ref, Outgoing, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "test-peer", Some(availablePeers.head._1), System.currentTimeMillis())) + + val newPeerHandler = TestProbe() + val newConnectedPeer: ConnectedPeer = ConnectedPeer(availablePeers.last._1, newPeerHandler.ref, Outgoing, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "test-peer", Some(availablePeers.last._1), System.currentTimeMillis())) + + networkController.send(peersKeeper, BanPeer(connectedPeer.socketAddress, ExpiredNumberOfConnections)) + networkController.send(peersKeeper, ConnectionStopped(availablePeers.head._1)) + networkController.send(peersKeeper, HandshakedDone(newConnectedPeer)) + peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false + + val peer = new InetSocketAddress("172.16.28.98", 9023) + val peers = Seq(availablePeers.last._1, availablePeers.head._1, peer) + + networkController.send(peersKeeper, DataFromPeer(PeersNetworkMessage(peers), newConnectedPeer.socketAddress)) + peersKeeper.underlyingActor.peersForConnection.contains(availablePeers.head._1) shouldBe false + peersKeeper.underlyingActor.peersForConnection.contains(peer) shouldBe true + } + "handle successful connection process" in { + val networkController: TestProbe = TestProbe() + val peerAddr = new InetSocketAddress("1.1.1.1", 1234) + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2, knownPeers = List(peerAddr)), settings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + + val peerHandler = TestProbe() + val connectedPeer: ConnectedPeer = ConnectedPeer(peerAddr, peerHandler.ref, Outgoing, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "test-peer", Some(peerAddr), System.currentTimeMillis())) + + networkController.send(peersKeeper, RequestPeerForConnection) + networkController.expectMsg(PeerForConnection(peerAddr)) + peersKeeper.underlyingActor.outgoingConnections.contains(peerAddr) shouldBe true + peersKeeper.underlyingActor.peersForConnection.contains(peerAddr) shouldBe true + + networkController.send(peersKeeper, NewConnection(peerAddr, peerHandler.ref)) + networkController.expectMsg( + ConnectionVerified(peerAddr, peerHandler.ref, Outgoing)) + + networkController.send(peersKeeper, HandshakedDone(connectedPeer)) + peersKeeper.underlyingActor.connectedPeers.contains(peerAddr) shouldBe true + } + "handle stop connection process" in { + val networkController: TestProbe = TestProbe() + val peerAddr = new InetSocketAddress("1.1.1.1", 1234) + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2, knownPeers = List(peerAddr)), settings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + val peersSenderProbe = TestProbe() + + networkController.send(peersKeeper, RequestPeerForConnection) + val msg = networkController.expectMsgType[PeerForConnection] + val connectedPeer: ConnectedPeer = ConnectedPeer(msg.peer, peersSenderProbe.ref, Outgoing, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "test-peer", Some(msg.peer), System.currentTimeMillis())) + peersKeeper.underlyingActor.outgoingConnections.contains(msg.peer) shouldBe true + peersKeeper.underlyingActor.peersForConnection.contains(msg.peer) shouldBe true + + networkController.send(peersKeeper, NewConnection(msg.peer, peersSenderProbe.ref)) + networkController.expectMsg(ConnectionVerified(msg.peer, peersSenderProbe.ref, Outgoing)) + + networkController.send(peersKeeper, HandshakedDone(connectedPeer)) + peersKeeper.underlyingActor.connectedPeers.contains(msg.peer) shouldBe true + + peersKeeper ! ConnectionStopped(msg.peer) + peersKeeper.underlyingActor.connectedPeers.contains(msg.peer) shouldBe false + peersKeeper.underlyingActor.peersForConnection.contains(msg.peer) shouldBe true + } // "handle failed connection process" in { // val networkController = TestProbe() // val peersSenderProbe = TestProbe() From 0e04d0a1599870a063ee18484fcc88e994664e42 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 18 Mar 2020 16:51:13 +0300 Subject: [PATCH 114/177] Applying modifiers pipeline test added --- src/main/resources/application.conf | 2 + .../encry/settings/EncryAppSettings.scala | 3 +- .../encry/settings/NodeSettingsReader.scala | 3 +- .../scala/encry/view/history/History.scala | 11 +- .../history/HistoryModifiersValidator.scala | 2 +- .../network/ConnectWithNewPeerTests.scala | 2 +- src/test/scala/encry/nvg/PipelinesTests.scala | 305 +++++++++++++++++- .../scala/encry/utils/EncryGenerator.scala | 22 +- 8 files changed, 337 insertions(+), 13 deletions(-) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index d89602ab55..26e889813b 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -18,6 +18,8 @@ encry { miningDelay = 5s # Is CLI available useCli = true + #Debug regime + isTestMod = false } mempool { # The time during which the transaction is considered valid diff --git a/src/main/scala/encry/settings/EncryAppSettings.scala b/src/main/scala/encry/settings/EncryAppSettings.scala index b9f052c1c4..5fd4d90442 100644 --- a/src/main/scala/encry/settings/EncryAppSettings.scala +++ b/src/main/scala/encry/settings/EncryAppSettings.scala @@ -164,4 +164,5 @@ final case class NodeSettings(blocksToKeep: Int, numberOfMiningWorkers: Int, miningDelay: FiniteDuration, offlineGeneration: Boolean, - useCli: Boolean) + useCli: Boolean, + isTestMod: Boolean) diff --git a/src/main/scala/encry/settings/NodeSettingsReader.scala b/src/main/scala/encry/settings/NodeSettingsReader.scala index 208c7ded16..9cad2bb564 100644 --- a/src/main/scala/encry/settings/NodeSettingsReader.scala +++ b/src/main/scala/encry/settings/NodeSettingsReader.scala @@ -14,6 +14,7 @@ trait NodeSettingsReader { cfg.as[Int](s"$path.numberOfMiningWorkers"), cfg.as[FiniteDuration](s"$path.miningDelay"), cfg.as[Boolean](s"$path.offlineGeneration"), - cfg.as[Boolean](s"$path.useCli") + cfg.as[Boolean](s"$path.useCli"), + cfg.as[Boolean](s"$path.isTestMod") ) } \ No newline at end of file diff --git a/src/main/scala/encry/view/history/History.scala b/src/main/scala/encry/view/history/History.scala index d2a6154eb3..6ae85f3887 100644 --- a/src/main/scala/encry/view/history/History.scala +++ b/src/main/scala/encry/view/history/History.scala @@ -1,6 +1,8 @@ package encry.view.history import java.io.File + +import cats.syntax.either._ import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.ProgressInfo import encry.settings._ @@ -10,14 +12,12 @@ import encry.storage.iodb.versionalIODB.IODBHistoryWrapper import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} import encry.utils.NetworkTimeProvider import encry.view.history.storage.HistoryStorage -import io.iohk.iodb.{ByteArrayWrapper, LSMStore} +import io.iohk.iodb.LSMStore import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.iq80.leveldb.Options -import cats.syntax.either._ -import supertagged.@@ /** * History implementation. It is processing persistent modifiers generated locally or received from the network. @@ -177,17 +177,16 @@ object History extends StrictLogging { if (settingsEncry.snapshotSettings.enableFastSynchronization && !settingsEncry.node.offlineGeneration) new History with HistoryHeadersProcessor with FastSyncProcessor { override val settings: EncryAppSettings = settingsEncry - override var isFullChainSynced: Boolean = settings.node.offlineGeneration + override var isFullChainSynced: Boolean = settingsEncry.node.offlineGeneration override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) override val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsEncry.ntp) } else new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { override val settings: EncryAppSettings = settingsEncry - override var isFullChainSynced: Boolean = settings.node.offlineGeneration + override var isFullChainSynced: Boolean = settingsEncry.node.offlineGeneration override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) override val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsEncry.ntp) } - } } \ No newline at end of file diff --git a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala index 0741251609..d777887796 100644 --- a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala +++ b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala @@ -71,7 +71,7 @@ trait HistoryModifiersValidator extends HistoryApi { _ <- Either.cond(heightOf(h.parentId).exists(h => getBestHeaderHeight - h < settings.constants.MaxRollbackDepth), (), HeaderFatalValidationError(s"Header ${h.encodedId} has height greater than max roll back depth")) powSchemeValidationResult = powScheme.verify(h) - _ <- Either.cond(powSchemeValidationResult.isRight, (), + _ <- Either.cond(if (settings.node.isTestMod) true else powSchemeValidationResult.isRight, (), HeaderFatalValidationError(s"Wrong proof-of-work solution in header ${h.encodedId}" + s" caused: $powSchemeValidationResult")) _ <- Either.cond(isSemanticallyValid(h.parentId) != ModifierSemanticValidity.Invalid, (), diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index a76153a211..3d56eb7d7d 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -166,7 +166,7 @@ class ConnectWithNewPeerTests extends WordSpecLike peersKeeper.underlyingActor.outgoingConnections.contains(nextPeer.peer) shouldBe false peersKeeper.underlyingActor.awaitingHandshakeConnections.contains(nextPeer.peer) shouldBe false peersKeeper.underlyingActor.knownPeers.contains(nextPeer.peer) shouldBe false - peersKeeper.underlyingActor.blackList.contains(nextPeer.peer) shouldBe true + //peersKeeper.underlyingActor.blackList.contains(nextPeer.peer) shouldBe true } // "remove peer from available if it has been banned" in { // val networkController = TestProbe() diff --git a/src/test/scala/encry/nvg/PipelinesTests.scala b/src/test/scala/encry/nvg/PipelinesTests.scala index 05d227e223..2feb7a6f77 100644 --- a/src/test/scala/encry/nvg/PipelinesTests.scala +++ b/src/test/scala/encry/nvg/PipelinesTests.scala @@ -1,5 +1,308 @@ package encry.nvg -class PipelinesTests { +import java.net.InetSocketAddress +import akka.actor.{ ActorRef, ActorSystem } +import akka.testkit.{ TestActorRef, TestKit, TestProbe } +import com.typesafe.scalalogging.StrictLogging +import encry.EncryApp +import encry.consensus.EncrySupplyController +import encry.modifiers.InstanceFactory +import encry.modifiers.mempool.TransactionFactory +import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, SendSyncInfo } +import encry.network.NetworkController.ReceivableMessages.RegisterMessagesHandler +import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } +import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier +import encry.utils.implicits.UTXO.{ combineAll, _ } +import encry.utils.{ FileHelper, Mnemonic, NetworkTimeProvider, TestHelper } +import encry.view.history.History +import encry.view.state.UtxoState +import encry.view.state.avlTree.utils.implicits.Instances._ +import encry.view.wallet.AccountManager +import org.encryfoundation.common.crypto.PrivateKey25519 +import org.encryfoundation.common.crypto.equihash.EquihashSolution +import org.encryfoundation.common.modifiers.history._ +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +import org.encryfoundation.common.modifiers.state.box.AssetBox +import org.encryfoundation.common.utils.TaggedTypes.{ Difficulty, Height } +import org.encryfoundation.common.utils.constants.TestNetConstants +import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } + +import scala.concurrent.duration._ + +class PipelinesTests + extends TestKit(ActorSystem("Tested-Akka-System")) + with WordSpecLike + with Matchers + with BeforeAndAfterAll + with InstanceFactory + with OneInstancePerTest + with StrictLogging { + + override def afterAll(): Unit = system.terminate() + + "Node view pipelines" should { + "correct process modifier from the network async" in { + val tmpFile = FileHelper.getRandomTempDir + val path = tmpFile.getAbsolutePath + val settingsWithNewPath = + settings + .copy(directory = path) + .copy(wallet = settings.wallet.map(_.copy(password = "123"))) + .copy(node = settings.node.copy(isTestMod = true)) + AccountManager.init(Mnemonic.entropyToMnemonicCode(scorex.utils.Random.randomBytes(16)), + "123", + settingsWithNewPath) + + val intermediaryParent = TestProbe() + val networkIntermediary = TestProbe() + val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsWithNewPath.ntp) + + val intermediary: ActorRef = intermediaryParent.childActorOf( + IntermediaryNVH.props( + settingsWithNewPath, + networkIntermediary.ref, + timeProvider, + None, + TestProbe().ref, + TestProbe().ref + ) + ) + + val remote = new InetSocketAddress("0.0.0.0", 9001) + + val (_, _, blocks) = PipelinesTests.generateValidForHistoryAndStateBlocks(300) + + blocks.reverse.foreach { block => + intermediary ! ModifierFromNetwork( + remote, + Header.modifierTypeId, + block.id, + HeaderProtoSerializer.toProto(block.header).toByteArray + ) + logger.info(s"Sent to nvh actor header ${block.encodedId}") + } + + Thread.sleep(8000) + + networkIntermediary.expectMsgPF(15.seconds) { + case SemanticallySuccessfulModifier(mod) => + blocks.exists(_.id.sameElements(mod.id)) shouldBe true + case msg @ SendSyncInfo(_) => + case msg @ BroadcastModifier(modifierTypeId, modifierId) => + blocks.exists(_.id.sameElements(modifierId)) shouldBe true + case RegisterMessagesHandler(_, _) => + case RegisterForModsHandling => + case RequestFromLocal(s, m, mods) => + mods.size shouldBe blocks.size + mods.zip(blocks).forall { + case (id, block) => id.sameElements(block.id) + } shouldBe true + } + + blocks.reverse.foreach { block => + intermediary ! ModifierFromNetwork( + remote, + Payload.modifierTypeId, + block.payload.id, + PayloadProtoSerializer.toProto(block.payload).toByteArray + ) + } + + Thread.sleep(10000) + + networkIntermediary.expectMsgPF(15.seconds) { + case SemanticallySuccessfulModifier(mod) => + blocks.exists(_.id.sameElements(mod.id)) shouldBe true + case msg @ SendSyncInfo(_) => + case RegisterForModsHandling => + case msg @ BroadcastModifier(modifierTypeId, modifierId) => + blocks.exists(_.id.sameElements(modifierId)) shouldBe true + case RegisterMessagesHandler(_, _) => + case RequestFromLocal(s, m, mods) => + } + + } + "correct process modifier from the network sync" in { + val tmpFile = FileHelper.getRandomTempDir + val path = tmpFile.getAbsolutePath + val settingsWithNewPath = + settings + .copy(directory = path) + .copy(wallet = settings.wallet.map(_.copy(password = "123"))) + .copy(node = settings.node.copy(isTestMod = true)) + AccountManager.init(Mnemonic.entropyToMnemonicCode(scorex.utils.Random.randomBytes(16)), + "123", + settingsWithNewPath) + + val networkIntermediary = TestProbe() + val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsWithNewPath.ntp) + + val intermediary = TestActorRef[IntermediaryNVH]( + IntermediaryNVH.props( + settingsWithNewPath, + networkIntermediary.ref, + timeProvider, + None, + TestProbe().ref, + TestProbe().ref + ) + ) + + val remote = new InetSocketAddress("0.0.0.0", 9001) + + val (_, _, blocks) = PipelinesTests.generateValidForHistoryAndStateBlocks(300) + + blocks.reverse.foreach { block => + intermediary ! ModifierFromNetwork( + remote, + Header.modifierTypeId, + block.id, + HeaderProtoSerializer.toProto(block.header).toByteArray + ) + logger.info(s"Sent to nvh actor header ${block.encodedId}") + } + + Thread.sleep(8000) + + intermediary.underlyingActor.historyReader.getBestHeaderHeight shouldBe 300 + intermediary.underlyingActor.historyReader.getBestBlockHeight shouldBe -1 + + blocks.reverse.foreach { block => + intermediary ! ModifierFromNetwork( + remote, + Payload.modifierTypeId, + block.payload.id, + PayloadProtoSerializer.toProto(block.payload).toByteArray + ) + } + + Thread.sleep(10000) + + intermediary.underlyingActor.historyReader.getBestHeaderHeight shouldBe 300 + intermediary.underlyingActor.historyReader.getBestBlockHeight shouldBe 300 + + } + } + +} + +object PipelinesTests extends InstanceFactory { + + val key: PrivateKey25519 = TestHelper.genKeys(1).head + + def generateValidForHistoryAndStateBlocks(blocksQty: Int): (History, UtxoState, List[Block]) = { + (0 to blocksQty).foldLeft(generateDummyHistory(settings), + UtxoState.genesis( + FileHelper.getRandomTempDir, + FileHelper.getRandomTempDir, + settings, + None + ), + List.empty[Block]) { + case ((history, state, blocks), i) => + val blockNext: Block = + if (i > 0) { + val boxes: Seq[AssetBox] = + history.getBestBlock.get.payload.txs.flatMap(_.newBoxes.toList).take(30).collect { + case a: AssetBox if a.amount > 13 => a + } + val txs: Vector[Transaction] = + generatePaymentTransactions(key, boxes.toIndexedSeq, 1, 2) + val feesTotal = txs.map(_.fee).sum + val supplyTotal = EncrySupplyController.supplyAt(Height @@ i, settings.constants) + val coinbase: Transaction = TransactionFactory + .coinbaseTransactionScratch(key.publicImage, timestamp, supplyTotal, feesTotal, Height @@ i) + val resTxs = txs :+ coinbase + val difficulty: Difficulty = history.getBestHeader + .map( + parent => + history.requiredDifficultyAfter(parent) match { + case Right(value) => value + case Left(value) => EncryApp.forceStopApplication(999, value.toString) + } + ) + .getOrElse(TestNetConstants.InitialDifficulty) + val combinedStateChange: UtxoState.StateChange = combineAll(resTxs.map(UtxoState.tx2StateChange).toList) + val newStateRoot = state.tree + .getOperationsRootHash( + combinedStateChange.outputsToDb.toList, + combinedStateChange.inputsToDb.toList + ) + .get + + val header = + Header( + TestNetConstants.Version, + history.getBestHeaderId.get, + Payload.rootHash(resTxs.map(_.id)), + System.currentTimeMillis(), + i, + 1, + difficulty, + EquihashSolution(Seq(1, 3)), + newStateRoot + ) + val payload = Payload(header.id, resTxs) + val block = Block(header, payload) + block + } else { + val supplyTotal = EncrySupplyController.supplyAt(Height @@ i, settings.constants) + val coinbase: Transaction = TransactionFactory + .coinbaseTransactionScratch(key.publicImage, timestamp, supplyTotal, 0, Height @@ i) + val resTxs = List(coinbase) + val difficulty: Difficulty = history.getBestHeader + .map( + parent => + history.requiredDifficultyAfter(parent) match { + case Right(value) => value + case Left(value) => EncryApp.forceStopApplication(999, value.toString) + } + ) + .getOrElse(TestNetConstants.InitialDifficulty) + val combinedStateChange: UtxoState.StateChange = combineAll(resTxs.map(UtxoState.tx2StateChange).toList) + val newStateRoot = state.tree + .getOperationsRootHash( + combinedStateChange.outputsToDb.toList, + combinedStateChange.inputsToDb.toList + ) + .get + + val header = + Header( + 0: Byte, + Header.GenesisParentId, + Payload.rootHash(resTxs.map(_.id)), + System.currentTimeMillis(), + i, + 1, + difficulty, + EquihashSolution(Seq(1, 3)), + newStateRoot + ) + val payload = Payload(header.id, resTxs) + val block = Block(header, payload) + block + } + val h = history + .append(blockNext.header) + .right + .get + ._1 + .append(blockNext.payload) + .right + .get + ._1 + .reportModifierIsValid(blockNext) + + val s = state + .applyModifier(blockNext.header) + .right + .get + .applyModifier(blockNext) + .right + .get + (h, s, blocks :+ blockNext) + } + } } diff --git a/src/test/scala/encry/utils/EncryGenerator.scala b/src/test/scala/encry/utils/EncryGenerator.scala index 05879b45cb..87e1bf442a 100644 --- a/src/test/scala/encry/utils/EncryGenerator.scala +++ b/src/test/scala/encry/utils/EncryGenerator.scala @@ -159,6 +159,24 @@ trait EncryGenerator extends Settings { ) } + def generatePaymentTransactions(privKey: PrivateKey25519, + boxes: IndexedSeq[AssetBox], + numberOfInputs: Int, + numberOfOutputs: Int): Vector[Transaction] = + (0 until boxes.size / numberOfInputs).foldLeft(boxes, Vector.empty[Transaction]) { + case ((boxesLocal, transactions), _) => + val tx: Transaction = defaultPaymentTransactionScratch( + privKey, + fee = 11, + timestamp = 11L, + useBoxes = boxesLocal.take(numberOfInputs), + recipient = randomAddress, + amount = 1, + numOfOutputs = numberOfOutputs + ) + (boxesLocal.drop(numberOfInputs), transactions :+ tx) + }._2 + def generatePaymentTransactions(boxes: IndexedSeq[AssetBox], numberOfInputs: Int, numberOfOutputs: Int): Vector[Transaction] = @@ -166,11 +184,11 @@ trait EncryGenerator extends Settings { case ((boxesLocal, transactions), _) => val tx: Transaction = defaultPaymentTransactionScratch( privKey, - fee = 111, + fee = 0, timestamp = 11L, useBoxes = boxesLocal.take(numberOfInputs), recipient = randomAddress, - amount = 10000, + amount = 1, numOfOutputs = numberOfOutputs ) (boxesLocal.drop(numberOfInputs), transactions :+ tx) From 93303834bc1fef7f426ba72146c02975459c03bb Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 18 Mar 2020 16:51:51 +0300 Subject: [PATCH 115/177] fixed modifiers propagation --- src/main/scala/encry/EncryApp.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/EncryApp.scala b/src/main/scala/encry/EncryApp.scala index db1f23ead8..a1d66eed3e 100644 --- a/src/main/scala/encry/EncryApp.scala +++ b/src/main/scala/encry/EncryApp.scala @@ -4,14 +4,14 @@ import java.io.File import java.net.InetAddress import java.nio.file.Files import akka.actor.SupervisorStrategy.Restart -import akka.actor.{ActorRef, ActorSystem, OneForOneStrategy, Props} +import akka.actor.{ ActorRef, ActorSystem, OneForOneStrategy, Props } import akka.http.scaladsl.Http import akka.http.scaladsl.model.HttpResponse import akka.http.scaladsl.server.ExceptionHandler import akka.stream.ActorMaterializer import com.typesafe.scalalogging.StrictLogging import encry.api.http.routes._ -import encry.api.http.{ApiRoute, CompositeHttpService} +import encry.api.http.{ ApiRoute, CompositeHttpService } import encry.settings.EncryAppSettings import encry.stats.Zombie import encry.utils.NetworkTimeProvider @@ -20,7 +20,7 @@ import kamon.influxdb.InfluxDBReporter import kamon.system.SystemMetrics import org.encryfoundation.common.utils.Algos import scala.concurrent.duration._ -import scala.concurrent.{Await, ExecutionContextExecutor, Future} +import scala.concurrent.{ Await, ExecutionContextExecutor, Future } import scala.io.Source import scala.language.postfixOps From 5adef1944c900c92350317e42f67ab3aafbd8acd Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 18 Mar 2020 20:09:00 +0300 Subject: [PATCH 116/177] added test for nvg forks --- .../encry/view/history/HistoryReader.scala | 9 +- src/test/scala/encry/nvg/PipelinesTests.scala | 266 +++++++++++++++++- 2 files changed, 262 insertions(+), 13 deletions(-) diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 7d781b3f00..233ee8f491 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -2,7 +2,6 @@ package encry.view.history import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Older } import encry.modifiers.history.HeaderChain -import io.iohk.iodb.ByteArrayWrapper import org.encryfoundation.common.modifiers.history.{ Block, Header } import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -44,6 +43,10 @@ trait HistoryReader { def syncInfo: SyncInfo def isFastSyncInProcess: Boolean + + def getBestHeader: Option[Header] + + def getBestBlock: Option[Block] } object HistoryReader { @@ -64,6 +67,8 @@ object HistoryReader { def getBlockByHeader(header: Header): Option[Block] = None def headerIdsAtHeight(height: Int): List[ModifierId] = List.empty[ModifierId] def lastHeaders(count: Int): HeaderChain = HeaderChain.empty + def getBestHeader: Option[Header] = None + def getBestBlock: Option[Block] = None } def apply(history: History): HistoryReader = new HistoryReader { @@ -83,5 +88,7 @@ object HistoryReader { def getBlockByHeaderId(id: ModifierId): Option[Block] = history.getBlockByHeaderId(id) def getBlockByHeader(header: Header): Option[Block] = history.getBlockByHeader(header) def lastHeaders(count: Int): HeaderChain = history.lastHeaders(count) + def getBestHeader: Option[Header] = history.getBestHeader + def getBestBlock: Option[Block] = history.getBestBlock } } diff --git a/src/test/scala/encry/nvg/PipelinesTests.scala b/src/test/scala/encry/nvg/PipelinesTests.scala index 2feb7a6f77..11691b812a 100644 --- a/src/test/scala/encry/nvg/PipelinesTests.scala +++ b/src/test/scala/encry/nvg/PipelinesTests.scala @@ -71,7 +71,14 @@ class PipelinesTests val remote = new InetSocketAddress("0.0.0.0", 9001) - val (_, _, blocks) = PipelinesTests.generateValidForHistoryAndStateBlocks(300) + val (_, _, blocks) = PipelinesTests.generateValidForHistoryAndStateBlocks(300, + generateDummyHistory(settings), + UtxoState.genesis( + FileHelper.getRandomTempDir, + FileHelper.getRandomTempDir, + settings, + None + )) blocks.reverse.foreach { block => intermediary ! ModifierFromNetwork( @@ -151,7 +158,14 @@ class PipelinesTests val remote = new InetSocketAddress("0.0.0.0", 9001) - val (_, _, blocks) = PipelinesTests.generateValidForHistoryAndStateBlocks(300) + val (_, _, blocks) = PipelinesTests.generateValidForHistoryAndStateBlocks(300, + generateDummyHistory(settings), + UtxoState.genesis( + FileHelper.getRandomTempDir, + FileHelper.getRandomTempDir, + settings, + None + )) blocks.reverse.foreach { block => intermediary ! ModifierFromNetwork( @@ -183,23 +197,103 @@ class PipelinesTests intermediary.underlyingActor.historyReader.getBestBlockHeight shouldBe 300 } + "work with forks correctly" in { + val tmpFile = FileHelper.getRandomTempDir + val path = tmpFile.getAbsolutePath + val settingsWithNewPath = + settings + .copy(directory = path) + .copy(wallet = settings.wallet.map(_.copy(password = "123"))) + .copy(node = settings.node.copy(isTestMod = true)) + AccountManager.init(Mnemonic.entropyToMnemonicCode(scorex.utils.Random.randomBytes(16)), + "123", + settingsWithNewPath) + + val networkIntermediary = TestProbe() + val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsWithNewPath.ntp) + + val intermediary = TestActorRef[IntermediaryNVH]( + IntermediaryNVH.props( + settingsWithNewPath, + networkIntermediary.ref, + timeProvider, + None, + TestProbe().ref, + TestProbe().ref + ) + ) + + val remote = new InetSocketAddress("0.0.0.0", 9001) + + val ((h1, s1, b1), (h2, s2, b2)) = PipelinesTests.genForOn(5) + + b1.foreach { block => + intermediary ! ModifierFromNetwork( + remote, + Header.modifierTypeId, + block.header.id, + HeaderProtoSerializer.toProto(block.header).toByteArray + ) + } + + Thread.sleep(3000) + + b1.foreach { block => + intermediary ! ModifierFromNetwork( + remote, + Payload.modifierTypeId, + block.payload.id, + PayloadProtoSerializer.toProto(block.payload).toByteArray + ) + } + + Thread.sleep(5000) + + b2.foreach { block => + intermediary ! ModifierFromNetwork( + remote, + Header.modifierTypeId, + block.header.id, + HeaderProtoSerializer.toProto(block.header).toByteArray + ) + } + + Thread.sleep(3000) + + b2.foreach { block => + intermediary ! ModifierFromNetwork( + remote, + Payload.modifierTypeId, + block.payload.id, + PayloadProtoSerializer.toProto(block.payload).toByteArray + ) + } + + Thread.sleep(5000) + + intermediary.underlyingActor.historyReader.getBestBlock.get.id.sameElements(b2.last.id) shouldBe true + intermediary.underlyingActor.historyReader.getBestHeader.get.id.sameElements(b2.last.id) shouldBe true + + } } } -object PipelinesTests extends InstanceFactory { +object PipelinesTests extends InstanceFactory with StrictLogging { val key: PrivateKey25519 = TestHelper.genKeys(1).head - def generateValidForHistoryAndStateBlocks(blocksQty: Int): (History, UtxoState, List[Block]) = { - (0 to blocksQty).foldLeft(generateDummyHistory(settings), - UtxoState.genesis( - FileHelper.getRandomTempDir, - FileHelper.getRandomTempDir, - settings, - None - ), - List.empty[Block]) { + def generateValidForHistoryAndStateBlocks( + blocksQty: Int, + history: History, + utxo: UtxoState, + from: Int = 0 + ): (History, UtxoState, List[Block]) = { + (from to from + blocksQty).foldLeft( + history, + utxo, + List.empty[Block] + ) { case ((history, state, blocks), i) => val blockNext: Block = if (i > 0) { @@ -305,4 +399,152 @@ object PipelinesTests extends InstanceFactory { (h, s, blocks :+ blockNext) } } + + def genForOn( + blocksQty: Int + ): ((History, UtxoState, List[Block]), (History, UtxoState, List[Block])) = { + val (h, s, h1, s1, b) = (0 to blocksQty).foldLeft( + generateDummyHistory(settings), + UtxoState.genesis( + FileHelper.getRandomTempDir, + FileHelper.getRandomTempDir, + settings, + None + ), + generateDummyHistory(settings), + UtxoState.genesis( + FileHelper.getRandomTempDir, + FileHelper.getRandomTempDir, + settings, + None + ), + List.empty[Block] + ) { + case ((history, state, h1, s1, blocks), i) => + val blockNext: Block = + if (i > 0) { + val boxes: Seq[AssetBox] = + history.getBestBlock.get.payload.txs.flatMap(_.newBoxes.toList).take(30).collect { + case a: AssetBox if a.amount > 13 => a + } + val txs: Vector[Transaction] = + generatePaymentTransactions(key, boxes.toIndexedSeq, 1, 2) + val feesTotal = txs.map(_.fee).sum + val supplyTotal = EncrySupplyController.supplyAt(Height @@ i, settings.constants) + val coinbase: Transaction = TransactionFactory + .coinbaseTransactionScratch(key.publicImage, timestamp, supplyTotal, feesTotal, Height @@ i) + val resTxs = txs :+ coinbase + val difficulty: Difficulty = history.getBestHeader + .map( + parent => + history.requiredDifficultyAfter(parent) match { + case Right(value) => value + case Left(value) => EncryApp.forceStopApplication(999, value.toString) + } + ) + .getOrElse(TestNetConstants.InitialDifficulty) + val combinedStateChange: UtxoState.StateChange = combineAll(resTxs.map(UtxoState.tx2StateChange).toList) + val newStateRoot = state.tree + .getOperationsRootHash( + combinedStateChange.outputsToDb.toList, + combinedStateChange.inputsToDb.toList + ) + .get + + val header = + Header( + TestNetConstants.Version, + history.getBestHeaderId.get, + Payload.rootHash(resTxs.map(_.id)), + System.currentTimeMillis(), + i, + 1, + difficulty, + EquihashSolution(Seq(1, 3)), + newStateRoot + ) + val payload = Payload(header.id, resTxs) + val block = Block(header, payload) + block + } else { + val supplyTotal = EncrySupplyController.supplyAt(Height @@ i, settings.constants) + val coinbase: Transaction = TransactionFactory + .coinbaseTransactionScratch(key.publicImage, timestamp, supplyTotal, 0, Height @@ i) + val resTxs = List(coinbase) + val difficulty: Difficulty = history.getBestHeader + .map( + parent => + history.requiredDifficultyAfter(parent) match { + case Right(value) => value + case Left(value) => EncryApp.forceStopApplication(999, value.toString) + } + ) + .getOrElse(TestNetConstants.InitialDifficulty) + val combinedStateChange: UtxoState.StateChange = combineAll(resTxs.map(UtxoState.tx2StateChange).toList) + val newStateRoot = state.tree + .getOperationsRootHash( + combinedStateChange.outputsToDb.toList, + combinedStateChange.inputsToDb.toList + ) + .get + + val header = + Header( + 0: Byte, + Header.GenesisParentId, + Payload.rootHash(resTxs.map(_.id)), + System.currentTimeMillis(), + i, + 1, + difficulty, + EquihashSolution(Seq(1, 3)), + newStateRoot + ) + val payload = Payload(header.id, resTxs) + val block = Block(header, payload) + block + } + val h = history + .append(blockNext.header) + .right + .get + ._1 + .append(blockNext.payload) + .right + .get + ._1 + .reportModifierIsValid(blockNext) + + val s = state + .applyModifier(blockNext.header) + .right + .get + .applyModifier(blockNext) + .right + .get + + val his1 = h1 + .append(blockNext.header) + .right + .get + ._1 + .append(blockNext.payload) + .right + .get + ._1 + .reportModifierIsValid(blockNext) + + val st1 = s1 + .applyModifier(blockNext.header) + .right + .get + .applyModifier(blockNext) + .right + .get + (h, s, his1, st1, blocks :+ blockNext) + } + val (h11, s11, b11) = generateValidForHistoryAndStateBlocks(5, h, s, h.getBestBlockHeight + 1) + val (h22, s22, b22) = generateValidForHistoryAndStateBlocks(10, h1, s1, h1.getBestBlockHeight + 1) + ((h11, s11, b11 ++ b), (h22, s22, b22)) + } } From 64c9e1b415b8577514539e05f67c763620f322b3 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 19 Mar 2020 10:24:40 +0300 Subject: [PATCH 117/177] miner pipeline --- .../encry/view/history/HistoryModifiersValidator.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala index d777887796..3a80a90dde 100644 --- a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala +++ b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala @@ -64,14 +64,14 @@ trait HistoryModifiersValidator extends HistoryApi { s" not greater by 1 than parent's ${parent.height}")) _ <- Either.cond(!historyStorage.containsMod(h.id), (), HeaderFatalValidationError(s"Header ${h.encodedId} is already in history")) - _ <- Either.cond(realDifficulty(h) >= h.requiredDifficulty, (), + _ <- Either.cond( if (settings.node.isTestMod) true else realDifficulty(h) >= h.requiredDifficulty, (), HeaderFatalValidationError(s"Incorrect real difficulty in header ${h.encodedId}")) - _ <- Either.cond(requiredDifficultyAfter(parent).exists(_ <= h.difficulty), (), + _ <- Either.cond( if (settings.node.isTestMod) true else requiredDifficultyAfter(parent).exists(_ <= h.difficulty), (), HeaderFatalValidationError(s"Incorrect required difficulty in header ${h.encodedId}")) _ <- Either.cond(heightOf(h.parentId).exists(h => getBestHeaderHeight - h < settings.constants.MaxRollbackDepth), (), HeaderFatalValidationError(s"Header ${h.encodedId} has height greater than max roll back depth")) powSchemeValidationResult = powScheme.verify(h) - _ <- Either.cond(if (settings.node.isTestMod) true else powSchemeValidationResult.isRight, (), + _ <- Either.cond( if (settings.node.isTestMod) true else powSchemeValidationResult.isRight, (), HeaderFatalValidationError(s"Wrong proof-of-work solution in header ${h.encodedId}" + s" caused: $powSchemeValidationResult")) _ <- Either.cond(isSemanticallyValid(h.parentId) != ModifierSemanticValidity.Invalid, (), From 155d83bc73692ddaef4e6a37d789e169666d11fe Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 19 Mar 2020 11:01:06 +0300 Subject: [PATCH 118/177] recover some tests, fix dm logic with txs --- src/main/scala/encry/network/DM.scala | 16 +- .../network/ConnectWithNewPeerTests.scala | 203 ++++++++++-------- .../DeliveryManagerTests/DMUtils.scala | 7 +- ...DeliveryManagerReRequestModifiesSpec.scala | 52 ++--- .../DeliveryManagerRequestModifiesSpec.scala | 184 ++++++++-------- 5 files changed, 237 insertions(+), 225 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index dbd2a2fc1f..2bd849cdae 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -15,6 +15,7 @@ import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import cats.syntax.option._ import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier} +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import scala.collection.mutable import scala.concurrent.Future @@ -40,18 +41,19 @@ case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging receivedModifiers += toKey(id) } else logger.info(s"Receive spam. ModId: ${Algos.encode(id)}!") } - case RequestSent(peer, modTypeId, modId) if !(expectedModifiers.contains(toKey(modId)) || receivedModifiers.contains(toKey(modId))) => + case RequestSent(peer, modTypeId, modId) if !(expectedModifiers.contains(toKey(modId)) || receivedModifiers.contains(toKey(modId)))=> expectedModifiers += toKey(modId) context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)( self ! AwaitingRequest(peer, modTypeId, modId, 1) ) case RequestSent(_, _, _) => //do nothing - case AwaitingRequest(peer, modTypeId, modId, attempts) if attempts < networkSettings.maxDeliveryChecks && expectedModifiers.contains(toKey(modId))=> - context.parent ! RequestFromLocal(peer.some, modTypeId, List(modId)) - logger.info(s"Re-request modifier ${Algos.encode(modId)}") - context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! - AwaitingRequest(peer, modTypeId, modId, attempts + 1) - ) + case AwaitingRequest(peer, modTypeId, modId, attempts) if + attempts <= networkSettings.maxDeliveryChecks && expectedModifiers.contains(toKey(modId)) && modTypeId != Transaction.modifierTypeId => + context.parent ! RequestFromLocal(peer.some, modTypeId, List(modId)) + logger.info(s"Re-request modifier ${Algos.encode(modId)}") + context.system.scheduler.scheduleOnce(networkSettings.deliveryTimeout)(self ! + AwaitingRequest(peer, modTypeId, modId, attempts + 1) + ) case AwaitingRequest(peer, _, modId, attempts) => logger.info(s"Stop requesting modifier ${Algos.encode(modId)} from peer $peer, qty of attempts $attempts." + s" Expected modifier contains: ${expectedModifiers.contains(toKey(modId))}") diff --git a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala index da28e377e8..a916791861 100644 --- a/src/test/scala/encry/network/ConnectWithNewPeerTests.scala +++ b/src/test/scala/encry/network/ConnectWithNewPeerTests.scala @@ -51,7 +51,6 @@ class ConnectWithNewPeerTests extends WordSpecLike val availablePeers: Set[InetSocketAddress] = peersKeeper.underlyingActor.knownPeers networkController.send(peersKeeper, RequestPeerForConnection) - println(peersKeeper.underlyingActor.knownPeers.map(PeerForConnection)) networkController.expectMsg(RegisterMessagesHandler(Seq( PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" @@ -288,95 +287,117 @@ class ConnectWithNewPeerTests extends WordSpecLike peersKeeper.underlyingActor.connectedPeers.contains(msg.peer) shouldBe false peersKeeper.underlyingActor.peersForConnection.contains(msg.peer) shouldBe true } -// "handle failed connection process" in { -// val networkController = TestProbe() -// val peersSenderProbe = TestProbe() -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) -// -// networkController.send(peersKeeper, RequestPeerForConnection) -// networkController.expectMsg(PeerForConnection(testNetSettings.network.knownPeers.head)) -// peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe true -// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true -// -// networkController.send(peersKeeper, NewConnection(testNetSettings.network.knownPeers.head, peersSenderProbe.ref)) -// networkController.expectMsg( -// ConnectionVerified(testNetSettings.network.knownPeers.head, peersSenderProbe.ref, Outgoing)) -// -// peersKeeper ! OutgoingConnectionFailed(testNetSettings.network.knownPeers.head) -// peersKeeper.underlyingActor.outgoingConnections.contains(testNetSettings.network.knownPeers.head) shouldBe false -// peersKeeper.underlyingActor.peersForConnection.contains(testNetSettings.network.knownPeers.head) shouldBe true -// } -// "handle incoming connections correctly while connection with only known peers false " + -// "and incoming peer doesn't contains in black list and connected peers collection" in { -// val networkController = TestProbe() -// val remoteConnectionTestProbe: TestProbe = TestProbe() -// val remoteAddress: InetSocketAddress = testNetSettings.network.knownPeers.head -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) -// -// networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) -// networkController.expectMsg(ConnectionVerified(remoteAddress, remoteConnectionTestProbe.ref, Incoming)) -// peersKeeper.stop() -// } -// "handle incoming connections correctly while connection with only known peers false " + -// "and incoming peer contain in black list" in { -// val networkController = TestProbe() -// val remoteConnectionTestProbe: TestProbe = TestProbe() -// val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.11", 9001) -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) -// val connectedPeer: ConnectedPeer = ConnectedPeer(remoteAddress, remoteConnectionTestProbe.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test-peer", Some(remoteAddress), System.currentTimeMillis())) -// -// peersKeeper ! BanPeer(connectedPeer, SyntacticallyInvalidPersistentModifier) -// networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) -// networkController.expectNoMsg() -// peersKeeper.stop() -// } -// "handle incoming connections correctly while connection with only known peers false " + -// "and incoming peer contain in connected peers" in { -// val networkController = TestProbe() -// val remoteConnectionTestProbe: TestProbe = TestProbe() -// val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.11", 9001) -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) -// val connectedPeer: ConnectedPeer = ConnectedPeer(remoteAddress, remoteConnectionTestProbe.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "test-peer", Some(remoteAddress), System.currentTimeMillis())) -// -// peersKeeper ! HandshakedDone(connectedPeer) -// networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) -// networkController.expectNoMsg() -// peersKeeper.stop() -// } -// "handle incoming connections correctly while connection with only known peers true" in { -// val networkController = TestProbe() -// val remoteConnectionTestProbe: TestProbe = TestProbe() -// val remoteAddress: InetSocketAddress = new InetSocketAddress("172.16.11.99", 9001) -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) -// -// networkController.send(peersKeeper, NewConnection(remoteAddress, remoteConnectionTestProbe.ref)) -// networkController.expectNoMsg() -// peersKeeper.stop() -// } -// "handle incoming connections correctly while peer is equal to local address" in { -// val networkController = TestProbe() -// val remoteConnectionTestProbe: TestProbe = TestProbe() -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(testNetSettings, TestProbe().ref, TestProbe().ref)) -// -// networkController.send(peersKeeper, NewConnection( -// new InetSocketAddress("0.0.0.0", 9001), remoteConnectionTestProbe.ref)) -// networkController.expectNoMsg() -// peersKeeper.stop() -// } -// "handle outgoing connection" in { -// val networkController = TestProbe() -// val remoteConnectionTestProbe: TestProbe = TestProbe() -// val peersKeeper: TestActorRef[PeersKeeper] = TestActorRef[PeersKeeper](PeersKeeper.props(knowPeersSettings, TestProbe().ref, TestProbe().ref)) -// -// peersKeeper ! RequestPeerForConnection -// peersKeeper.underlyingActor.outgoingConnections.contains(knowPeersSettings.network.knownPeers.head) shouldBe true -// networkController.send(peersKeeper, NewConnection(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref)) -// networkController.expectMsg( -// ConnectionVerified(knowPeersSettings.network.knownPeers.head, remoteConnectionTestProbe.ref, Outgoing)) -// } + "handle failed connection process" in { + val networkController: TestProbe = TestProbe() + val peerAddr = new InetSocketAddress("1.1.1.1", 1234) + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2, knownPeers = List(peerAddr)), settings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + val peersSenderProbe = TestProbe() + networkController.send(peersKeeper, RequestPeerForConnection) + val msg = networkController.expectMsgType[PeerForConnection] + peersKeeper.underlyingActor.outgoingConnections.contains(msg.peer) shouldBe true + peersKeeper.underlyingActor.peersForConnection.contains(msg.peer) shouldBe true + + networkController.send(peersKeeper, NewConnection(msg.peer, peersSenderProbe.ref)) + networkController.expectMsg(ConnectionVerified(msg.peer, peersSenderProbe.ref, Outgoing)) + + peersKeeper ! OutgoingConnectionFailed(msg.peer) + peersKeeper.underlyingActor.outgoingConnections.contains(msg.peer) shouldBe false + peersKeeper.underlyingActor.peersForConnection.contains(msg.peer) shouldBe true + } + "handle incoming connections correctly while connection with only known peers false " + + "and incoming peer doesn't contains in black list and connected peers collection" in { + val networkController: TestProbe = TestProbe() + val peerAddr = new InetSocketAddress("1.1.1.1", 1234) + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2, knownPeers = List(peerAddr)), settings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + val remoteConnectionTestProbe = TestProbe() + networkController.send(peersKeeper, NewConnection(peerAddr, remoteConnectionTestProbe.ref)) + networkController.expectMsg(ConnectionVerified(peerAddr, remoteConnectionTestProbe.ref, Incoming)) + peersKeeper.stop() + } + "handle incoming connections correctly while connection with only known peers false " + + "and incoming peer contain in black list" in { + val networkController: TestProbe = TestProbe() + val peerAddr = new InetSocketAddress("1.1.1.1", 1234) + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(settings.network.copy(maxConnections = 2, knownPeers = List(peerAddr)), settings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + + val remoteConnectionTestProbe = TestProbe() + + peersKeeper ! BanPeer(peerAddr, SyntacticallyInvalidPersistentModifier) + networkController.send(peersKeeper, NewConnection(peerAddr, remoteConnectionTestProbe.ref)) + networkController.expectNoMsg() + peersKeeper.stop() + } + "handle incoming connections correctly while connection with only known peers true" in { + val networkController: TestProbe = TestProbe() + val peerAddr = new InetSocketAddress("1.1.1.1", 1234) + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(knowPeersSettings.network, knowPeersSettings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + val remoteConnectionTestProbe = TestProbe() + networkController.send(peersKeeper, NewConnection(peerAddr, remoteConnectionTestProbe.ref)) + networkController.expectNoMsg() + peersKeeper.stop() + } + "handle incoming connections correctly while peer is equal to local address" in { + val networkController: TestProbe = TestProbe() + val peerAddr = new InetSocketAddress("1.1.1.1", 1234) + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(knowPeersSettings.network, knowPeersSettings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + val remoteConnectionTestProbe = TestProbe() + networkController.send(peersKeeper, NewConnection( + new InetSocketAddress("0.0.0.0", 9001), remoteConnectionTestProbe.ref)) + networkController.expectNoMsg() + peersKeeper.stop() + } + "handle outgoing connection" in { + val networkController: TestProbe = TestProbe() + val peersKeeper: TestActorRef[PK] = TestActorRef[PK]( + PK.props(knowPeersSettings.network, knowPeersSettings.blackList), + networkController.ref + ) + networkController.expectMsg(RegisterMessagesHandler(Seq( + PeersNetworkMessage.NetworkMessageTypeID -> "PeersNetworkMessage", + GetPeersNetworkMessage.NetworkMessageTypeID -> "GetPeersNetworkMessage" + ), peersKeeper.underlying.self)) + networkController.send(peersKeeper, RequestPeerForConnection) + val msg = networkController.expectMsgType[PeerForConnection] + val remoteConnectionTestProbe = TestProbe() + peersKeeper.underlyingActor.outgoingConnections.contains(msg.peer) shouldBe true + networkController.send(peersKeeper, NewConnection(msg.peer, remoteConnectionTestProbe.ref)) + networkController.expectMsg( + ConnectionVerified(msg.peer, remoteConnectionTestProbe.ref, Outgoing)) + } } } \ No newline at end of file diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala b/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala index 0f62964c0d..b0d124080b 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DMUtils.scala @@ -24,14 +24,15 @@ object DMUtils extends InstanceFactory { def initialiseDeliveryManager(isBlockChainSynced: Boolean, isMining: Boolean, settings: EncryAppSettings) - (implicit actorSystem: ActorSystem): (TestActorRef[DM], History) = { + (implicit actorSystem: ActorSystem): (TestProbe, TestActorRef[DM], History) = { val history: History = generateDummyHistory(settings) - val deliveryManager: TestActorRef[DM] = TestActorRef[DM](DM.props(settings.network)) + val networkRouter = TestProbe() + val deliveryManager: TestActorRef[DM] = TestActorRef[DM](DM.props(settings.network), networkRouter.ref) deliveryManager ! UpdatedHistory(history) if (isMining) deliveryManager ! StartMining else deliveryManager ! DisableMining if (isBlockChainSynced) deliveryManager ! FullBlockChainIsSynced - (deliveryManager, history) + (networkRouter, deliveryManager, history) } def generateBlocks(qty: Int, history: History): (History, List[Block]) = diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala index e29d0daafd..d3fe361312 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala @@ -10,7 +10,8 @@ import encry.modifiers.InstanceFactory import encry.network.DM.RequestSent import encry.network.{DM, DeliveryManager} import encry.network.DeliveryManagerTests.DMUtils._ -import encry.network.NetworkController.ReceivableMessages.DataFromPeer +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} import encry.network.PeersKeeper.UpdatedPeersCollection @@ -21,7 +22,7 @@ import encry.settings.TestNetSettings import encry.view.history.History import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{GetPeersNetworkMessage, Handshake, ModifiersNetworkMessage, PeersNetworkMessage, RequestModifiersNetworkMessage} import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} @@ -39,16 +40,16 @@ class DeliveryManagerReRequestModifiesSpec extends WordSpecLike override def afterAll(): Unit = system.terminate() - def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DM], + def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestProbe, TestActorRef[DM], ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte], History) = { - val (deliveryManager, history) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) + val (networkRouter, deliveryManager, history) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 val headersIds: List[ModifierId] = blocks.map(_.header.id) val headersAsKey = headersIds.map(toKey) - (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey, history) + (networkRouter, deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey, history) } "ReRequestModifies" should { @@ -84,28 +85,25 @@ class DeliveryManagerReRequestModifiesSpec extends WordSpecLike // deliveryManager.stop() // } "not re-ask unnecessary modifiers" in { - val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() - + val (networkRouter, deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() + networkRouter.expectMsg(RegisterMessagesHandler(Seq( + ModifiersNetworkMessage.NetworkMessageTypeID -> "ModifiersNetworkMessage", + ), deliveryManager.underlying.self)) val address1 = new InetSocketAddress("123.123.123.123", 9001) val handler1: TestProbe = TestProbe() val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, Handshake(protocolToBytes(testNetSettings.network.appVersion), "123.123.123.123", Some(address1), System.currentTimeMillis())) - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) - val header: ModifierId = headersIds.head deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header) //await one re-ask - handler1.expectMsgAllOf( + networkRouter.expectMsgAllOf( testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)) + RequestFromLocal(Some(address1), Header.modifierTypeId, List(header)), + RequestFromLocal(Some(address1), Header.modifierTypeId, List(header)) ) val headerBytes: Array[Byte] = HeaderProtoSerializer.toProto(genHeader).toByteArray @@ -115,7 +113,7 @@ class DeliveryManagerReRequestModifiesSpec extends WordSpecLike deliveryManager.stop() } "remove peer from expectedModifiers if expected modifiers collection from this peer is empty" in { - val (deliveryManager, cp1, _, _, _, headerIds, _, _) = initialiseState() + val (_, deliveryManager, cp1, _, _, _, headerIds, _, _) = initialiseState() deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, headerIds.head) //this thread sleep is using for expecting modifier removal @@ -124,27 +122,17 @@ class DeliveryManagerReRequestModifiesSpec extends WordSpecLike deliveryManager.stop() } "not re-ask transactions" in { - val (deliveryManager, _, _, _, _, _, _, _) = initialiseState() - + val (networkRouter, deliveryManager, _, _, _, _, _, _, _) = initialiseState() + networkRouter.expectMsg(RegisterMessagesHandler(Seq( + ModifiersNetworkMessage.NetworkMessageTypeID -> "ModifiersNetworkMessage", + ), deliveryManager.underlying.self)) val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address1 -> (cp1, Older, InitialPriority)) - - deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) val transactions: Seq[ModifierId] = genValidPaymentTxs(1).map(_.id) - transactions.foreach(txId => deliveryManager ! RequestSent(cp1.socketAddress, Transaction.modifierTypeId, txId)) + transactions.foreach(txId => deliveryManager ! RequestSent(address1, Transaction.modifierTypeId, txId)) - handler1.expectMsgAllOf( - RequestModifiersNetworkMessage(Transaction.modifierTypeId -> transactions) - ) - handler1.expectNoMsg(10.seconds) + networkRouter.expectNoMsg(testNetSettings.network.deliveryTimeout + 10.seconds) assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) deliveryManager.stop() } diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala index 3e2d241362..4aa685b794 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerRequestModifiesSpec.scala @@ -37,21 +37,21 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte override def afterAll(): Unit = TestKit.shutdownActorSystem(system) - def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestActorRef[DM], + def initialiseState(isChainSynced: Boolean = true, isMining: Boolean = true): (TestProbe, TestActorRef[DM], ConnectedPeer, ConnectedPeer, ConnectedPeer, List[Block], List[ModifierId], List[WrappedArray.ofByte]) = { - val (deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) + val (networkRouter, deliveryManager, _) = initialiseDeliveryManager(isBlockChainSynced = isChainSynced, isMining = isMining, testNetSettings) val (_: InetSocketAddress, cp1: ConnectedPeer) = createPeer(9001, "172.16.13.10", testNetSettings) val (_: InetSocketAddress, cp2: ConnectedPeer) = createPeer(9002, "172.16.13.11", testNetSettings) val (_: InetSocketAddress, cp3: ConnectedPeer) = createPeer(9003, "172.16.13.12", testNetSettings) val blocks: List[Block] = generateBlocks(10, generateDummyHistory(testNetSettings))._2 val headersIds: List[ModifierId] = blocks.map(_.header.id) val headersAsKey = headersIds.map(toKey) - (deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) + (networkRouter, deliveryManager, cp1, cp2, cp3, blocks, headersIds, headersAsKey) } "RequestModifies" should { "handle uniq modifiers from RequestFromLocal message correctly" in { - val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() + val (_, deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) @@ -61,7 +61,7 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte deliveryManager.stop() } "not handle repeating modifiers from RequestFromLocal message" in { - val (deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() + val (_, deliveryManager, cp1, _, _, _, headersIds, headersAsKey) = initialiseState() val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) @@ -72,7 +72,7 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte deliveryManager.stop() } "Delivery Manager should handle received modifier which were requested correctly" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() + val (_, deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) @@ -86,7 +86,7 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte deliveryManager.stop() } "Delivery manager should not handle repeating modifiers" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() + val (_, deliveryManager, cp1, _, _, blocks, headersIds, headersAsKey) = initialiseState() val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) @@ -101,7 +101,7 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte deliveryManager.stop() } "handle priority request for payload correctly" in { - val (deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() + val (_, deliveryManager, cp1, _, _, blocks, headersIds, _) = initialiseState() val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = Map(cp1.socketAddress -> (cp1, Older, InitialPriority)) @@ -114,89 +114,89 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte assert(deliveryManager.underlyingActor.expectedModifiers.size == blocks.size) deliveryManager.stop() } - "choose correct peer in priority request" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val address2 = new InetSocketAddress("123.123.123.124", 9001) - val handler2: TestProbe = TestProbe() - val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.124", Some(address2), System.currentTimeMillis())) - - val address3 = new InetSocketAddress("123.123.123.125", 9001) - val handler3: TestProbe = TestProbe() - val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.125", Some(address3), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map( - address1 -> (cp1, Older, InitialPriority), - address2 -> (cp2, Older, InitialPriority), - address3 -> (cp3, Older, InitialPriority) - ) - - - - val header: Header = blocks.head.header - - deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) - deliveryManager ! RequestSent(cp2.socketAddress, Header.modifierTypeId, header.id) - deliveryManager ! RequestSent(cp3.socketAddress, Header.modifierTypeId, header.id) - - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1.socketAddress) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2.socketAddress) - deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3.socketAddress) - - deliveryManager ! RequestSent(cp1.socketAddress, Payload.modifierTypeId, header.payloadId) - - handler1.expectMsgAnyOf( - RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), - RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), - SyncInfoNetworkMessage(SyncInfo(List())) - ) - - handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - deliveryManager.stop() - } - "not ask modifiers from peer which is not contained in status tracker" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() - - val address1 = new InetSocketAddress("123.123.123.123", 9001) - val handler1: TestProbe = TestProbe() - val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.123", Some(address1), System.currentTimeMillis())) - - val address2 = new InetSocketAddress("123.123.123.124", 9001) - val handler2: TestProbe = TestProbe() - val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, - Handshake(protocolToBytes(testNetSettings.network.appVersion), - "123.123.123.124", Some(address2), System.currentTimeMillis())) - - val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = - Map(address2 -> (cp2, Older, InitialPriority)) - - - - val header: Header = blocks.head.header - - deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) - deliveryManager ! RequestSent(cp2.socketAddress, Header.modifierTypeId, header.id) - - handler1.expectNoMsg() - handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) - deliveryManager.stop() - } + //todo: reinit +// "choose correct peer in priority request" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val address2 = new InetSocketAddress("123.123.123.124", 9001) +// val handler2: TestProbe = TestProbe() +// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.124", Some(address2), System.currentTimeMillis())) +// +// val address3 = new InetSocketAddress("123.123.123.125", 9001) +// val handler3: TestProbe = TestProbe() +// val cp3: ConnectedPeer = ConnectedPeer(address3, handler3.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.125", Some(address3), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map( +// address1 -> (cp1, Older, InitialPriority), +// address2 -> (cp2, Older, InitialPriority), +// address3 -> (cp3, Older, InitialPriority) +// ) +// +// +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) +// deliveryManager ! RequestSent(cp2.socketAddress, Header.modifierTypeId, header.id) +// deliveryManager ! RequestSent(cp3.socketAddress, Header.modifierTypeId, header.id) +// +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp1.socketAddress) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp2.socketAddress) +// deliveryManager ! DataFromPeer(ModifiersNetworkMessage(Header.modifierTypeId, Map(header.id -> header.bytes)), cp3.socketAddress) +// +// deliveryManager ! RequestSent(cp1.socketAddress, Payload.modifierTypeId, header.payloadId) +// +// handler1.expectMsgAnyOf( +// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id)), +// RequestModifiersNetworkMessage(Payload.modifierTypeId -> Seq(header.payloadId)), +// SyncInfoNetworkMessage(SyncInfo(List())) +// ) +// +// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// handler3.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// deliveryManager.stop() +// } + //todo: reinit +// "not ask modifiers from peer which is not contained in status tracker" in { +// val (deliveryManager, _, _, _, blocks, _, _) = initialiseState() +// +// val address1 = new InetSocketAddress("123.123.123.123", 9001) +// val handler1: TestProbe = TestProbe() +// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.123", Some(address1), System.currentTimeMillis())) +// +// val address2 = new InetSocketAddress("123.123.123.124", 9001) +// val handler2: TestProbe = TestProbe() +// val cp2: ConnectedPeer = ConnectedPeer(address2, handler2.ref, Incoming, +// Handshake(protocolToBytes(testNetSettings.network.appVersion), +// "123.123.123.124", Some(address2), System.currentTimeMillis())) +// +// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = +// Map(address2 -> (cp2, Older, InitialPriority)) +// +// val header: Header = blocks.head.header +// +// deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header.id) +// deliveryManager ! RequestSent(cp2.socketAddress, Header.modifierTypeId, header.id) +// +// handler1.expectNoMsg() +// handler2.expectMsgAllOf(RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header.id))) +// deliveryManager.stop() +// } "not ask transactions while block chain is not synced" in { - val (deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) + val (_, deliveryManager, _, _, _, _, _, _) = initialiseState(isChainSynced = false) val txs: Seq[Transaction] = genInvalidPaymentTxs(1) val address1 = new InetSocketAddress("123.123.123.123", 9001) @@ -216,7 +216,7 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte deliveryManager.stop() } "not ask transaction while node is not mining" in { - val (deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) + val (_, deliveryManager, _, _, _, _, _, _) = initialiseState(isMining = false) val txs: Seq[Transaction] = genInvalidPaymentTxs(1) val address1 = new InetSocketAddress("123.123.123.123", 9001) @@ -236,7 +236,7 @@ class DeliveryManagerRequestModifiesSpec extends WordSpecLike with BeforeAndAfte deliveryManager.stop() } "not re-ask modifiers which already have been received" in { - val (deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) + val (_, deliveryManager, _, _, _, blocks, _, _) = initialiseState(isChainSynced = false) val address1 = new InetSocketAddress("123.123.123.123", 9001) val handler1: TestProbe = TestProbe() From b40968df21976b49a0a43e2419a375a8e419a945 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 19 Mar 2020 14:34:36 +0300 Subject: [PATCH 119/177] fix some tests --- src/main/scala/encry/network/DM.scala | 4 -- src/main/scala/encry/network/PK.scala | 2 +- .../scala/encry/network/BlackListTests.scala | 4 +- ...DeliveryManagerReRequestModifiesSpec.scala | 67 ++++++++----------- 4 files changed, 31 insertions(+), 46 deletions(-) diff --git a/src/main/scala/encry/network/DM.scala b/src/main/scala/encry/network/DM.scala index 2bd849cdae..fa4944da65 100644 --- a/src/main/scala/encry/network/DM.scala +++ b/src/main/scala/encry/network/DM.scala @@ -1,9 +1,7 @@ package encry.network import java.net.InetSocketAddress - import akka.actor.{Actor, Props} -import akka.pattern._ import com.typesafe.scalalogging.StrictLogging import encry.network.DM.{AwaitingRequest, IsRequested, RequestSent, RequestStatus} import encry.network.Messages.MessageToNetwork.RequestFromLocal @@ -16,9 +14,7 @@ import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} import cats.syntax.option._ import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction - import scala.collection.mutable -import scala.concurrent.Future case class DM(networkSettings: NetworkSettings) extends Actor with StrictLogging { diff --git a/src/main/scala/encry/network/PK.scala b/src/main/scala/encry/network/PK.scala index fd0362a5e1..a5375ec127 100644 --- a/src/main/scala/encry/network/PK.scala +++ b/src/main/scala/encry/network/PK.scala @@ -2,7 +2,7 @@ package encry.network import java.net.{InetAddress, InetSocketAddress} -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{Actor, Props} import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.UpdatingPeersInfo import encry.network.BlackList.BanReason.SentPeersMessageWithoutRequest diff --git a/src/test/scala/encry/network/BlackListTests.scala b/src/test/scala/encry/network/BlackListTests.scala index ada46f14a7..30ea0749ef 100644 --- a/src/test/scala/encry/network/BlackListTests.scala +++ b/src/test/scala/encry/network/BlackListTests.scala @@ -53,8 +53,8 @@ class BlackListTests extends WordSpecLike val newBL1 = newBL.cleanupBlackList newBL1.contains(peer) shouldBe false } - "don't remove peer from black list before ban time expired" in { - val blackList: BlackList = BlackList(settings.blackList.copy(banTime = 1 millisecond)) + "not remove peer from black list before ban time expired" in { + val blackList: BlackList = BlackList(settings.blackList.copy(banTime = 1 minute)) val peer: InetAddress = new InetSocketAddress("0.0.0.0", 9000).getAddress val newBL = blackList.banPeer(SentInvForPayload, peer) val newBL1 = newBL.cleanupBlackList diff --git a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala index d3fe361312..112f3a28b0 100644 --- a/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala +++ b/src/test/scala/encry/network/DeliveryManagerTests/DeliveryManagerReRequestModifiesSpec.scala @@ -4,30 +4,23 @@ import java.net.InetSocketAddress import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestProbe} -import encry.consensus.HistoryConsensus -import encry.consensus.HistoryConsensus.Older import encry.modifiers.InstanceFactory +import encry.network.DM import encry.network.DM.RequestSent -import encry.network.{DM, DeliveryManager} import encry.network.DeliveryManagerTests.DMUtils._ import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NetworkController.ReceivableMessages.{DataFromPeer, RegisterMessagesHandler} -import encry.network.NodeViewSynchronizer.ReceivableMessages._ import encry.network.PeerConnectionHandler.{ConnectedPeer, Incoming} -import encry.network.PeersKeeper.UpdatedPeersCollection -import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus.InitialPriority -import encry.network.PrioritiesCalculator.PeersPriorityStatus.PeersPriorityStatus -import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier import encry.settings.TestNetSettings import encry.view.history.History import org.encryfoundation.common.modifiers.history.{Block, Header, HeaderProtoSerializer} import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.network.BasicMessagesRepo.{GetPeersNetworkMessage, Handshake, ModifiersNetworkMessage, PeersNetworkMessage, RequestModifiersNetworkMessage} +import org.encryfoundation.common.network.BasicMessagesRepo.{Handshake, ModifiersNetworkMessage} import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} -import scala.concurrent.duration._ import scala.collection.mutable.WrappedArray +import scala.concurrent.duration._ class DeliveryManagerReRequestModifiesSpec extends WordSpecLike with BeforeAndAfterAll @@ -55,35 +48,31 @@ class DeliveryManagerReRequestModifiesSpec extends WordSpecLike "ReRequestModifies" should { "re-ask necessary modifier several times (number of attempts from testNetSettings) and remove modifier from " + //todo: move to message builder tests -// "expectedModifiers collection after all attempts will expire" in { -// val (deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() -// -// val address1 = new InetSocketAddress("123.123.123.123", 9001) -// val handler1: TestProbe = TestProbe() -// val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, -// Handshake(protocolToBytes(testNetSettings.network.appVersion), -// "123.123.123.123", Some(address1), System.currentTimeMillis())) -// -// val updatedPeersCollection: Map[InetSocketAddress, (ConnectedPeer, HistoryConsensus.Older.type, PeersPriorityStatus)] = -// Map(address1 -> (cp1, Older, InitialPriority)) -// -// deliveryManager ! UpdatedPeersCollection(updatedPeersCollection) -// -// val header: ModifierId = headersIds.head -// -// deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header) -// handler1.expectMsgAllOf( -// testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)), -// RequestModifiersNetworkMessage(Header.modifierTypeId -> Seq(header)) -// ) -// //this thread sleep is using for expecting modifier removal -// Thread.sleep(6000) -// -// assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) -// deliveryManager.stop() -// } + "expectedModifiers collection after all attempts will expire" in { + val (networkRouter, deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() + networkRouter.expectMsg(RegisterMessagesHandler(Seq( + ModifiersNetworkMessage.NetworkMessageTypeID -> "ModifiersNetworkMessage", + ), deliveryManager.underlying.self)) + val address1 = new InetSocketAddress("123.123.123.123", 9001) + val handler1: TestProbe = TestProbe() + val cp1: ConnectedPeer = ConnectedPeer(address1, handler1.ref, Incoming, + Handshake(protocolToBytes(testNetSettings.network.appVersion), + "123.123.123.123", Some(address1), System.currentTimeMillis())) + + val header: ModifierId = headersIds.head + + deliveryManager ! RequestSent(cp1.socketAddress, Header.modifierTypeId, header) + networkRouter.expectMsgAllOf( + testNetSettings.network.deliveryTimeout * (testNetSettings.network.maxDeliveryChecks + 2), + RequestFromLocal(Some(cp1.socketAddress), Header.modifierTypeId, List(header)), + RequestFromLocal(Some(cp1.socketAddress), Header.modifierTypeId, List(header)), + ) + //this thread sleep is using for expecting modifier removal + Thread.sleep(6000) + + assert(deliveryManager.underlyingActor.expectedModifiers.isEmpty) + deliveryManager.stop() + } "not re-ask unnecessary modifiers" in { val (networkRouter, deliveryManager, _, _, _, _, headersIds, _, _) = initialiseState() networkRouter.expectMsg(RegisterMessagesHandler(Seq( From becdbcd0e35b1ecc00a888adb701e2fd31298813 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 19 Mar 2020 17:53:09 +0300 Subject: [PATCH 120/177] add NVH(History/State) --- src/main/scala/encry/api/http/DataHolderForApi.scala | 1 + src/main/scala/encry/nvg/NVHHistory.scala | 5 +++++ src/main/scala/encry/nvg/NVHState.scala | 5 +++++ 3 files changed, 11 insertions(+) create mode 100644 src/main/scala/encry/nvg/NVHHistory.scala create mode 100644 src/main/scala/encry/nvg/NVHState.scala diff --git a/src/main/scala/encry/api/http/DataHolderForApi.scala b/src/main/scala/encry/api/http/DataHolderForApi.scala index 4382cab089..5970961611 100644 --- a/src/main/scala/encry/api/http/DataHolderForApi.scala +++ b/src/main/scala/encry/api/http/DataHolderForApi.scala @@ -57,6 +57,7 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) def awaitNVHRef: Receive = { case UpdateHistoryReader(history) => unstashAll() + logger.info("Got updated history at nvh") context.become(workingCycle(nvhRef = sender(), history = Some(history))) case PassForStorage(_) => stash() diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala new file mode 100644 index 0000000000..3c7c4d3b77 --- /dev/null +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -0,0 +1,5 @@ +package encry.nvg + +class NVHHistory { + +} diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala new file mode 100644 index 0000000000..1242319d62 --- /dev/null +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -0,0 +1,5 @@ +package encry.nvg + +class NVHState { + +} From 3f3bd360e677b21fca139c08df176e63ef17f0ea Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 19 Mar 2020 18:16:31 +0300 Subject: [PATCH 121/177] add simple messages for nvhState --- src/main/scala/encry/nvg/NVHState.scala | 28 ++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index 1242319d62..465c9bf230 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -1,5 +1,31 @@ package encry.nvg -class NVHState { +import java.io.File +import akka.actor.{Actor, ActorRef, Props} +import encry.settings.{EncryAppSettings, NodeSettings} +import encry.view.NodeViewErrors.ModifierApplyError +import encry.view.state.UtxoState +import org.encryfoundation.common.utils.TaggedTypes.ModifierId + +class NVHState(influxRef: Option[ActorRef]) extends Actor { + override def receive: Receive = ??? +} + +object NVHState { + + sealed trait StateAction + object StateAction { + case class ModifierApplied(modifierId: ModifierId) extends StateAction + case class Rollback(branchPoint: ModifierId) extends StateAction + case class ValidationFailed(modifierId: ModifierId, errs: List[ModifierApplyError]) extends StateAction + } + + def props(settings: EncryAppSettings, branchPoint: ModifierId, influxRef: Option[ActorRef]): Props = { + val stateDir: File = UtxoState.getStateDir(settings) + val rootsDir: File = UtxoState.getRootsDir(settings) + val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) + Props(new NVHState(influxRef)) + } + def props(settings: EncryAppSettings, influxRef: Option[ActorRef]): Props = Props(new NVHState(influxRef)) } From 16c15e92bcf1601c4203814010a95634cd12b6c1 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 19 Mar 2020 18:34:01 +0300 Subject: [PATCH 122/177] simple props --- src/main/scala/encry/nvg/NVHState.scala | 78 +++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 5 deletions(-) diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index 465c9bf230..ff8c458ada 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -3,16 +3,24 @@ package encry.nvg import java.io.File import akka.actor.{Actor, ActorRef, Props} +import cats.syntax.option.none +import com.typesafe.scalalogging.StrictLogging import encry.settings.{EncryAppSettings, NodeSettings} +import encry.utils.CoreTaggedTypes.VersionTag import encry.view.NodeViewErrors.ModifierApplyError +import encry.view.history.History import encry.view.state.UtxoState -import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import org.encryfoundation.common.modifiers.history.Block +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId} + +class NVHState(influxRef: Option[ActorRef], var state: UtxoState) extends Actor { -class NVHState(influxRef: Option[ActorRef]) extends Actor { override def receive: Receive = ??? + } -object NVHState { +object NVHState extends StrictLogging { sealed trait StateAction object StateAction { @@ -21,11 +29,71 @@ object NVHState { case class ValidationFailed(modifierId: ModifierId, errs: List[ModifierApplyError]) extends StateAction } + //genesis state def props(settings: EncryAppSettings, branchPoint: ModifierId, influxRef: Option[ActorRef]): Props = { val stateDir: File = UtxoState.getStateDir(settings) val rootsDir: File = UtxoState.getRootsDir(settings) val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) - Props(new NVHState(influxRef)) + Props(new NVHState(influxRef, state)) + } + + //restoreConsistentState + def props(settings: EncryAppSettings, + branchPoint: ModifierId, + bestBlock: Option[Block], + influxRef: Option[ActorRef]): Props = { + val stateDir: File = UtxoState.getStateDir(settings) + val rootsDir: File = UtxoState.getRootsDir(settings) + val state = UtxoState.create(stateDir, rootsDir, settings, influxRef) + Props(new NVHState(influxRef, state)) + } + + def restoreConsistentState(stateIn: UtxoState, + history: History, + influxRefActor: Option[ActorRef], + appSettings: EncryAppSettings): UtxoState = + (stateIn.version, history.getBestBlock, stateIn, stateIn.safePointHeight) match { + case (stateId, None, _, _) if stateId sameElements Array.emptyByteArray => + logger.info(s"State and history are both empty on startup") + stateIn + case (_, None, _, _) => + logger.info( + s"State and history are inconsistent." + + s" History is empty on startup, rollback state to genesis." + ) + getRecreatedState(influxRef = influxRefActor, settings = appSettings) + case (_, Some(historyBestBlock), state: UtxoState, safePointHeight) => + val headerAtSafePointHeight = history.getBestHeaderAtHeight(safePointHeight) + val (rollbackId, newChain) = history.getChainToHeader(headerAtSafePointHeight, historyBestBlock.header) + logger.info( + s"State and history are inconsistent." + + s" Going to rollback to ${rollbackId.map(Algos.encode)} and " + + s"apply ${newChain.length} modifiers. State safe point: ${safePointHeight}. ${newChain.headers.head.height}. ${newChain.headers.last.height}" + ) + val additionalBlocks = + (state.safePointHeight + 1 to historyBestBlock.header.height).foldLeft(List.empty[Block]) { + case (blocks, height) => + val headerAtHeight = history.getBestHeaderAtHeight(height).get + val blockAtHeight = history.getBlockByHeader(headerAtHeight).get + blocks :+ blockAtHeight + } + logger.info(s"Qty of additional blocks: ${additionalBlocks.length}") + rollbackId + .map(_ => state.restore(additionalBlocks).get) + .getOrElse(getRecreatedState(influxRef = influxRefActor, settings = appSettings)) + } + + def getRecreatedState(version: Option[VersionTag] = none, + digest: Option[ADDigest] = none, + influxRef: Option[ActorRef], + settings: EncryAppSettings): UtxoState = { + val dir: File = UtxoState.getStateDir(settings) + dir.mkdirs() + dir.listFiles.foreach(_.delete()) + val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdirs() + val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() + UtxoState.create(stateDir, rootsDir, settings, influxRef) } - def props(settings: EncryAppSettings, influxRef: Option[ActorRef]): Props = Props(new NVHState(influxRef)) } From 341eb4641523e7f64a087a8a6b98a9a86e61290c Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 20 Mar 2020 11:18:55 +0300 Subject: [PATCH 123/177] add some pipelines to nvhState --- src/main/scala/encry/nvg/NVHState.scala | 116 ++++++++++++++++-- .../encry/view/history/HistoryReader.scala | 8 ++ .../scala/encry/view/state/UtxoState.scala | 13 ++ 3 files changed, 127 insertions(+), 10 deletions(-) diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index ff8c458ada..b41abd5e7b 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -5,28 +5,74 @@ import java.io.File import akka.actor.{Actor, ActorRef, Props} import cats.syntax.option.none import com.typesafe.scalalogging.StrictLogging +import encry.api.http.DataHolderForApi.BlockAndHeaderInfo +import encry.consensus.HistoryConsensus.ProgressInfo +import encry.nvg.NVHState.StateAction.{ApplyFailed, ApplyModifier, CreateTreeChunks, ModifierApplied} +import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier, UpdateInformation} +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId +import encry.nvg.fast.sync.SnapshotProcessor.{SnapshotChunk, TreeChunks} import encry.settings.{EncryAppSettings, NodeSettings} +import encry.stats.StatsSender.{HeightStatistics, ModifierAppendedToState, TransactionsInBlock} +import encry.storage.{RootNodesStorage, VersionalStorage} +import encry.storage.VersionalStorage.{StorageKey, StorageValue} +import encry.storage.iodb.versionalIODB.IODBWrapper +import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} import encry.utils.CoreTaggedTypes.VersionTag import encry.view.NodeViewErrors.ModifierApplyError -import encry.view.history.History +import encry.view.history.{History, HistoryReader} import encry.view.state.UtxoState -import org.encryfoundation.common.modifiers.history.Block +import encry.view.state.UtxoState.logger +import encry.view.state.avlTree.AvlTree +import io.iohk.iodb.{ByteArrayWrapper, LSMStore} +import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId} +import org.encryfoundation.common.utils.constants.Constants +import org.iq80.leveldb.Options +import encry.view.state.avlTree.utils.implicits.Instances._ -class NVHState(influxRef: Option[ActorRef], var state: UtxoState) extends Actor { - - override def receive: Receive = ??? +class NVHState(influxRef: Option[ActorRef], var state: UtxoState, settings: EncryAppSettings) extends Actor { + override def receive: Receive = { + case ApplyModifier(modifier: PersistentModifier, + saveRootNodesFlag: Boolean, + isFullChainSynced: Boolean) => + state.applyModifier(modifier, saveRootNodesFlag) match { + case Right(stateAfterApply) => + modifier match { + case b: Block if isFullChainSynced => context.parent ! TransactionsInBlock(b.payload.txs.size) + case _ => + } + state = stateAfterApply + context.parent ! ModifierApplied(modifier.id) + case Left(e: List[ModifierApplyError]) => + logger.info(s"Application to state failed cause $e") + context.parent ! ApplyFailed(modifier.id, e) + } + case CreateTreeChunks => + context.parent ! AvlTree.getChunks( + state.tree.rootNode, + currentChunkHeight = settings.snapshotSettings.chunkDepth, + state.tree.avlStorage + ) + } } object NVHState extends StrictLogging { + import encry.view.state.avlTree.utils.implicits.Instances._ + sealed trait StateAction object StateAction { case class ModifierApplied(modifierId: ModifierId) extends StateAction case class Rollback(branchPoint: ModifierId) extends StateAction - case class ValidationFailed(modifierId: ModifierId, errs: List[ModifierApplyError]) extends StateAction + case class ApplyFailed(modifierId: ModifierId, errs: List[ModifierApplyError]) extends StateAction + case class ApplyModifier(modifier: PersistentModifier, + saveRootNodesFlag: Boolean, + isFullChainSynced: Boolean) extends StateAction + case object CreateTreeChunks extends StateAction + case class TreeChunks(chunks: List[SnapshotChunk]) extends StateAction } //genesis state @@ -39,17 +85,67 @@ object NVHState extends StrictLogging { //restoreConsistentState def props(settings: EncryAppSettings, - branchPoint: ModifierId, - bestBlock: Option[Block], + historyReader: HistoryReader, influxRef: Option[ActorRef]): Props = { val stateDir: File = UtxoState.getStateDir(settings) val rootsDir: File = UtxoState.getRootsDir(settings) - val state = UtxoState.create(stateDir, rootsDir, settings, influxRef) + val state: UtxoState = restoreConsistentState( + UtxoState.create(stateDir, rootsDir, settings, influxRef), + historyReader, + influxRef, + settings + ) + Props(new NVHState(influxRef, state)) + } + + //rollback + def props(settings: EncryAppSettings, + influxRef: Option[ActorRef], + safePointHeight: Int, + branchPoint: VersionTag, + historyReader: HistoryReader, + constants: Constants): Props = { + val branchPointHeight: Int = historyReader.getHeaderById(ModifierId !@@ branchPoint).get.height + val additionalBlocks: List[Block] = + (safePointHeight + 1 to branchPointHeight).foldLeft(List.empty[Block]) { + case (blocks: List[Block], height: Int) => + val headerAtHeight: Header = historyReader.getBestHeaderAtHeight(height).get + val blockAtHeight: Block = historyReader.getBlockByHeader(headerAtHeight).get + blocks :+ blockAtHeight + } + val dir: File = UtxoState.getStateDir(settings) + dir.mkdirs() + dir.listFiles.foreach(_.delete()) + val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdirs() + val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() + val versionalStorage = settings.storage.state match { + case VersionalStorage.IODB => + logger.info("Init state with iodb storage") + IODBWrapper(new LSMStore(stateDir, keepVersions = settings.constants.DefaultKeepVersions)) + case VersionalStorage.LevelDB => + logger.info("Init state with levelDB storage") + val levelDBInit = LevelDbFactory.factory.open(stateDir, new Options) + VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settings.levelDB.copy(keySize = 33), keySize = 33)) + } + val rootStorage = { + val levelDBInit = LevelDbFactory.factory.open(rootsDir, new Options) + RootNodesStorage[StorageKey, StorageValue](levelDBInit, settings.constants.MaxRollbackDepth, rootsDir) + } + val state: UtxoState = UtxoState.rollbackTo( + branchPoint, + additionalBlocks, + versionalStorage, + rootStorage, + constants, + influxRef + ).get Props(new NVHState(influxRef, state)) } def restoreConsistentState(stateIn: UtxoState, - history: History, + history: HistoryReader, influxRefActor: Option[ActorRef], appSettings: EncryAppSettings): UtxoState = (stateIn.version, history.getBestBlock, stateIn, stateIn.safePointHeight) match { diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 233ee8f491..238e2f069e 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -22,6 +22,9 @@ trait HistoryReader { def getHeaderById(id: ModifierId): Option[Header] + def getChainToHeader(fromHeaderOpt: Option[Header], + toHeader: Header): (Option[ModifierId], HeaderChain) + def getBlockByHeaderId(id: ModifierId): Option[Block] def getBlockByHeader(header: Header): Option[Block] @@ -69,6 +72,8 @@ object HistoryReader { def lastHeaders(count: Int): HeaderChain = HeaderChain.empty def getBestHeader: Option[Header] = None def getBestBlock: Option[Block] = None + def getChainToHeader(fromHeaderOpt: Option[Header], + toHeader: Header): (Option[ModifierId], HeaderChain) = (None, HeaderChain.empty) } def apply(history: History): HistoryReader = new HistoryReader { @@ -90,5 +95,8 @@ object HistoryReader { def lastHeaders(count: Int): HeaderChain = history.lastHeaders(count) def getBestHeader: Option[Header] = history.getBestHeader def getBestBlock: Option[Block] = history.getBestBlock + def getChainToHeader(fromHeaderOpt: Option[Header], + toHeader: Header): (Option[ModifierId], HeaderChain) = history.getChainToHeader(fromHeaderOpt, + toHeader) } } diff --git a/src/main/scala/encry/view/state/UtxoState.scala b/src/main/scala/encry/view/state/UtxoState.scala index ec9d901776..4ada50c733 100644 --- a/src/main/scala/encry/view/state/UtxoState.scala +++ b/src/main/scala/encry/view/state/UtxoState.scala @@ -226,6 +226,19 @@ object UtxoState extends StrictLogging { def initialStateBoxes: List[AssetBox] = List(AssetBox(EncryProposition.open, -9, 0)) + def rollbackTo(version: VersionTag, + additionalBlocks: List[Block], + avlStorage: VersionalStorage, + rootNodesStorage: RootNodesStorage[StorageKey, StorageValue], + constants: Constants, + influxRef: Option[ActorRef]): Try[UtxoState] = Try { + logger.info(s"Rollback utxo to version: ${Algos.encode(version)}") + val rollbackedAvl = AvlTree.rollbackTo(StorageVersion !@@ version, additionalBlocks, avlStorage, rootNodesStorage).get + logger.info(s"UTXO -> rollbackTo -> ${avlStorage.get(UtxoState.bestHeightKey).map(Ints.fromByteArray)}.") + val height: Height = Height !@@ Ints.fromByteArray(avlStorage.get(UtxoState.bestHeightKey).get) + UtxoState(rollbackedAvl, height, constants, influxRef) + } + def getStateDir(settings: EncryAppSettings): File = { logger.info(s"Invoke getStateDir") if (settings.snapshotSettings.enableFastSynchronization) { From a61651b43f89bb3b69cd97a81bf993b0dbfcdc01 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 20 Mar 2020 11:34:03 +0300 Subject: [PATCH 124/177] add IntermediaryNVHView actor --- .../scala/encry/nvg/IntermediaryNVHView.scala | 19 +++++++++++++++++++ src/main/scala/encry/nvg/NVHState.scala | 6 +++--- 2 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 src/main/scala/encry/nvg/IntermediaryNVHView.scala diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala new file mode 100644 index 0000000000..49592db3b6 --- /dev/null +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -0,0 +1,19 @@ +package encry.nvg + +import akka.actor.{Actor, ActorRef, Props} +import encry.network.NetworkRouter.ModifierFromNetwork +import encry.view.history.History +import encry.view.state.UtxoState + +class IntermediaryNVHView() extends Actor { + + override def receive: Receive = ??? + + def viewReceive(history: ActorRef, state: ActorRef): Receive = { + case ModifierFromNetwork(remote, typeId, modifierId, modifierBytes) => history ! ModifierFromNetwork + } +} + +object IntermediaryNVHView { + def props(): Props = Props(new IntermediaryNVHView()) +} diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index b41abd5e7b..367a48096b 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -80,7 +80,7 @@ object NVHState extends StrictLogging { val stateDir: File = UtxoState.getStateDir(settings) val rootsDir: File = UtxoState.getRootsDir(settings) val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) - Props(new NVHState(influxRef, state)) + Props(new NVHState(influxRef, state, settings)) } //restoreConsistentState @@ -95,7 +95,7 @@ object NVHState extends StrictLogging { influxRef, settings ) - Props(new NVHState(influxRef, state)) + Props(new NVHState(influxRef, state, settings)) } //rollback @@ -141,7 +141,7 @@ object NVHState extends StrictLogging { constants, influxRef ).get - Props(new NVHState(influxRef, state)) + Props(new NVHState(influxRef, state, settings)) } def restoreConsistentState(stateIn: UtxoState, From 83f83f028441c4e0daca52d5d283f4b54d7993fe Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 20 Mar 2020 11:55:46 +0300 Subject: [PATCH 125/177] add awaitingViewActors: Receive to IntermediaryNVHView --- .../scala/encry/nvg/IntermediaryNVHView.scala | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index 49592db3b6..390e76022b 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -1,7 +1,10 @@ package encry.nvg import akka.actor.{Actor, ActorRef, Props} +import cats.syntax.option.none import encry.network.NetworkRouter.ModifierFromNetwork +import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{RegisterHistory, RegisterState} +import encry.nvg.NodeViewHolder.NodeView import encry.view.history.History import encry.view.state.UtxoState @@ -9,11 +12,33 @@ class IntermediaryNVHView() extends Actor { override def receive: Receive = ??? + def awaitingViewActors(history: Option[ActorRef], state: Option[ActorRef]): Receive = { + case RegisterHistory if state.isEmpty => + context.become(awaitingViewActors(Some(sender()), state), discardOld = true) + case RegisterHistory => + context.become(viewReceive(sender(), state.get)) + case RegisterState if history.isEmpty => + context.become(awaitingViewActors(history, Some(sender())), discardOld = true) + case RegisterHistory => + context.become(viewReceive(history.get, sender())) + } + def viewReceive(history: ActorRef, state: ActorRef): Receive = { case ModifierFromNetwork(remote, typeId, modifierId, modifierBytes) => history ! ModifierFromNetwork } + + def restoreState(influxRef: Option[ActorRef] = none): Unit = { + + } } object IntermediaryNVHView { + + sealed trait IntermediaryNVHViewActions + object IntermediaryNVHViewActions { + case object RegisterHistory extends IntermediaryNVHViewActions + case object RegisterState extends IntermediaryNVHViewActions + } + def props(): Props = Props(new IntermediaryNVHView()) } From 3c1d7fa8a4395861787abca6b9f909967fcc4f2e Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 20 Mar 2020 12:32:52 +0300 Subject: [PATCH 126/177] change some props in nvhstate --- .../scala/encry/nvg/IntermediaryNVHView.scala | 39 +++++++++++++------ src/main/scala/encry/nvg/NVHState.scala | 33 +++++++++------- 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index 390e76022b..c23031995a 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -1,21 +1,38 @@ package encry.nvg +import java.io.File + import akka.actor.{Actor, ActorRef, Props} import cats.syntax.option.none import encry.network.NetworkRouter.ModifierFromNetwork import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{RegisterHistory, RegisterState} import encry.nvg.NodeViewHolder.NodeView -import encry.view.history.History +import encry.settings.EncryAppSettings +import encry.utils.NetworkTimeProvider +import encry.view.history.{History, HistoryReader} import encry.view.state.UtxoState +import encry.view.wallet.EncryWallet +import org.apache.commons.io.FileUtils + +class IntermediaryNVHView(settings: EncryAppSettings, + ntp: NetworkTimeProvider, + influx: Option[ActorRef]) extends Actor { -class IntermediaryNVHView() extends Actor { + override def preStart(): Unit = { + //init history? + } - override def receive: Receive = ??? + override def receive: Receive = awaitingViewActors() - def awaitingViewActors(history: Option[ActorRef], state: Option[ActorRef]): Receive = { - case RegisterHistory if state.isEmpty => + def awaitingViewActors(history: Option[ActorRef] = None, state: Option[ActorRef] = None): Receive = { + case RegisterHistory(reader) if state.isEmpty => context.become(awaitingViewActors(Some(sender()), state), discardOld = true) - case RegisterHistory => + context.actorOf( + NVHState.restoreConsistentStateProps(settings, reader, influx).getOrElse( + NVHState.genesisProps(settings, influx) + ) + ) + case RegisterHistory(_) => context.become(viewReceive(sender(), state.get)) case RegisterState if history.isEmpty => context.become(awaitingViewActors(history, Some(sender())), discardOld = true) @@ -26,19 +43,17 @@ class IntermediaryNVHView() extends Actor { def viewReceive(history: ActorRef, state: ActorRef): Receive = { case ModifierFromNetwork(remote, typeId, modifierId, modifierBytes) => history ! ModifierFromNetwork } - - def restoreState(influxRef: Option[ActorRef] = none): Unit = { - - } } object IntermediaryNVHView { sealed trait IntermediaryNVHViewActions object IntermediaryNVHViewActions { - case object RegisterHistory extends IntermediaryNVHViewActions + case class RegisterHistory(historyReader: HistoryReader) extends IntermediaryNVHViewActions case object RegisterState extends IntermediaryNVHViewActions } - def props(): Props = Props(new IntermediaryNVHView()) + def props(settings: EncryAppSettings, + ntp: NetworkTimeProvider, + influxRef: Option[ActorRef]): Props = Props(new IntermediaryNVHView(settings, ntp, influxRef)) } diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index 367a48096b..3af02da625 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -7,6 +7,7 @@ import cats.syntax.option.none import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo +import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.RegisterState import encry.nvg.NVHState.StateAction.{ApplyFailed, ApplyModifier, CreateTreeChunks, ModifierApplied} import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier, UpdateInformation} import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId @@ -32,8 +33,12 @@ import org.encryfoundation.common.utils.constants.Constants import org.iq80.leveldb.Options import encry.view.state.avlTree.utils.implicits.Instances._ +import scala.util.Try + class NVHState(influxRef: Option[ActorRef], var state: UtxoState, settings: EncryAppSettings) extends Actor { + override def preStart(): Unit = context.parent ! RegisterState + override def receive: Receive = { case ApplyModifier(modifier: PersistentModifier, saveRootNodesFlag: Boolean, @@ -76,7 +81,7 @@ object NVHState extends StrictLogging { } //genesis state - def props(settings: EncryAppSettings, branchPoint: ModifierId, influxRef: Option[ActorRef]): Props = { + def genesisProps(settings: EncryAppSettings, influxRef: Option[ActorRef]): Props = { val stateDir: File = UtxoState.getStateDir(settings) val rootsDir: File = UtxoState.getRootsDir(settings) val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) @@ -84,18 +89,20 @@ object NVHState extends StrictLogging { } //restoreConsistentState - def props(settings: EncryAppSettings, - historyReader: HistoryReader, - influxRef: Option[ActorRef]): Props = { - val stateDir: File = UtxoState.getStateDir(settings) - val rootsDir: File = UtxoState.getRootsDir(settings) - val state: UtxoState = restoreConsistentState( - UtxoState.create(stateDir, rootsDir, settings, influxRef), - historyReader, - influxRef, - settings - ) - Props(new NVHState(influxRef, state, settings)) + def restoreConsistentStateProps(settings: EncryAppSettings, + historyReader: HistoryReader, + influxRef: Option[ActorRef]): Option[Props] = { + Try { + val stateDir: File = UtxoState.getStateDir(settings) + val rootsDir: File = UtxoState.getRootsDir(settings) + val state: UtxoState = restoreConsistentState( + UtxoState.create(stateDir, rootsDir, settings, influxRef), + historyReader, + influxRef, + settings + ) + Props(new NVHState(influxRef, state, settings)) + }.toOption } //rollback From cdbf7d10d787a54d927ed7cbaa9a30a4609d48c5 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Fri, 20 Mar 2020 18:35:12 +0300 Subject: [PATCH 127/177] add simple test for init genesis state by nvhstate and restoring state from storage by nvhState --- src/main/scala/encry/nvg/NVHState.scala | 19 +++- .../scala/encry/view/state/UtxoState.scala | 5 +- src/test/scala/encry/nvg/NVHStateTest.scala | 103 ++++++++++++++++++ 3 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 src/test/scala/encry/nvg/NVHStateTest.scala diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index 3af02da625..0902a9568d 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -35,7 +35,8 @@ import encry.view.state.avlTree.utils.implicits.Instances._ import scala.util.Try -class NVHState(influxRef: Option[ActorRef], var state: UtxoState, settings: EncryAppSettings) extends Actor { +class NVHState(influxRef: Option[ActorRef], var state: UtxoState, settings: EncryAppSettings) + extends Actor with StrictLogging with AutoCloseable { override def preStart(): Unit = context.parent ! RegisterState @@ -50,6 +51,7 @@ class NVHState(influxRef: Option[ActorRef], var state: UtxoState, settings: Encr case _ => } state = stateAfterApply + logger.info(s"Successfully apply modifier: ${Algos.encode(modifier.id)} of type ${modifier.modifierTypeId}") context.parent ! ModifierApplied(modifier.id) case Left(e: List[ModifierApplyError]) => logger.info(s"Application to state failed cause $e") @@ -62,6 +64,13 @@ class NVHState(influxRef: Option[ActorRef], var state: UtxoState, settings: Encr state.tree.avlStorage ) } + + override def postStop(): Unit = { + logger.info("Close state!") + state.close() + } + + override def close(): Unit = state.close() } object NVHState extends StrictLogging { @@ -83,7 +92,9 @@ object NVHState extends StrictLogging { //genesis state def genesisProps(settings: EncryAppSettings, influxRef: Option[ActorRef]): Props = { val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdir() val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) Props(new NVHState(influxRef, state, settings)) } @@ -91,10 +102,12 @@ object NVHState extends StrictLogging { //restoreConsistentState def restoreConsistentStateProps(settings: EncryAppSettings, historyReader: HistoryReader, - influxRef: Option[ActorRef]): Option[Props] = { + influxRef: Option[ActorRef]): Try[Props] = { Try { val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdirs() val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() val state: UtxoState = restoreConsistentState( UtxoState.create(stateDir, rootsDir, settings, influxRef), historyReader, @@ -102,7 +115,7 @@ object NVHState extends StrictLogging { settings ) Props(new NVHState(influxRef, state, settings)) - }.toOption + } } //rollback diff --git a/src/main/scala/encry/view/state/UtxoState.scala b/src/main/scala/encry/view/state/UtxoState.scala index 4ada50c733..c37e81bb0b 100644 --- a/src/main/scala/encry/view/state/UtxoState.scala +++ b/src/main/scala/encry/view/state/UtxoState.scala @@ -260,7 +260,7 @@ object UtxoState extends StrictLogging { val versionalStorage = settings.storage.state match { case VersionalStorage.IODB => logger.info("Init state with iodb storage") - IODBWrapper(new LSMStore(stateDir, keepVersions = settings.constants.DefaultKeepVersions)) + IODBWrapper(new LSMStore(stateDir, keepVersions = settings.constants.DefaultKeepVersions, keySize = 33)) case VersionalStorage.LevelDB => logger.info("Init state with levelDB storage") val levelDBInit = LevelDbFactory.factory.open(stateDir, new Options) @@ -297,7 +297,8 @@ object UtxoState extends StrictLogging { } storage.insert( StorageVersion @@ Array.fill(32)(0: Byte), - initialStateBoxes.map(bx => (StorageKey !@@ AvlTree.elementKey(bx.id), StorageValue @@ bx.bytes)) + initialStateBoxes.map(bx => (StorageKey !@@ AvlTree.elementKey(bx.id), StorageValue @@ bx.bytes)) :+ + (StorageKey @@ UtxoState.bestHeightKey -> StorageValue @@ Ints.toByteArray(-1)) ) UtxoState(AvlTree[StorageKey, StorageValue](storage, rootStorage), Height @@ 0, settings.constants, influxRef) } diff --git a/src/test/scala/encry/nvg/NVHStateTest.scala b/src/test/scala/encry/nvg/NVHStateTest.scala new file mode 100644 index 0000000000..1f97d1915e --- /dev/null +++ b/src/test/scala/encry/nvg/NVHStateTest.scala @@ -0,0 +1,103 @@ +package encry.nvg + +import akka.actor.ActorSystem +import akka.testkit.{TestActorRef, TestKit, TestProbe} +import com.typesafe.scalalogging.StrictLogging +import encry.consensus.HistoryConsensus +import encry.consensus.HistoryConsensus.Equal +import encry.modifiers.InstanceFactory +import encry.modifiers.history.HeaderChain +import encry.nvg.NVHState.StateAction.{ApplyModifier, ModifierApplied} +import encry.utils.FileHelper +import encry.view.history.HistoryReader +import org.encryfoundation.common.modifiers.history.{Block, Header} +import org.encryfoundation.common.network.SyncInfo +import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers, OneInstancePerTest, WordSpecLike} + +class NVHStateTest + extends TestKit(ActorSystem("Tested-Akka-System")) + with WordSpecLike + with Matchers + with BeforeAndAfterAll + with InstanceFactory + with OneInstancePerTest + with StrictLogging { + + "Nvh state" should { + "correctly init genesis state" in { + val stateDir = FileHelper.getRandomTempDir + val actor = TestActorRef[NVHState](NVHState.genesisProps(settings.copy(directory = stateDir.getAbsolutePath), None)) + } + "correctly recover state" in { + + val ((_, _, blocks), _) = PipelinesTests.genForOn(3) + + val historyReader = new HistoryReader { + + override def getBestHeaderHeight: Int = blocks.last.header.height + + override def getBestBlockHeight: Int = blocks.last.header.height + + override def getBestHeaderAtHeight(h: Int): Option[Header] = Some(blocks(h).header) + + override def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty + + override def compare(si: SyncInfo): HistoryConsensus.HistoryComparisonResult = Equal + + override def getHeaderById(id: ModifierId): Option[Header] = blocks.find(_.id sameElements id).map(_.header) + + override def getChainToHeader(fromHeaderOpt: Option[Header], + toHeader: Header): (Option[ModifierId], HeaderChain) = + fromHeaderOpt.map(header => + Some(header.id) -> HeaderChain(blocks.dropWhile(block => !(block.id sameElements header.id)).map(_.header)) + ).getOrElse(None -> HeaderChain.empty) + + override def getBlockByHeaderId(id: ModifierId): Option[Block] = blocks.find(_.id sameElements id) + + override def getBlockByHeader(header: Header): Option[Block] = blocks.find(_.id sameElements header.id) + + override var isFullChainSynced: Boolean = true + + override def isModifierDefined(id: ModifierId): Boolean = blocks.exists(_.id sameElements id) + + override def headerIdsAtHeight(height: Int): List[ModifierId] = List(blocks(height).id) + + override def modifierBytesById(id: ModifierId): Option[Array[Byte]] = blocks.find(_.id sameElements id).map(_.bytes) + + override def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty + + override def lastHeaders(count: Int): HeaderChain = HeaderChain.empty + + override def syncInfo: SyncInfo = SyncInfo(List.empty) + + override def isFastSyncInProcess: Boolean = false + + override def getBestHeader: Option[Header] = Some(blocks.last.header) + + override def getBestBlock: Option[Block] = Some(blocks.last) + + } + + val stateDir = FileHelper.getRandomTempDir + val emptyHistoryReader = HistoryReader.empty + val parent = TestProbe() + val actor = TestActorRef[NVHState]( + NVHState.genesisProps(settings.copy(directory = stateDir.getAbsolutePath), None), parent.ref + ) + blocks.sortBy(_.header.height).foreach { block => + actor ! ApplyModifier(block.header, saveRootNodesFlag = true, isFullChainSynced = true) + actor ! ApplyModifier(block, saveRootNodesFlag = true, isFullChainSynced = true) + } + Thread.sleep(5000) + actor.stop() + val recreatedActor = TestActorRef[NVHState](NVHState.restoreConsistentStateProps( + settings.copy(directory = stateDir.getAbsolutePath), + historyReader, + Some(TestProbe().ref) + ).get) + recreatedActor.underlyingActor.state.height shouldEqual blocks.last.header.height + recreatedActor.underlyingActor.state.tree.avlStorage.currentVersion shouldEqual blocks.last.id + } + } +} From cf4daafff2ec5f3ad2dd8d3c597fa85096d97c5d Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 23 Mar 2020 11:01:25 +0300 Subject: [PATCH 128/177] code cleanup --- .../scala/encry/nvg/IntermediaryNVHView.scala | 2 +- src/main/scala/encry/nvg/NVHState.scala | 4 +++- src/test/scala/encry/nvg/NVHStateTest.scala | 4 ++-- src/test/scala/encry/nvg/PipelinesTests.scala | 24 +++++++++++-------- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index c23031995a..56ba4c9250 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -28,7 +28,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, case RegisterHistory(reader) if state.isEmpty => context.become(awaitingViewActors(Some(sender()), state), discardOld = true) context.actorOf( - NVHState.restoreConsistentStateProps(settings, reader, influx).getOrElse( + NVHState.restoreConsistentStateProps(settings, reader, influx).getOrElse ( NVHState.genesisProps(settings, influx) ) ) diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index 0902a9568d..c9e2c7be64 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -93,8 +93,10 @@ object NVHState extends StrictLogging { def genesisProps(settings: EncryAppSettings, influxRef: Option[ActorRef]): Props = { val stateDir: File = UtxoState.getStateDir(settings) stateDir.mkdir() + stateDir.listFiles.foreach(_.delete()) val rootsDir: File = UtxoState.getRootsDir(settings) rootsDir.mkdir() + rootsDir.listFiles.foreach(_.delete()) val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) Props(new NVHState(influxRef, state, settings)) } @@ -108,7 +110,7 @@ object NVHState extends StrictLogging { stateDir.mkdirs() val rootsDir: File = UtxoState.getRootsDir(settings) rootsDir.mkdir() - val state: UtxoState = restoreConsistentState( + val state: UtxoState = restoreConsistentState ( UtxoState.create(stateDir, rootsDir, settings, influxRef), historyReader, influxRef, diff --git a/src/test/scala/encry/nvg/NVHStateTest.scala b/src/test/scala/encry/nvg/NVHStateTest.scala index 1f97d1915e..8f09d88cfb 100644 --- a/src/test/scala/encry/nvg/NVHStateTest.scala +++ b/src/test/scala/encry/nvg/NVHStateTest.scala @@ -7,13 +7,13 @@ import encry.consensus.HistoryConsensus import encry.consensus.HistoryConsensus.Equal import encry.modifiers.InstanceFactory import encry.modifiers.history.HeaderChain -import encry.nvg.NVHState.StateAction.{ApplyModifier, ModifierApplied} +import encry.nvg.NVHState.StateAction.ApplyModifier import encry.utils.FileHelper import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.history.{Block, Header} import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.ModifierId -import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers, OneInstancePerTest, WordSpecLike} +import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} class NVHStateTest extends TestKit(ActorSystem("Tested-Akka-System")) diff --git a/src/test/scala/encry/nvg/PipelinesTests.scala b/src/test/scala/encry/nvg/PipelinesTests.scala index 11691b812a..541de76946 100644 --- a/src/test/scala/encry/nvg/PipelinesTests.scala +++ b/src/test/scala/encry/nvg/PipelinesTests.scala @@ -2,19 +2,19 @@ package encry.nvg import java.net.InetSocketAddress -import akka.actor.{ ActorRef, ActorSystem } -import akka.testkit.{ TestActorRef, TestKit, TestProbe } +import akka.actor.{ActorRef, ActorSystem} +import akka.testkit.{TestActorRef, TestKit, TestProbe} import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.consensus.EncrySupplyController import encry.modifiers.InstanceFactory import encry.modifiers.mempool.TransactionFactory -import encry.network.Messages.MessageToNetwork.{ BroadcastModifier, RequestFromLocal, SendSyncInfo } +import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, SendSyncInfo} import encry.network.NetworkController.ReceivableMessages.RegisterMessagesHandler -import encry.network.NetworkRouter.{ ModifierFromNetwork, RegisterForModsHandling } +import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier -import encry.utils.implicits.UTXO.{ combineAll, _ } -import encry.utils.{ FileHelper, Mnemonic, NetworkTimeProvider, TestHelper } +import encry.utils.implicits.UTXO.{combineAll, _} +import encry.utils.{FileHelper, Mnemonic, NetworkTimeProvider, TestHelper} import encry.view.history.History import encry.view.state.UtxoState import encry.view.state.avlTree.utils.implicits.Instances._ @@ -24,9 +24,10 @@ import org.encryfoundation.common.crypto.equihash.EquihashSolution import org.encryfoundation.common.modifiers.history._ import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.modifiers.state.box.AssetBox -import org.encryfoundation.common.utils.TaggedTypes.{ Difficulty, Height } +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height} import org.encryfoundation.common.utils.constants.TestNetConstants -import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } +import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} import scala.concurrent.duration._ @@ -195,9 +196,9 @@ class PipelinesTests intermediary.underlyingActor.historyReader.getBestHeaderHeight shouldBe 300 intermediary.underlyingActor.historyReader.getBestBlockHeight shouldBe 300 - } "work with forks correctly" in { + logger.info("================ work with forks correctly start ====================") val tmpFile = FileHelper.getRandomTempDir val path = tmpFile.getAbsolutePath val settingsWithNewPath = @@ -205,6 +206,7 @@ class PipelinesTests .copy(directory = path) .copy(wallet = settings.wallet.map(_.copy(password = "123"))) .copy(node = settings.node.copy(isTestMod = true)) + AccountManager.init(Mnemonic.entropyToMnemonicCode(scorex.utils.Random.randomBytes(16)), "123", settingsWithNewPath) @@ -271,6 +273,8 @@ class PipelinesTests Thread.sleep(5000) + println("getBestBlock = " + Algos.encode(intermediary.underlyingActor.historyReader.getBestBlock.get.id)) + println("b2 = " + Algos.encode(b2.last.id)) intermediary.underlyingActor.historyReader.getBestBlock.get.id.sameElements(b2.last.id) shouldBe true intermediary.underlyingActor.historyReader.getBestHeader.get.id.sameElements(b2.last.id) shouldBe true @@ -545,6 +549,6 @@ object PipelinesTests extends InstanceFactory with StrictLogging { } val (h11, s11, b11) = generateValidForHistoryAndStateBlocks(5, h, s, h.getBestBlockHeight + 1) val (h22, s22, b22) = generateValidForHistoryAndStateBlocks(10, h1, s1, h1.getBestBlockHeight + 1) - ((h11, s11, b11 ++ b), (h22, s22, b22)) + ((h11, s11, (b11 ++ b).sortBy(_.header.height)), (h22, s22, b22.sortBy(_.header.height))) } } From 27fd73ba3c7537f84fb3aaddbe982faf1033cc20 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 23 Mar 2020 11:37:11 +0300 Subject: [PATCH 129/177] add msg about state failed mod application on interNvh --- src/main/scala/encry/nvg/IntermediaryNVHView.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index 56ba4c9250..bed7904ad0 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -6,6 +6,7 @@ import akka.actor.{Actor, ActorRef, Props} import cats.syntax.option.none import encry.network.NetworkRouter.ModifierFromNetwork import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{RegisterHistory, RegisterState} +import encry.nvg.NVHState.StateAction import encry.nvg.NodeViewHolder.NodeView import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider @@ -42,6 +43,9 @@ class IntermediaryNVHView(settings: EncryAppSettings, def viewReceive(history: ActorRef, state: ActorRef): Receive = { case ModifierFromNetwork(remote, typeId, modifierId, modifierBytes) => history ! ModifierFromNetwork + case StateAction.ApplyFailed(modId, errs) => + // todo: Notify history + } } From 198b27b1eeaf10d38d2dafccbafa1d5e306f0dd0 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 23 Mar 2020 11:59:22 +0300 Subject: [PATCH 130/177] miner pipeline --- src/main/scala/encry/nvg/IntermediaryNVHView.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index bed7904ad0..44fc26dcf7 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -45,8 +45,11 @@ class IntermediaryNVHView(settings: EncryAppSettings, case ModifierFromNetwork(remote, typeId, modifierId, modifierBytes) => history ! ModifierFromNetwork case StateAction.ApplyFailed(modId, errs) => // todo: Notify history - + case StateAction.ModifierApplied(modId) => + //todo: Notify history } + + def awaitingHistoryBranchPoint(history: ActorRef): Receive = ??? } object IntermediaryNVHView { From 86ae6eded333002a770cd342e4f79577b3b27da7 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 23 Mar 2020 13:09:18 +0300 Subject: [PATCH 131/177] nvh history actor frame added --- .../scala/encry/nvg/IntermediaryNVHView.scala | 79 ++++++++++---- src/main/scala/encry/nvg/ModifiersCache.scala | 70 ++++++------ src/main/scala/encry/nvg/NVHHistory.scala | 96 ++++++++++++++++- src/main/scala/encry/nvg/NodeViewHolder.scala | 3 +- .../view/history/FastSyncProcessor.scala | 13 ++- .../scala/encry/view/history/History.scala | 101 +++++++++-------- .../history/HistoryHeadersProcessor.scala | 50 ++++----- .../history/HistoryPayloadsProcessor.scala | 102 ++++++++++-------- .../encry/view/history/HistoryReader.scala | 31 ++++-- .../scala/encry/utils/EncryGenerator.scala | 2 + src/test/scala/encry/view/QWE.scala | 98 +++++++++++++++++ 11 files changed, 464 insertions(+), 181 deletions(-) create mode 100644 src/test/scala/encry/view/QWE.scala diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index 44fc26dcf7..2bae973585 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -1,37 +1,43 @@ package encry.nvg -import java.io.File - import akka.actor.{Actor, ActorRef, Props} -import cats.syntax.option.none -import encry.network.NetworkRouter.ModifierFromNetwork +import com.typesafe.scalalogging.StrictLogging import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{RegisterHistory, RegisterState} +import encry.nvg.IntermediaryNVHView.ModifierToAppend +import encry.nvg.ModifiersValidator.ValidatedModifier +import encry.nvg.NVHHistory.ProgressInfoForState import encry.nvg.NVHState.StateAction -import encry.nvg.NodeViewHolder.NodeView import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider -import encry.view.history.{History, HistoryReader} -import encry.view.state.UtxoState -import encry.view.wallet.EncryWallet -import org.apache.commons.io.FileUtils +import encry.view.history.HistoryReader +import org.encryfoundation.common.modifiers.PersistentModifier -class IntermediaryNVHView(settings: EncryAppSettings, - ntp: NetworkTimeProvider, - influx: Option[ActorRef]) extends Actor { +import scala.collection.mutable - override def preStart(): Unit = { - //init history? - } +class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, influx: Option[ActorRef]) + extends Actor + with StrictLogging { + + var historyReader: HistoryReader = HistoryReader.empty + + val historyRef: ActorRef = ActorRef.noSender + + var isProcessingModifierByHistory: Boolean = false + + var isProcessingModifierByState: Boolean = false override def receive: Receive = awaitingViewActors() def awaitingViewActors(history: Option[ActorRef] = None, state: Option[ActorRef] = None): Receive = { case RegisterHistory(reader) if state.isEmpty => + logger.info(s"NodeViewParent actor got init history. Going to init state actor.") context.become(awaitingViewActors(Some(sender()), state), discardOld = true) context.actorOf( - NVHState.restoreConsistentStateProps(settings, reader, influx).getOrElse ( - NVHState.genesisProps(settings, influx) - ) + NVHState + .restoreConsistentStateProps(settings, reader, influx) + .getOrElse( + NVHState.genesisProps(settings, influx) + ) ) case RegisterHistory(_) => context.become(viewReceive(sender(), state.get)) @@ -42,7 +48,32 @@ class IntermediaryNVHView(settings: EncryAppSettings, } def viewReceive(history: ActorRef, state: ActorRef): Receive = { - case ModifierFromNetwork(remote, typeId, modifierId, modifierBytes) => history ! ModifierFromNetwork + case ValidatedModifier(modifier: PersistentModifier) => + val wrappedKey: mutable.WrappedArray.ofByte = NodeViewHolder.toKey(modifier.id) + + val isInHistory: Boolean = historyReader.isModifierDefined(modifier.id) + + val isInCache: Boolean = ModifiersCache.contains(wrappedKey) + + if (isInHistory || isInCache) + logger.info( + s"Modifier ${modifier.encodedId} can't be placed into the cache cause: " + + s"contains in cache: $isInCache, contains in history: $isInHistory." + ) + else ModifiersCache.put(wrappedKey, modifier, historyReader, settings) + + if (!isProcessingModifierByHistory && !isProcessingModifierByState) { + isProcessingModifierByHistory = true + ModifiersCache + .popCandidate(historyReader, settings) + .foreach { mod: PersistentModifier => + logger.info(s"Got new modifiers in compute application function: ${mod.encodedId}.") + historyRef ! ModifierToAppend(mod, isLocallyGenerated = false) + } + } + + case ProgressInfoForState(pi) => + //todo work with state starts here case StateAction.ApplyFailed(modId, errs) => // todo: Notify history case StateAction.ModifierApplied(modId) => @@ -50,6 +81,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, } def awaitingHistoryBranchPoint(history: ActorRef): Receive = ??? + } object IntermediaryNVHView { @@ -57,10 +89,11 @@ object IntermediaryNVHView { sealed trait IntermediaryNVHViewActions object IntermediaryNVHViewActions { case class RegisterHistory(historyReader: HistoryReader) extends IntermediaryNVHViewActions - case object RegisterState extends IntermediaryNVHViewActions + case object RegisterState extends IntermediaryNVHViewActions } - def props(settings: EncryAppSettings, - ntp: NetworkTimeProvider, - influxRef: Option[ActorRef]): Props = Props(new IntermediaryNVHView(settings, ntp, influxRef)) + final case class ModifierToAppend(modifier: PersistentModifier, isLocallyGenerated: Boolean) + + def props(settings: EncryAppSettings, ntp: NetworkTimeProvider, influxRef: Option[ActorRef]): Props = + Props(new IntermediaryNVHView(settings, ntp, influxRef)) } diff --git a/src/main/scala/encry/nvg/ModifiersCache.scala b/src/main/scala/encry/nvg/ModifiersCache.scala index 9a2df13159..aeb3229040 100644 --- a/src/main/scala/encry/nvg/ModifiersCache.scala +++ b/src/main/scala/encry/nvg/ModifiersCache.scala @@ -1,12 +1,14 @@ package encry.nvg import com.typesafe.scalalogging.StrictLogging -import encry.view.history.History +import encry.settings.EncryAppSettings +import encry.view.history.HistoryReader import encry.view.history.ValidationError.{ FatalValidationError, NonFatalValidationError } import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId + import scala.collection.concurrent.TrieMap import scala.collection.immutable.SortedMap import scala.collection.mutable @@ -15,7 +17,8 @@ object ModifiersCache extends StrictLogging { private type Key = mutable.WrappedArray[Byte] - val cache: TrieMap[Key, PersistentModifier] = TrieMap.empty + val cache: TrieMap[Key, PersistentModifier] = TrieMap.empty + private var headersCollection: SortedMap[Int, List[ModifierId]] = SortedMap.empty[Int, List[ModifierId]] private var isChainSynced = false @@ -28,44 +31,45 @@ object ModifiersCache extends StrictLogging { def contains(key: Key): Boolean = cache.contains(key) - def put(key: Key, value: PersistentModifier, history: History): Unit = if (!contains(key)) { - logger.debug(s"Put ${value.encodedId} of type ${value.modifierTypeId} to cache.") - cache.put(key, value) - value match { - case header: Header => - val possibleHeadersAtCurrentHeight: List[ModifierId] = headersCollection.getOrElse(header.height, List()) - logger.debug( - s"possibleHeadersAtCurrentHeight(${header.height}): ${possibleHeadersAtCurrentHeight.map(Algos.encode).mkString(",")}" - ) - val updatedHeadersAtCurrentHeight: List[ModifierId] = header.id :: possibleHeadersAtCurrentHeight - logger.debug( - s"updatedHeadersAtCurrentHeight(${header.height}): ${updatedHeadersAtCurrentHeight.map(Algos.encode).mkString(",")}" - ) - headersCollection = headersCollection.updated(header.height, updatedHeadersAtCurrentHeight) - case _ => - } + def put(key: Key, value: PersistentModifier, history: HistoryReader, settings: EncryAppSettings): Unit = + if (!contains(key)) { + logger.debug(s"Put ${value.encodedId} of type ${value.modifierTypeId} to cache.") + cache.put(key, value) + value match { + case header: Header => + val possibleHeadersAtCurrentHeight: List[ModifierId] = headersCollection.getOrElse(header.height, List()) + logger.debug( + s"possibleHeadersAtCurrentHeight(${header.height}): ${possibleHeadersAtCurrentHeight.map(Algos.encode).mkString(",")}" + ) + val updatedHeadersAtCurrentHeight: List[ModifierId] = header.id :: possibleHeadersAtCurrentHeight + logger.debug( + s"updatedHeadersAtCurrentHeight(${header.height}): ${updatedHeadersAtCurrentHeight.map(Algos.encode).mkString(",")}" + ) + headersCollection = headersCollection.updated(header.height, updatedHeadersAtCurrentHeight) + case _ => + } - if (size > history.settings.node.modifiersCacheSize) cache.find { - case (_, modifier) => - history.testApplicable(modifier) match { - case Right(_) | Left(_: NonFatalValidationError) => false - case _ => true - } - }.map(mod => remove(mod._1)) - } + if (size > settings.node.modifiersCacheSize) cache.find { + case (_, modifier) => + history.testApplicable(modifier) match { + case Right(_) | Left(_: NonFatalValidationError) => false + case _ => true + } + }.map(mod => remove(mod._1)) + } def remove(key: Key): Option[PersistentModifier] = { logger.debug(s"Going to delete ${Algos.encode(key.toArray)}. Cache contains: ${cache.get(key).isDefined}.") cache.remove(key) } - def popCandidate(history: History): List[PersistentModifier] = synchronized { - findCandidateKey(history).take(1).flatMap(k => remove(k)) + def popCandidate(history: HistoryReader, settings: EncryAppSettings): List[PersistentModifier] = synchronized { + findCandidateKey(history, settings).take(1).flatMap(k => remove(k)) } override def toString: String = cache.keys.map(key => Algos.encode(key.toArray)).mkString(",") - def findCandidateKey(history: History): List[Key] = { + def findCandidateKey(history: HistoryReader, settings: EncryAppSettings): List[Key] = { def isApplicable(key: Key): Boolean = cache @@ -93,7 +97,7 @@ object ModifiersCache extends StrictLogging { List(cache.find { case (k, v) => v match { - case _: Header if history.getBestHeaderId.exists(headerId => headerId sameElements v.parentId) => true + case _: Header if history.getBestHeaderId.exists(_ sameElements v.parentId) => true case _ => val isApplicableMod: Boolean = isApplicable(k) isApplicableMod @@ -108,7 +112,7 @@ object ModifiersCache extends StrictLogging { logger.debug(s"Drop height ${history.getBestHeaderHeight + 1} in HeadersCollection") val res = value.map(cache.get(_)).collect { case Some(v: Header) - if (history.getBestHeaderHeight == history.settings.constants.PreGenesisHeight && + if (history.getBestHeaderHeight == settings.constants.PreGenesisHeight && (v.parentId sameElements Header.GenesisParentId) || history.getHeaderById(v.parentId).nonEmpty) && isApplicable(new mutable.WrappedArray.ofByte(v.id)) => logger.debug(s"Find new bestHeader in cache: ${Algos.encode(v.id)}") @@ -122,9 +126,9 @@ object ModifiersCache extends StrictLogging { logger.debug(s"${headersCollection.get(history.getBestHeaderHeight + 1).map(_.map(Algos.encode))}") logger.debug( s"No header in cache at height ${history.getBestHeaderHeight + 1}. " + - s"Trying to find in range [${history.getBestHeaderHeight - history.settings.constants.MaxRollbackDepth}, ${history.getBestHeaderHeight}]" + s"Trying to find in range [${history.getBestHeaderHeight - settings.constants.MaxRollbackDepth}, ${history.getBestHeaderHeight}]" ) - (history.getBestHeaderHeight - history.settings.constants.MaxRollbackDepth to history.getBestHeaderHeight) + (history.getBestHeaderHeight - settings.constants.MaxRollbackDepth to history.getBestHeaderHeight) .flatMap(height => getHeadersKeysAtHeight(height)) .toList } diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index 3c7c4d3b77..4732244d71 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -1,5 +1,99 @@ package encry.nvg -class NVHHistory { +import java.io.File +import akka.actor.Actor +import cats.syntax.option._ +import com.typesafe.scalalogging.StrictLogging +import encry.consensus.HistoryConsensus.ProgressInfo +import encry.network.Messages.MessageToNetwork.RequestFromLocal +import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.RegisterHistory +import encry.nvg.IntermediaryNVHView.ModifierToAppend +import encry.nvg.NVHHistory.ProgressInfoForState +import encry.nvg.NodeViewHolder.{SemanticallySuccessfulModifier, SyntacticallyFailedModification} +import encry.settings.EncryAppSettings +import encry.stats.StatsSender.{EndOfApplyingModifier, ModifierAppendedToHistory, StartApplyingModifier} +import encry.utils.NetworkTimeProvider +import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError +import encry.view.history.History.AwaitingAppendToHistory +import encry.view.history.{History, HistoryReader} +import org.apache.commons.io.FileUtils +import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.history.{Header, Payload} +import org.encryfoundation.common.utils.Algos +import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} + +class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends Actor with StrictLogging { + + var history: History = initHistory + + context.parent ! RegisterHistory(HistoryReader(history)) + + var historyUpdateInformation: Option[AwaitingAppendToHistory] = None + + override def receive: Receive = { + case ModifierToAppend(mod, isLocallyGenerated) if !history.isModifierDefined(mod.id) => + val startProcessingTime: Long = System.currentTimeMillis() + logger.info(s"Start modifier ${mod.encodedId} of type ${mod.modifierTypeId} processing by history.") + context.parent ! StartApplyingModifier(mod.id, mod.modifierTypeId, startProcessingTime) + history.append(mod) match { + case Left(error) => + logger.info(s"Error ${error.getMessage} has occurred during processing modifier by history component.") + context.parent ! SyntacticallyFailedModification(mod, List(HistoryApplyError(error.getMessage))) + case Right((progressInfo, newUpdateInformation)) => + logger.info( + s"Modifier ${mod.encodedId} of type ${mod.modifierTypeId} processed successfully by history. " + + s"Time of processing is: ${(System.currentTimeMillis() - startProcessingTime) / 1000}s." + ) + context.parent ! ModifierAppendedToHistory(mod match { + case _: Header => true + case _: Payload => false + }, success = true) + if (progressInfo.toApply.nonEmpty) { + logger.info(s"Progress info contains an non empty toApply. Going to notify state about new toApply.") + historyUpdateInformation = newUpdateInformation + context.parent ! ProgressInfoForState(progressInfo) + } else { + logger.info(s"Progress info contains an empty toApply. Going to form request download.") + if (!isLocallyGenerated) requestDownloads(progressInfo, mod.id.some) + context.parent ! SemanticallySuccessfulModifier(mod) + } + } + + case ModifierToAppend(mod, _) => + logger.info(s"Got modifier ${mod.encodedId} on history actor which already contains in history.") + } + + def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = none): Unit = + pi.toDownload.foreach { + case (tid: ModifierTypeId, id: ModifierId) => + logger.info( + s"Node view holder created download request for modifier ${Algos.encode(id)} of type $tid. " + + s"Previous modifier is ${previousModifier.map(Algos.encode)}." + ) + if (tid != Payload.modifierTypeId || (history.isFullChainSynced && tid == Payload.modifierTypeId)) + context.parent ! RequestFromLocal(none, tid, List(id)) + else + logger.info( + s"Ignore sending download request for modifier ${Algos.encode(id)} because full chain is not synced." + ) + } + + def initHistory: History = + try { + val history: History = History.readOrGenerate(settings, ntp) + history.updateIdsForSyncInfo() + logger.info(s"History best block height: ${history.getBestBlockHeight}") + logger.info(s"History best header height: ${history.getBestHeaderHeight}") + history + } catch { + case error: Throwable => + logger.info(s"During history initialization error ${error.getMessage} has happened.") + new File(settings.directory).listFiles.foreach(FileUtils.cleanDirectory) + initHistory + } +} + +object NVHHistory { + final case class ProgressInfoForState(pi: ProgressInfo) extends AnyVal } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 7ecff66a98..c838a35262 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -369,7 +369,8 @@ class NodeViewHolder( logger.debug(s"Can't apply modifier ${modifier.encodedId}, contents: $modifier to history cause $e.") context.parent ! SyntacticallyFailedModification(modifier, List(HistoryApplyError(e.getMessage))) } - } else logger.info(s"Trying to apply modifier ${modifier.encodedId} that's already in history.") + } + else logger.info(s"Trying to apply modifier ${modifier.encodedId} that's already in history.") def sendUpdatedInfoToMemoryPool(toRemove: Seq[PersistentModifier], toApply: Seq[PersistentModifier]): Unit = { val toRemoveTxs: IndexedSeq[Transaction] = toRemove diff --git a/src/main/scala/encry/view/history/FastSyncProcessor.scala b/src/main/scala/encry/view/history/FastSyncProcessor.scala index 6e71c10856..68b09cf212 100644 --- a/src/main/scala/encry/view/history/FastSyncProcessor.scala +++ b/src/main/scala/encry/view/history/FastSyncProcessor.scala @@ -2,12 +2,13 @@ package encry.view.history import cats.syntax.option.none import encry.consensus.HistoryConsensus.ProgressInfo -import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} +import encry.storage.VersionalStorage.{ StorageKey, StorageValue, StorageVersion } +import encry.view.history.History.AwaitingAppendToHistory import org.encryfoundation.common.modifiers.history.Payload trait FastSyncProcessor extends HistoryApi { - def processPayload(payload: Payload): ProgressInfo = { + def processPayload(payload: Payload): (ProgressInfo, Option[AwaitingAppendToHistory]) = { val startTime: Long = System.currentTimeMillis() getBlockByPayload(payload).foreach { block => logger.info(s"processPayloadFastSync") @@ -18,9 +19,11 @@ trait FastSyncProcessor extends HistoryApi { StorageVersion @@ validityKey(block.payload.id).untag(StorageKey), List(block.header.id, block.payload.id).map(id => validityKey(id) -> StorageValue @@ Array(1.toByte)) ) - logger.info(s"Finished processing block ${block.encodedId}. " + - s"Processing time is ${(System.currentTimeMillis() - startTime) / 1000} s") + logger.info( + s"Finished processing block ${block.encodedId}. " + + s"Processing time is ${(System.currentTimeMillis() - startTime) / 1000} s" + ) } - ProgressInfo(none, Seq.empty, Seq.empty, none) + ProgressInfo(none, Seq.empty, Seq.empty, none) -> None } } diff --git a/src/main/scala/encry/view/history/History.scala b/src/main/scala/encry/view/history/History.scala index 6ae85f3887..781e4576b9 100644 --- a/src/main/scala/encry/view/history/History.scala +++ b/src/main/scala/encry/view/history/History.scala @@ -7,39 +7,42 @@ import com.typesafe.scalalogging.StrictLogging import encry.consensus.HistoryConsensus.ProgressInfo import encry.settings._ import encry.storage.VersionalStorage -import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} +import encry.storage.VersionalStorage.{ StorageKey, StorageValue, StorageVersion } import encry.storage.iodb.versionalIODB.IODBHistoryWrapper -import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} +import encry.storage.levelDb.versionalLevelDB.{ LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion } import encry.utils.NetworkTimeProvider +import encry.view.history.History.AwaitingAppendToHistory import encry.view.history.storage.HistoryStorage import io.iohk.iodb.LSMStore import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId import org.iq80.leveldb.Options /** - * History implementation. It is processing persistent modifiers generated locally or received from the network. + * History implementation. It is processing persistent modifiers generated locally or received from the network. **/ trait History extends HistoryModifiersValidator with AutoCloseable { var isFullChainSynced: Boolean /** Appends modifier to the history if it is applicable. */ - def append(modifier: PersistentModifier): Either[Throwable, (History, ProgressInfo)] = { + def append( + modifier: PersistentModifier + ): Either[Throwable, (ProgressInfo, Option[AwaitingAppendToHistory])] = { logger.info(s"Trying to append modifier ${Algos.encode(modifier.id)} of type ${modifier.modifierTypeId} to history") Either.catchNonFatal(modifier match { - case header: Header => + case header: Header => logger.info(s"Append header ${header.encodedId} at height ${header.height} to history") - (this, processHeader(header)) - case payload: Payload => (this, processPayload(payload)) + processHeader(header) + case payload: Payload => processPayload(payload) }) } - def processHeader(h: Header): ProgressInfo + def processHeader(h: Header): (ProgressInfo, Option[AwaitingAppendToHistory]) - def processPayload(payload: Payload): ProgressInfo + def processPayload(payload: Payload): (ProgressInfo, Option[AwaitingAppendToHistory]) /** @return header, that corresponds to modifier */ private def correspondingHeader(modifier: PersistentModifier): Option[Header] = modifier match { @@ -49,22 +52,28 @@ trait History extends HistoryModifiersValidator with AutoCloseable { } /** - * Marks modifier and all modifiers in child chains as invalid - * - * @param modifier that is invalid against the State - * @return ProgressInfo with next modifier to try to apply - */ + * Marks modifier and all modifiers in child chains as invalid + * + * @param modifier that is invalid against the State + * @return ProgressInfo with next modifier to try to apply + */ def reportModifierIsInvalid(modifier: PersistentModifier): (History, ProgressInfo) = { logger.info(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is marked as invalid") correspondingHeader(modifier) match { case Some(invalidatedHeader) => val invalidatedHeaders: Seq[Header] = continuationHeaderChains(invalidatedHeader, _ => true).flatten.distinct val validityRow: List[(StorageKey, StorageValue)] = invalidatedHeaders - .flatMap(h => Seq(h.id, h.payloadId) - .map(id => validityKey(id) -> StorageValue @@ Array(0.toByte))).toList + .flatMap( + h => + Seq(h.id, h.payloadId) + .map(id => validityKey(id) -> StorageValue @@ Array(0.toByte)) + ) + .toList logger.info(s"Going to invalidate ${invalidatedHeader.encodedId} and ${invalidatedHeaders.map(_.encodedId)}") - val bestHeaderIsInvalidated: Boolean = getBestHeaderId.exists(id => invalidatedHeaders.exists(_.id sameElements id)) - val bestFullIsInvalidated: Boolean = getBestBlockId.exists(id => invalidatedHeaders.exists(_.id sameElements id)) + val bestHeaderIsInvalidated: Boolean = + getBestHeaderId.exists(id => invalidatedHeaders.exists(_.id sameElements id)) + val bestFullIsInvalidated: Boolean = + getBestBlockId.exists(id => invalidatedHeaders.exists(_.id sameElements id)) (bestHeaderIsInvalidated, bestFullIsInvalidated) match { case (false, false) => // Modifiers from best header and best full chain are not involved, no rollback and links change required. @@ -86,17 +95,20 @@ trait History extends HistoryModifiersValidator with AutoCloseable { this -> ProgressInfo(None, Seq.empty, Seq.empty, None) } else { val invalidatedChain: Seq[Block] = getBestBlock.toSeq - .flatMap(f => headerChainBack(getBestBlockHeight + 1, f.header, h => !invalidatedHeaders.contains(h)).headers) + .flatMap( + f => headerChainBack(getBestBlockHeight + 1, f.header, h => !invalidatedHeaders.contains(h)).headers + ) .flatMap(h => getBlockByHeader(h)) .ensuring(_.lengthCompare(1) > 0, "invalidatedChain should contain at least bestFullBlock and parent") val branchPoint: Block = invalidatedChain.head val validChain: Seq[Block] = - continuationHeaderChains(branchPoint.header, h => getBlockByHeader(h).isDefined && !invalidatedHeaders.contains(h)) + continuationHeaderChains(branchPoint.header, + h => getBlockByHeader(h).isDefined && !invalidatedHeaders.contains(h)) .maxBy(chain => scoreOf(chain.last.id).getOrElse(BigInt(0))) .flatMap(h => getBlockByHeader(h)) val changedLinks: Seq[(StorageKey, StorageValue)] = List( - BestBlockKey -> StorageValue @@ validChain.last.id.untag(ModifierId), + BestBlockKey -> StorageValue @@ validChain.last.id.untag(ModifierId), BestHeaderKey -> StorageValue @@ newBestHeader.id.untag(ModifierId) ) val toInsert: List[(StorageKey, StorageValue)] = validityRow ++ changedLinks @@ -115,21 +127,22 @@ trait History extends HistoryModifiersValidator with AutoCloseable { } /** - * Marks modifier as valid - * - * @param modifier that is valid against the State - * @return ProgressInfo with next modifier to try to apply - */ + * Marks modifier as valid + * + * @param modifier that is valid against the State + * @return ProgressInfo with next modifier to try to apply + */ def reportModifierIsValid(modifier: PersistentModifier): History = { logger.info(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is marked as valid ") modifier match { case block: Block => val nonMarkedIds: Seq[ModifierId] = Seq(block.header.id, block.payload.id) .filter(id => historyStorage.get(validityKey(id)).isEmpty) - if (nonMarkedIds.nonEmpty) historyStorage.insert( - StorageVersion @@ validityKey(nonMarkedIds.head).untag(StorageKey), - nonMarkedIds.map(id => validityKey(id) -> StorageValue @@ Array(1.toByte)).toList - ) + if (nonMarkedIds.nonEmpty) + historyStorage.insert( + StorageVersion @@ validityKey(nonMarkedIds.head).untag(StorageKey), + nonMarkedIds.map(id => validityKey(id) -> StorageValue @@ Array(1.toByte)).toList + ) this case _ => historyStorage.insert( @@ -147,6 +160,11 @@ trait History extends HistoryModifiersValidator with AutoCloseable { object History extends StrictLogging { + final case class AwaitingAppendToHistory( + toUpdate: Seq[(StorageKey, StorageValue)], + modifier: PersistentModifier + ) + def getHistoryIndexDir(settings: EncryAppSettings): File = { val dir: File = new File(s"${settings.directory}/history/index") dir.mkdirs() @@ -166,8 +184,8 @@ object History extends StrictLogging { case VersionalStorage.IODB => logger.info("Init history with iodb storage") val historyObjectsDir: File = getHistoryObjectsDir(settingsEncry) - val indexStore: LSMStore = new LSMStore(historyIndexDir, keepVersions = 0) - val objectsStore: LSMStore = new LSMStore(historyObjectsDir, keepVersions = 0) + val indexStore: LSMStore = new LSMStore(historyIndexDir, keepVersions = 0) + val objectsStore: LSMStore = new LSMStore(historyObjectsDir, keepVersions = 0) IODBHistoryWrapper(indexStore, objectsStore) case VersionalStorage.LevelDB => logger.info("Init history with levelDB storage") @@ -176,17 +194,16 @@ object History extends StrictLogging { } if (settingsEncry.snapshotSettings.enableFastSynchronization && !settingsEncry.node.offlineGeneration) new History with HistoryHeadersProcessor with FastSyncProcessor { - override val settings: EncryAppSettings = settingsEncry - override var isFullChainSynced: Boolean = settingsEncry.node.offlineGeneration - override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) + override val settings: EncryAppSettings = settingsEncry + override var isFullChainSynced: Boolean = settingsEncry.node.offlineGeneration + override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) override val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsEncry.ntp) - } - else + } else new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { - override val settings: EncryAppSettings = settingsEncry - override var isFullChainSynced: Boolean = settingsEncry.node.offlineGeneration - override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) + override val settings: EncryAppSettings = settingsEncry + override var isFullChainSynced: Boolean = settingsEncry.node.offlineGeneration + override val historyStorage: HistoryStorage = HistoryStorage(vldbInit) override val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsEncry.ntp) } } -} \ No newline at end of file +} diff --git a/src/main/scala/encry/view/history/HistoryHeadersProcessor.scala b/src/main/scala/encry/view/history/HistoryHeadersProcessor.scala index b8e414cce8..f91008b89c 100644 --- a/src/main/scala/encry/view/history/HistoryHeadersProcessor.scala +++ b/src/main/scala/encry/view/history/HistoryHeadersProcessor.scala @@ -6,53 +6,53 @@ import encry.EncryApp.forceStopApplication import encry.consensus.ConsensusSchemeReaders import encry.consensus.HistoryConsensus.ProgressInfo import encry.storage.VersionalStorage.{ StorageKey, StorageValue } +import encry.view.history.History.AwaitingAppendToHistory import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.utils.TaggedTypes.{ Difficulty, ModifierId } trait HistoryHeadersProcessor extends HistoryApi { - def processHeader(h: Header): ProgressInfo = getHeaderInfoUpdate(h) match { - case dataToUpdate: Seq[_] if dataToUpdate.nonEmpty => - historyStorage.bulkInsert(h.id, dataToUpdate, Seq(h)) - getBestHeaderId match { + def processHeader(h: Header): (ProgressInfo, Option[AwaitingAppendToHistory]) = getHeaderInfoUpdate(h) match { + case (bestHeaderId, dataToUpdate: Seq[_]) if dataToUpdate.nonEmpty => + //historyStorage.bulkInsert(h.id, dataToUpdate, Seq(h)) + bestHeaderId match { case Some(bestHeaderId) => - ProgressInfo(none, Seq.empty, if (!bestHeaderId.sameElements(h.id)) Seq.empty else Seq(h), toDownload(h)) + ProgressInfo(none, Seq.empty, if (!bestHeaderId.sameElements(h.id)) Seq.empty else Seq(h), toDownload(h)) -> + Some(AwaitingAppendToHistory(dataToUpdate, h)) case _ => forceStopApplication(errorMessage = "Should always have best header after header application") } - case _ => ProgressInfo(none, Seq.empty, Seq.empty, none) + case _ => ProgressInfo(none, Seq.empty, Seq.empty, none) -> None } - private def getHeaderInfoUpdate(header: Header): Seq[(StorageKey, StorageValue)] = { + private def getHeaderInfoUpdate(header: Header): (Option[ModifierId], Seq[(StorageKey, StorageValue)]) = { addHeaderToCacheIfNecessary(header) if (header.isGenesis) { logger.info(s"Initialize header chain with genesis header ${header.encodedId}") - Seq( - BestHeaderKey -> StorageValue @@ header.id, - heightIdsKey(settings.constants.GenesisHeight) -> StorageValue @@ header.id, - headerHeightKey(header.id) -> StorageValue @@ Ints.toByteArray(settings.constants.GenesisHeight), - headerScoreKey(header.id) -> StorageValue @@ header.difficulty.toByteArray - ) + Some(header.id) -> + Seq( + BestHeaderKey -> StorageValue @@ header.id, + heightIdsKey(settings.constants.GenesisHeight) -> StorageValue @@ header.id, + headerHeightKey(header.id) -> StorageValue @@ Ints.toByteArray(settings.constants.GenesisHeight), + headerScoreKey(header.id) -> StorageValue @@ header.difficulty.toByteArray + ) } else scoreOf(header.parentId).map { parentScore => logger.info(s"getHeaderInfoUpdate for header $header") val score: Difficulty = Difficulty @@ (parentScore + ConsensusSchemeReaders.consensusScheme.realDifficulty(header)) - val bestHeaderHeight: Int = getBestHeaderHeight - val bestHeadersChainScore: BigInt = getBestHeadersChainScore - val bestRow: Seq[(StorageKey, StorageValue)] = - if ((header.height > bestHeaderHeight) || (header.height == bestHeaderHeight && score > bestHeadersChainScore)) - Seq(BestHeaderKey -> StorageValue @@ header.id.untag(ModifierId)) - else Seq.empty + val bestHeaderHeight: Int = getBestHeaderHeight + val bestHeadersChainScore: BigInt = getBestHeadersChainScore val scoreRow: (StorageKey, StorageValue) = headerScoreKey(header.id) -> StorageValue @@ score.toByteArray val heightRow: (StorageKey, StorageValue) = headerHeightKey(header.id) -> StorageValue @@ Ints.toByteArray(header.height) - val headerIdsRow: Seq[(StorageKey, StorageValue)] = - if ((header.height > bestHeaderHeight) || (header.height == bestHeaderHeight && score > bestHeadersChainScore)) - bestBlockHeaderIdsRow(header, score) - else orphanedBlockHeaderIdsRow(header, score) - Seq(scoreRow, heightRow) ++ bestRow ++ headerIdsRow - }.getOrElse(Seq.empty) + if ((header.height > bestHeaderHeight) || (header.height == bestHeaderHeight && score > bestHeadersChainScore)) + Some(header.id) -> + (Seq(scoreRow, heightRow) ++ + Seq(BestHeaderKey -> StorageValue @@ header.id.untag(ModifierId)) ++ + bestBlockHeaderIdsRow(header, score)) + else getBestHeaderId -> (Seq(scoreRow, heightRow) ++ orphanedBlockHeaderIdsRow(header, score)) + }.getOrElse(None -> Seq.empty) } private def bestBlockHeaderIdsRow(h: Header, score: Difficulty): Seq[(StorageKey, StorageValue)] = { diff --git a/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala b/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala index 9732bd6af6..130a966a88 100644 --- a/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala +++ b/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala @@ -4,20 +4,25 @@ import cats.syntax.either._ import cats.syntax.option._ import encry.consensus.HistoryConsensus.ProgressInfo import encry.modifiers.history.HeaderChain +import encry.storage.VersionalStorage.{ StorageKey, StorageValue } +import encry.view.history.History.AwaitingAppendToHistory import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{Height, ModifierId} +import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } trait HistoryPayloadsProcessor extends HistoryApi { - def processPayload(payload: Payload): ProgressInfo = + def processPayload(payload: Payload): (ProgressInfo, Option[AwaitingAppendToHistory]) = getBlockByPayload(payload).flatMap { block => logger.info(s"proc block ${block.header.encodedId}!") processBlock(block).some - }.getOrElse(putToHistory(payload)) + }.getOrElse { + //putToHistory(payload) + ProgressInfo(none, Seq.empty, Seq.empty, none) -> Some(AwaitingAppendToHistory(Seq.empty, payload)) + } - private def processBlock(blockToProcess: Block): ProgressInfo = { + private def processBlock(blockToProcess: Block): (ProgressInfo, Option[AwaitingAppendToHistory]) = { logger.info( s"Starting processing block to history ||${blockToProcess.encodedId}||${blockToProcess.header.height}||" ) @@ -29,26 +34,31 @@ trait HistoryPayloadsProcessor extends HistoryApi { case Some(header) if isBestBlockDefined && isBetterChain(header.id) => processBetterChain(blockToProcess, header, Seq.empty, settings.node.blocksToKeep) case Some(header) => - logger.info(s"\n\nnonBestBlock. id: ${blockToProcess.header.encodedId}. cause: ${isBestBlockDefined} or ${ isBetterChain(header.id)}\n\n") + logger.info( + s"\n\nnonBestBlock. id: ${blockToProcess.header.encodedId}. cause: ${isBestBlockDefined} or ${isBetterChain(header.id)}\n\n" + ) nonBestBlock(blockToProcess) case None => logger.debug(s"Best full chain is empty. Returning empty progress info") - ProgressInfo(none, Seq.empty, Seq.empty, none) + ProgressInfo(none, Seq.empty, Seq.empty, none) -> None } } - private def processValidFirstBlock(fullBlock: Block, - newBestHeader: Header, - newBestChain: Seq[Block]): ProgressInfo = { + private def processValidFirstBlock( + fullBlock: Block, + newBestHeader: Header, + newBestChain: Seq[Block] + ): (ProgressInfo, Option[AwaitingAppendToHistory]) = { logger.info(s"Appending ${fullBlock.encodedId} as a valid first block with height ${fullBlock.header.height}") - updateStorage(fullBlock.payload, newBestHeader.id) - ProgressInfo(none, Seq.empty, newBestChain, none) + ProgressInfo(none, Seq.empty, newBestChain, none) -> Some(updateStorage(fullBlock.payload, newBestHeader.id)) } - private def processBetterChain(fullBlock: Block, - newBestHeader: Header, - newBestChain: Seq[Block], - blocksToKeep: Int): ProgressInfo = + private def processBetterChain( + fullBlock: Block, + newBestHeader: Header, + newBestChain: Seq[Block], + blocksToKeep: Int + ): (ProgressInfo, Option[AwaitingAppendToHistory]) = getHeaderOfBestBlock.map { header => val (prevChain: HeaderChain, newChain: HeaderChain) = commonBlockThenSuffixes(header, newBestHeader) val toRemove: Seq[Block] = prevChain.tail.headers @@ -59,8 +69,7 @@ trait HistoryPayloadsProcessor extends HistoryApi { if (toApply.lengthCompare(newChain.length - 1) != 0) { logger.info(s"To apply. processBetterChain. nonBestBlock.") nonBestBlock(fullBlock) - } - else { + } else { //application of this block leads to full chain with higher score logger.info(s"Appending ${fullBlock.encodedId}|${fullBlock.header.height} as a better chain") val branchPoint: Option[ModifierId] = toRemove.headOption.map(_ => prevChain.head.id) @@ -74,22 +83,23 @@ trait HistoryPayloadsProcessor extends HistoryApi { ) val updatedHeadersAtHeightIds = newChain.headers.map(header => updatedBestHeaderAtHeightRaw(header.id, Height @@ header.height)).toList - updateStorage(fullBlock.payload, newBestHeader.id, updateBestHeader, updatedHeadersAtHeightIds) + val toUpdateInfo: AwaitingAppendToHistory = + updateStorage(fullBlock.payload, newBestHeader.id, updateBestHeader, updatedHeadersAtHeightIds) if (blocksToKeep >= 0) { val lastKept: Int = blockDownloadProcessor.updateBestBlock(fullBlock.header) val bestHeight: Int = toApply.lastOption.map(_.header.height).getOrElse(0) val diff: Int = bestHeight - header.height clipBlockDataAt(((lastKept - diff) until lastKept).filter(_ >= 0)) } - ProgressInfo(branchPoint, toRemove, toApply, none) + ProgressInfo(branchPoint, toRemove, toApply, none) -> Some(toUpdateInfo) } - }.getOrElse(ProgressInfo(none, Seq.empty, Seq.empty, none)) + }.getOrElse(ProgressInfo(none, Seq.empty, Seq.empty, none) -> None) - private def nonBestBlock(fullBlock: Block): ProgressInfo = { + private def nonBestBlock(fullBlock: Block): (ProgressInfo, Option[AwaitingAppendToHistory]) = { //Orphaned block or full chain is not initialized yet logger.info(s"Process block to history ${fullBlock.encodedId}||${fullBlock.header.height}||") - historyStorage.bulkInsert(fullBlock.payload.id, Seq.empty, Seq(fullBlock.payload)) - ProgressInfo(none, Seq.empty, Seq.empty, none) + //historyStorage.bulkInsert(fullBlock.payload.id, Seq.empty, Seq(fullBlock.payload)) + ProgressInfo(none, Seq.empty, Seq.empty, none) -> Some(AwaitingAppendToHistory(Seq.empty, fullBlock.payload)) } private def updatedBestHeaderAtHeightRaw(headerId: ModifierId, height: Height): (Array[Byte], Array[Byte]) = @@ -97,10 +107,9 @@ trait HistoryPayloadsProcessor extends HistoryApi { (Seq(headerId) ++ headerIdsAtHeight(height).filterNot(_ sameElements headerId)).flatten.toArray - private def putToHistory(payload: Payload): ProgressInfo = { - historyStorage.insertObjects(Seq(payload)) + private def putToHistory(payload: Payload): ProgressInfo = + //historyStorage.insertObjects(Seq(payload)) ProgressInfo(none, Seq.empty, Seq.empty, none) - } private def isBetterChain(id: ModifierId): Boolean = (for { @@ -110,18 +119,19 @@ trait HistoryPayloadsProcessor extends HistoryApi { score <- scoreOf(id) bestBlockHeight = getBestBlockHeight } yield { - logger.info(s"isBetterChain. id: ${Algos.encode(id)}. \n " + - s"bestBlockHeight: $bestBlockHeight.\n " + - s"heightOfThisHeader $heightOfThisHeader.\n " + - s"score: $score.\n " + - s"prevBestScore: $prevBestScore.\n " + - s"res is: ${(bestBlockHeight < heightOfThisHeader) || (bestBlockHeight == heightOfThisHeader && score > prevBestScore)}") + logger.info( + s"isBetterChain. id: ${Algos.encode(id)}. \n " + + s"bestBlockHeight: $bestBlockHeight.\n " + + s"heightOfThisHeader $heightOfThisHeader.\n " + + s"score: $score.\n " + + s"prevBestScore: $prevBestScore.\n " + + s"res is: ${(bestBlockHeight < heightOfThisHeader) || (bestBlockHeight == heightOfThisHeader && score > prevBestScore)}" + ) (bestBlockHeight < heightOfThisHeader) || (bestBlockHeight == heightOfThisHeader && score > prevBestScore) - }) - .getOrElse { - logger.info(s"isBetterChain. id: ${Algos.encode(id)}. getOrElse. false.") - false - } + }).getOrElse { + logger.info(s"isBetterChain. id: ${Algos.encode(id)}. getOrElse. false.") + false + } private def calculateBestFullChain(block: Block): Seq[Block] = { val continuations: Seq[Seq[Header]] = continuationHeaderChains(block.header, h => isBlockDefined(h)).map(_.tail) @@ -139,14 +149,20 @@ trait HistoryPayloadsProcessor extends HistoryApi { historyStorage.removeObjects(toRemove) } - private def updateStorage(newModRow: PersistentModifier, - bestFullHeaderId: ModifierId, - updateHeaderInfo: Boolean = false, - additionalIndexes: List[(Array[Byte], Array[Byte])] = List.empty): Unit = { + private def updateStorage( + newModRow: PersistentModifier, + bestFullHeaderId: ModifierId, + updateHeaderInfo: Boolean = false, + additionalIndexes: List[(Array[Byte], Array[Byte])] = List.empty + ): AwaitingAppendToHistory = { val indicesToInsert: Seq[(Array[Byte], Array[Byte])] = if (updateHeaderInfo) Seq(BestBlockKey -> bestFullHeaderId, BestHeaderKey -> bestFullHeaderId) else Seq(BestBlockKey -> bestFullHeaderId) - historyStorage.bulkInsert(newModRow.id, indicesToInsert ++ additionalIndexes, Seq(newModRow)) + AwaitingAppendToHistory((indicesToInsert ++ additionalIndexes).map { + case (bytes, bytes1) => + StorageKey @@ bytes -> StorageValue @@ bytes1 + }, newModRow) + //historyStorage.bulkInsert(newModRow.id, indicesToInsert ++ additionalIndexes, Seq(newModRow)) } private def isValidFirstBlock(header: Header): Boolean = diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 238e2f069e..7ae699175f 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -2,14 +2,17 @@ package encry.view.history import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Older } import encry.modifiers.history.HeaderChain +import encry.view.history.ValidationError.HistoryApiError +import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Block, Header } import org.encryfoundation.common.network.SyncInfo import org.encryfoundation.common.utils.TaggedTypes.ModifierId - import scala.collection.immutable.HashSet trait HistoryReader { + def getBestHeaderId: Option[ModifierId] + def getBestHeaderHeight: Int def getBestBlockHeight: Int @@ -22,8 +25,7 @@ trait HistoryReader { def getHeaderById(id: ModifierId): Option[Header] - def getChainToHeader(fromHeaderOpt: Option[Header], - toHeader: Header): (Option[ModifierId], HeaderChain) + def getChainToHeader(fromHeaderOpt: Option[Header], toHeader: Header): (Option[ModifierId], HeaderChain) def getBlockByHeaderId(id: ModifierId): Option[Block] @@ -50,6 +52,8 @@ trait HistoryReader { def getBestHeader: Option[Header] def getBestBlock: Option[Block] + + def testApplicable(modifier: PersistentModifier): Either[ValidationError, PersistentModifier] } object HistoryReader { @@ -72,8 +76,14 @@ object HistoryReader { def lastHeaders(count: Int): HeaderChain = HeaderChain.empty def getBestHeader: Option[Header] = None def getBestBlock: Option[Block] = None - def getChainToHeader(fromHeaderOpt: Option[Header], - toHeader: Header): (Option[ModifierId], HeaderChain) = (None, HeaderChain.empty) + def getChainToHeader( + fromHeaderOpt: Option[Header], + toHeader: Header + ): (Option[ModifierId], HeaderChain) = + (None, HeaderChain.empty) + def testApplicable(modifier: PersistentModifier): Either[ValidationError, PersistentModifier] = + Left(HistoryApiError("")) + def getBestHeaderId: Option[ModifierId] = None } def apply(history: History): HistoryReader = new HistoryReader { @@ -95,8 +105,13 @@ object HistoryReader { def lastHeaders(count: Int): HeaderChain = history.lastHeaders(count) def getBestHeader: Option[Header] = history.getBestHeader def getBestBlock: Option[Block] = history.getBestBlock - def getChainToHeader(fromHeaderOpt: Option[Header], - toHeader: Header): (Option[ModifierId], HeaderChain) = history.getChainToHeader(fromHeaderOpt, - toHeader) + def getChainToHeader( + fromHeaderOpt: Option[Header], + toHeader: Header + ): (Option[ModifierId], HeaderChain) = + history.getChainToHeader(fromHeaderOpt, toHeader) + def testApplicable(modifier: PersistentModifier): Either[ValidationError, PersistentModifier] = + history.testApplicable(modifier) + def getBestHeaderId: Option[ModifierId] = history.getBestHeaderId } } diff --git a/src/test/scala/encry/utils/EncryGenerator.scala b/src/test/scala/encry/utils/EncryGenerator.scala index 87e1bf442a..a731392410 100644 --- a/src/test/scala/encry/utils/EncryGenerator.scala +++ b/src/test/scala/encry/utils/EncryGenerator.scala @@ -21,6 +21,8 @@ import scala.util.{Random => ScRand} trait EncryGenerator extends Settings { + Box + val mnemonicKey: String = "index another island accuse valid aerobic little absurd bunker keep insect scissors" val privKey: PrivateKey25519 = createPrivKey(Some(mnemonicKey)) diff --git a/src/test/scala/encry/view/QWE.scala b/src/test/scala/encry/view/QWE.scala new file mode 100644 index 0000000000..7baca8edeb --- /dev/null +++ b/src/test/scala/encry/view/QWE.scala @@ -0,0 +1,98 @@ +package encry.view + +import com.typesafe.scalalogging.StrictLogging +import encry.modifiers.InstanceFactory +import encry.settings.TestNetSettings +import encry.view.history.History +import org.encryfoundation.common.modifiers.history.Block +import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } + +class QWE + extends WordSpecLike + with Matchers + with InstanceFactory + with BeforeAndAfterAll + with OneInstancePerTest + with TestNetSettings + with StrictLogging { + + "qwer" should { + "qr" in { + val (history1_10, history2_10, _) = (0 until 10).foldLeft( + generateDummyHistory(testNetSettings), + generateDummyHistory(testNetSettings), + List.empty[Block] + ) { + case ((prevHistory1, prevHistory2, blocks: List[Block]), _) => + val block: Block = generateNextBlock(prevHistory1) + val a = prevHistory1 + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + val b = prevHistory2 + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) + (a, b, (block +: blocks)) + } + logger.info(s"\n\n\n\nStart processing 1 fork blocks\n\n\n\n\n") + val (history3_15norm, blocksNorm15) = (0 until 5).foldLeft(history1_10, List.empty[Block]) { + case ((prevHistory, blocks: List[Block]), _) => + val block: Block = generateNextBlock(prevHistory) + prevHistory + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) -> (block +: blocks) + } + logger.info(s"\n\n\n\nStart processing 2 blocks\n\n\n\n\n") + val (h4_20, blocks4_fork) = (0 until 10).foldLeft(history2_10, List.empty[Block]) { + case ((prevHistory, blocks: List[Block]), _) => + val block: Block = generateNextBlock(prevHistory) + prevHistory + .append(block.header) + .right + .get + ._1 + .append(block.payload) + .right + .get + ._1 + .reportModifierIsValid(block) -> (block +: blocks) + } + + var tmpH = history3_15norm + logger.info(s"\n\n\n\nApplying fork to normal\n\n\n\n\n") + blocks4_fork.reverse.foreach { nextBlock => + val a = tmpH.append(nextBlock.header) + logger.info(s"after forkapp header: ${a}") + tmpH = a.right.get._1 + } + + blocks4_fork.reverse.foreach { nextBlock => + val a = tmpH.append(nextBlock.payload) + logger.info(s"after forkapp payload: ${a}") + tmpH = a.right.get._1 + logger.info(s"tmpH.getBestHeader -> ${tmpH.getBestHeader}") + logger.info(s"tmpH.getBestBlock -> ${tmpH.getBestBlock}") + } + } + } +} From 4ca159b82b60bc31161f9f41a1d415c8c6c1f78c Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 23 Mar 2020 14:22:13 +0300 Subject: [PATCH 132/177] improved history holder --- .../scala/encry/nvg/IntermediaryNVHView.scala | 40 +++++---- src/main/scala/encry/nvg/NVHHistory.scala | 84 +++++++++++++------ src/main/scala/encry/nvg/NVHState.scala | 8 +- src/main/scala/encry/nvg/NodeViewHolder.scala | 8 +- .../view/history/FastSyncProcessor.scala | 4 +- .../scala/encry/view/history/History.scala | 20 +++-- .../history/HistoryHeadersProcessor.scala | 7 +- .../history/HistoryPayloadsProcessor.scala | 39 ++++----- 8 files changed, 125 insertions(+), 85 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index 2bae973585..1df19c4e67 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -1,11 +1,11 @@ package encry.nvg -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import com.typesafe.scalalogging.StrictLogging -import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{RegisterHistory, RegisterState} +import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{ RegisterHistory, RegisterState } import encry.nvg.IntermediaryNVHView.ModifierToAppend import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NVHHistory.ProgressInfoForState +import encry.nvg.NVHHistory.{ ModifierAppliedToHistory, ProgressInfoForState } import encry.nvg.NVHState.StateAction import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider @@ -30,6 +30,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, def awaitingViewActors(history: Option[ActorRef] = None, state: Option[ActorRef] = None): Receive = { case RegisterHistory(reader) if state.isEmpty => + historyReader = reader logger.info(s"NodeViewParent actor got init history. Going to init state actor.") context.become(awaitingViewActors(Some(sender()), state), discardOld = true) context.actorOf( @@ -62,26 +63,29 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, ) else ModifiersCache.put(wrappedKey, modifier, historyReader, settings) - if (!isProcessingModifierByHistory && !isProcessingModifierByState) { - isProcessingModifierByHistory = true - ModifiersCache - .popCandidate(historyReader, settings) - .foreach { mod: PersistentModifier => - logger.info(s"Got new modifiers in compute application function: ${mod.encodedId}.") - historyRef ! ModifierToAppend(mod, isLocallyGenerated = false) - } - } + if (!isProcessingModifierByHistory && !isProcessingModifierByState) + getNextModifier() + + case ModifierAppliedToHistory => + isProcessingModifierByHistory = false + getNextModifier() - case ProgressInfoForState(pi) => - //todo work with state starts here - case StateAction.ApplyFailed(modId, errs) => - // todo: Notify history - case StateAction.ModifierApplied(modId) => - //todo: Notify history + case ProgressInfoForState(pi, flag, isFullChainSynced) => //todo work with state starts here + case msg: StateAction.ApplyFailed => historyRef ! msg + case msg: StateAction.ModifierApplied => historyRef ! msg } def awaitingHistoryBranchPoint(history: ActorRef): Receive = ??? + def getNextModifier(): Unit = + ModifiersCache + .popCandidate(historyReader, settings) + .foreach { mod: PersistentModifier => + isProcessingModifierByHistory = true + logger.info(s"Got new modifiers in compute application function: ${mod.encodedId}.") + historyRef ! ModifierToAppend(mod, isLocallyGenerated = false) + } + } object IntermediaryNVHView { diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index 4732244d71..b8d0e8da54 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -9,38 +9,45 @@ import encry.consensus.HistoryConsensus.ProgressInfo import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.RegisterHistory import encry.nvg.IntermediaryNVHView.ModifierToAppend -import encry.nvg.NVHHistory.ProgressInfoForState -import encry.nvg.NodeViewHolder.{SemanticallySuccessfulModifier, SyntacticallyFailedModification} +import encry.nvg.NVHHistory.{ ModifierAppliedToHistory, ProgressInfoForState } +import encry.nvg.NVHState.StateAction +import encry.nvg.NodeViewHolder.{ + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification +} import encry.settings.EncryAppSettings -import encry.stats.StatsSender.{EndOfApplyingModifier, ModifierAppendedToHistory, StartApplyingModifier} +import encry.stats.StatsSender.{ ModifierAppendedToHistory, StartApplyingModifier } import encry.utils.NetworkTimeProvider import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError -import encry.view.history.History.AwaitingAppendToHistory -import encry.view.history.{History, HistoryReader} +import encry.view.history.History.HistoryUpdateInfoAcc +import encry.view.history.{ History, HistoryReader } import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Header, Payload } import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends Actor with StrictLogging { - var history: History = initHistory + var history: History = initializeHistory context.parent ! RegisterHistory(HistoryReader(history)) - var historyUpdateInformation: Option[AwaitingAppendToHistory] = None - override def receive: Receive = { case ModifierToAppend(mod, isLocallyGenerated) if !history.isModifierDefined(mod.id) => val startProcessingTime: Long = System.currentTimeMillis() logger.info(s"Start modifier ${mod.encodedId} of type ${mod.modifierTypeId} processing by history.") context.parent ! StartApplyingModifier(mod.id, mod.modifierTypeId, startProcessingTime) history.append(mod) match { - case Left(error) => - logger.info(s"Error ${error.getMessage} has occurred during processing modifier by history component.") + case Left(error: Throwable) => + logger.info( + s"Error ${error.getMessage} has occurred during processing modifier by history component. " + + s"Time of processing is: ${(System.currentTimeMillis() - startProcessingTime) / 1000}s." + ) context.parent ! SyntacticallyFailedModification(mod, List(HistoryApplyError(error.getMessage))) - case Right((progressInfo, newUpdateInformation)) => + context.parent ! ModifierAppliedToHistory + case Right((progressInfo: ProgressInfo, newUpdateInformation: Option[HistoryUpdateInfoAcc])) => logger.info( s"Modifier ${mod.encodedId} of type ${mod.modifierTypeId} processed successfully by history. " + s"Time of processing is: ${(System.currentTimeMillis() - startProcessingTime) / 1000}s." @@ -51,8 +58,16 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends A }, success = true) if (progressInfo.toApply.nonEmpty) { logger.info(s"Progress info contains an non empty toApply. Going to notify state about new toApply.") - historyUpdateInformation = newUpdateInformation - context.parent ! ProgressInfoForState(progressInfo) + context.parent ! ProgressInfoForState( + progressInfo, + (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2, + history.isFullChainSynced + ) + history.insertUpdateInfo(newUpdateInformation) + if (!isLocallyGenerated) progressInfo.toApply.foreach { + case header: Header => requestDownloads(progressInfo, header.id.some) + case _ => requestDownloads(progressInfo, none) + } } else { logger.info(s"Progress info contains an empty toApply. Going to form request download.") if (!isLocallyGenerated) requestDownloads(progressInfo, mod.id.some) @@ -60,26 +75,43 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends A } } - case ModifierToAppend(mod, _) => + case ModifierToAppend(mod, _) => + context.parent ! ModifierAppliedToHistory logger.info(s"Got modifier ${mod.encodedId} on history actor which already contains in history.") + + case StateAction.ModifierApplied(mod: PersistentModifier) => + history = history.reportModifierIsValid(mod) + context.parent ! HistoryReader(history) + context.parent ! ModifierAppliedToHistory + + case StateAction.ApplyFailed(mod, e) => + val (newHistory: History, progressInfo: ProgressInfo) = history.reportModifierIsInvalid(mod) + context.parent ! SemanticallyFailedModification(mod, e) + context.parent ! ProgressInfoForState( + progressInfo, + (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2, + history.isFullChainSynced + ) + history = newHistory + } def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = none): Unit = pi.toDownload.foreach { case (tid: ModifierTypeId, id: ModifierId) => - logger.info( - s"Node view holder created download request for modifier ${Algos.encode(id)} of type $tid. " + - s"Previous modifier is ${previousModifier.map(Algos.encode)}." - ) - if (tid != Payload.modifierTypeId || (history.isFullChainSynced && tid == Payload.modifierTypeId)) + if (tid != Payload.modifierTypeId || (history.isFullChainSynced && tid == Payload.modifierTypeId)) { + logger.info( + s"History holder created download request for modifier ${Algos.encode(id)} of type $tid. " + + s"Previous modifier is ${previousModifier.map(Algos.encode)}." + ) context.parent ! RequestFromLocal(none, tid, List(id)) - else + } else logger.info( s"Ignore sending download request for modifier ${Algos.encode(id)} because full chain is not synced." ) } - def initHistory: History = + def initializeHistory: History = try { val history: History = History.readOrGenerate(settings, ntp) history.updateIdsForSyncInfo() @@ -90,10 +122,12 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends A case error: Throwable => logger.info(s"During history initialization error ${error.getMessage} has happened.") new File(settings.directory).listFiles.foreach(FileUtils.cleanDirectory) - initHistory + initializeHistory } } object NVHHistory { - final case class ProgressInfoForState(pi: ProgressInfo) extends AnyVal + final case class ProgressInfoForState(pi: ProgressInfo, saveRootNodeFlag: Boolean, isFullChainSynced: Boolean) + case object ModifierAppliedToHistory + final case object InsertNewUpdates } diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index c9e2c7be64..3e1c193f68 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -52,10 +52,10 @@ class NVHState(influxRef: Option[ActorRef], var state: UtxoState, settings: Encr } state = stateAfterApply logger.info(s"Successfully apply modifier: ${Algos.encode(modifier.id)} of type ${modifier.modifierTypeId}") - context.parent ! ModifierApplied(modifier.id) + context.parent ! ModifierApplied(modifier) case Left(e: List[ModifierApplyError]) => logger.info(s"Application to state failed cause $e") - context.parent ! ApplyFailed(modifier.id, e) + context.parent ! ApplyFailed(modifier, e) } case CreateTreeChunks => context.parent ! AvlTree.getChunks( @@ -79,9 +79,9 @@ object NVHState extends StrictLogging { sealed trait StateAction object StateAction { - case class ModifierApplied(modifierId: ModifierId) extends StateAction + case class ModifierApplied(modifierId: PersistentModifier) extends StateAction case class Rollback(branchPoint: ModifierId) extends StateAction - case class ApplyFailed(modifierId: ModifierId, errs: List[ModifierApplyError]) extends StateAction + case class ApplyFailed(modifierId: PersistentModifier, errs: List[ModifierApplyError]) extends StateAction case class ApplyModifier(modifier: PersistentModifier, saveRootNodesFlag: Boolean, isFullChainSynced: Boolean) extends StateAction diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index c838a35262..9e96d00e6a 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -207,10 +207,10 @@ class NodeViewHolder( isLocallyGenerated: Boolean = false ): (History, UtxoState, Seq[PersistentModifier]) = { logger.info(s"Starting updating state in updateState function!") - if (!isLocallyGenerated) progressInfo.toApply.foreach { - case header: Header => requestDownloads(progressInfo, header.id.some) - case _ => requestDownloads(progressInfo, none) - } +// if (!isLocallyGenerated) progressInfo.toApply.foreach { +// case header: Header => requestDownloads(progressInfo, header.id.some) +// case _ => requestDownloads(progressInfo, none) +// } val branchingPointOpt: Option[VersionTag] = progressInfo.branchPoint.map(VersionTag !@@ _) val (stateToApplyTry: Try[UtxoState], suffixTrimmed: IndexedSeq[PersistentModifier] @unchecked) = if (progressInfo.chainSwitchingNeeded) { diff --git a/src/main/scala/encry/view/history/FastSyncProcessor.scala b/src/main/scala/encry/view/history/FastSyncProcessor.scala index 68b09cf212..dfb265812d 100644 --- a/src/main/scala/encry/view/history/FastSyncProcessor.scala +++ b/src/main/scala/encry/view/history/FastSyncProcessor.scala @@ -3,12 +3,12 @@ package encry.view.history import cats.syntax.option.none import encry.consensus.HistoryConsensus.ProgressInfo import encry.storage.VersionalStorage.{ StorageKey, StorageValue, StorageVersion } -import encry.view.history.History.AwaitingAppendToHistory +import encry.view.history.History.HistoryUpdateInfoAcc import org.encryfoundation.common.modifiers.history.Payload trait FastSyncProcessor extends HistoryApi { - def processPayload(payload: Payload): (ProgressInfo, Option[AwaitingAppendToHistory]) = { + def processPayload(payload: Payload): (ProgressInfo, Option[HistoryUpdateInfoAcc]) = { val startTime: Long = System.currentTimeMillis() getBlockByPayload(payload).foreach { block => logger.info(s"processPayloadFastSync") diff --git a/src/main/scala/encry/view/history/History.scala b/src/main/scala/encry/view/history/History.scala index 781e4576b9..57fafb1eec 100644 --- a/src/main/scala/encry/view/history/History.scala +++ b/src/main/scala/encry/view/history/History.scala @@ -11,7 +11,7 @@ import encry.storage.VersionalStorage.{ StorageKey, StorageValue, StorageVersion import encry.storage.iodb.versionalIODB.IODBHistoryWrapper import encry.storage.levelDb.versionalLevelDB.{ LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion } import encry.utils.NetworkTimeProvider -import encry.view.history.History.AwaitingAppendToHistory +import encry.view.history.History.HistoryUpdateInfoAcc import encry.view.history.storage.HistoryStorage import io.iohk.iodb.LSMStore import org.encryfoundation.common.modifiers.PersistentModifier @@ -30,7 +30,7 @@ trait History extends HistoryModifiersValidator with AutoCloseable { /** Appends modifier to the history if it is applicable. */ def append( modifier: PersistentModifier - ): Either[Throwable, (ProgressInfo, Option[AwaitingAppendToHistory])] = { + ): Either[Throwable, (ProgressInfo, Option[HistoryUpdateInfoAcc])] = { logger.info(s"Trying to append modifier ${Algos.encode(modifier.id)} of type ${modifier.modifierTypeId} to history") Either.catchNonFatal(modifier match { case header: Header => @@ -40,9 +40,16 @@ trait History extends HistoryModifiersValidator with AutoCloseable { }) } - def processHeader(h: Header): (ProgressInfo, Option[AwaitingAppendToHistory]) + def insertUpdateInfo(updateInfo: Option[HistoryUpdateInfoAcc]): Unit = + updateInfo.foreach { info: HistoryUpdateInfoAcc => + logger.info(s"Going to update history in insertUpdateInfo function.") + if (info.insertToObjectStore) historyStorage.insertObjects(Seq(info.modifier)) + else historyStorage.bulkInsert(info.modifier.id, info.toUpdate, Seq(info.modifier)) + } + + def processHeader(h: Header): (ProgressInfo, Option[HistoryUpdateInfoAcc]) - def processPayload(payload: Payload): (ProgressInfo, Option[AwaitingAppendToHistory]) + def processPayload(payload: Payload): (ProgressInfo, Option[HistoryUpdateInfoAcc]) /** @return header, that corresponds to modifier */ private def correspondingHeader(modifier: PersistentModifier): Option[Header] = modifier match { @@ -160,9 +167,10 @@ trait History extends HistoryModifiersValidator with AutoCloseable { object History extends StrictLogging { - final case class AwaitingAppendToHistory( + final case class HistoryUpdateInfoAcc( toUpdate: Seq[(StorageKey, StorageValue)], - modifier: PersistentModifier + modifier: PersistentModifier, + insertToObjectStore: Boolean ) def getHistoryIndexDir(settings: EncryAppSettings): File = { diff --git a/src/main/scala/encry/view/history/HistoryHeadersProcessor.scala b/src/main/scala/encry/view/history/HistoryHeadersProcessor.scala index f91008b89c..7a4caf1bfe 100644 --- a/src/main/scala/encry/view/history/HistoryHeadersProcessor.scala +++ b/src/main/scala/encry/view/history/HistoryHeadersProcessor.scala @@ -6,19 +6,18 @@ import encry.EncryApp.forceStopApplication import encry.consensus.ConsensusSchemeReaders import encry.consensus.HistoryConsensus.ProgressInfo import encry.storage.VersionalStorage.{ StorageKey, StorageValue } -import encry.view.history.History.AwaitingAppendToHistory +import encry.view.history.History.HistoryUpdateInfoAcc import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.utils.TaggedTypes.{ Difficulty, ModifierId } trait HistoryHeadersProcessor extends HistoryApi { - def processHeader(h: Header): (ProgressInfo, Option[AwaitingAppendToHistory]) = getHeaderInfoUpdate(h) match { + def processHeader(h: Header): (ProgressInfo, Option[HistoryUpdateInfoAcc]) = getHeaderInfoUpdate(h) match { case (bestHeaderId, dataToUpdate: Seq[_]) if dataToUpdate.nonEmpty => - //historyStorage.bulkInsert(h.id, dataToUpdate, Seq(h)) bestHeaderId match { case Some(bestHeaderId) => ProgressInfo(none, Seq.empty, if (!bestHeaderId.sameElements(h.id)) Seq.empty else Seq(h), toDownload(h)) -> - Some(AwaitingAppendToHistory(dataToUpdate, h)) + Some(HistoryUpdateInfoAcc(dataToUpdate, h, insertToObjectStore = false)) case _ => forceStopApplication(errorMessage = "Should always have best header after header application") } diff --git a/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala b/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala index 130a966a88..b1a824efde 100644 --- a/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala +++ b/src/main/scala/encry/view/history/HistoryPayloadsProcessor.scala @@ -5,7 +5,7 @@ import cats.syntax.option._ import encry.consensus.HistoryConsensus.ProgressInfo import encry.modifiers.history.HeaderChain import encry.storage.VersionalStorage.{ StorageKey, StorageValue } -import encry.view.history.History.AwaitingAppendToHistory +import encry.view.history.History.HistoryUpdateInfoAcc import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.utils.Algos @@ -13,16 +13,16 @@ import org.encryfoundation.common.utils.TaggedTypes.{ Height, ModifierId } trait HistoryPayloadsProcessor extends HistoryApi { - def processPayload(payload: Payload): (ProgressInfo, Option[AwaitingAppendToHistory]) = + def processPayload(payload: Payload): (ProgressInfo, Option[HistoryUpdateInfoAcc]) = getBlockByPayload(payload).flatMap { block => logger.info(s"proc block ${block.header.encodedId}!") processBlock(block).some - }.getOrElse { - //putToHistory(payload) - ProgressInfo(none, Seq.empty, Seq.empty, none) -> Some(AwaitingAppendToHistory(Seq.empty, payload)) - } + }.getOrElse( + ProgressInfo(none, Seq.empty, Seq.empty, none) -> + HistoryUpdateInfoAcc(Seq.empty, payload, insertToObjectStore = true).some + ) - private def processBlock(blockToProcess: Block): (ProgressInfo, Option[AwaitingAppendToHistory]) = { + private def processBlock(blockToProcess: Block): (ProgressInfo, Option[HistoryUpdateInfoAcc]) = { logger.info( s"Starting processing block to history ||${blockToProcess.encodedId}||${blockToProcess.header.height}||" ) @@ -40,7 +40,7 @@ trait HistoryPayloadsProcessor extends HistoryApi { nonBestBlock(blockToProcess) case None => logger.debug(s"Best full chain is empty. Returning empty progress info") - ProgressInfo(none, Seq.empty, Seq.empty, none) -> None + ProgressInfo(none, Seq.empty, Seq.empty, none) -> none } } @@ -48,7 +48,7 @@ trait HistoryPayloadsProcessor extends HistoryApi { fullBlock: Block, newBestHeader: Header, newBestChain: Seq[Block] - ): (ProgressInfo, Option[AwaitingAppendToHistory]) = { + ): (ProgressInfo, Option[HistoryUpdateInfoAcc]) = { logger.info(s"Appending ${fullBlock.encodedId} as a valid first block with height ${fullBlock.header.height}") ProgressInfo(none, Seq.empty, newBestChain, none) -> Some(updateStorage(fullBlock.payload, newBestHeader.id)) } @@ -58,7 +58,7 @@ trait HistoryPayloadsProcessor extends HistoryApi { newBestHeader: Header, newBestChain: Seq[Block], blocksToKeep: Int - ): (ProgressInfo, Option[AwaitingAppendToHistory]) = + ): (ProgressInfo, Option[HistoryUpdateInfoAcc]) = getHeaderOfBestBlock.map { header => val (prevChain: HeaderChain, newChain: HeaderChain) = commonBlockThenSuffixes(header, newBestHeader) val toRemove: Seq[Block] = prevChain.tail.headers @@ -83,7 +83,7 @@ trait HistoryPayloadsProcessor extends HistoryApi { ) val updatedHeadersAtHeightIds = newChain.headers.map(header => updatedBestHeaderAtHeightRaw(header.id, Height @@ header.height)).toList - val toUpdateInfo: AwaitingAppendToHistory = + val toUpdateInfo: HistoryUpdateInfoAcc = updateStorage(fullBlock.payload, newBestHeader.id, updateBestHeader, updatedHeadersAtHeightIds) if (blocksToKeep >= 0) { val lastKept: Int = blockDownloadProcessor.updateBestBlock(fullBlock.header) @@ -95,11 +95,11 @@ trait HistoryPayloadsProcessor extends HistoryApi { } }.getOrElse(ProgressInfo(none, Seq.empty, Seq.empty, none) -> None) - private def nonBestBlock(fullBlock: Block): (ProgressInfo, Option[AwaitingAppendToHistory]) = { + private def nonBestBlock(fullBlock: Block): (ProgressInfo, Option[HistoryUpdateInfoAcc]) = { //Orphaned block or full chain is not initialized yet logger.info(s"Process block to history ${fullBlock.encodedId}||${fullBlock.header.height}||") - //historyStorage.bulkInsert(fullBlock.payload.id, Seq.empty, Seq(fullBlock.payload)) - ProgressInfo(none, Seq.empty, Seq.empty, none) -> Some(AwaitingAppendToHistory(Seq.empty, fullBlock.payload)) + ProgressInfo(none, Seq.empty, Seq.empty, none) -> + HistoryUpdateInfoAcc(Seq.empty, fullBlock.payload, insertToObjectStore = false).some } private def updatedBestHeaderAtHeightRaw(headerId: ModifierId, height: Height): (Array[Byte], Array[Byte]) = @@ -107,10 +107,6 @@ trait HistoryPayloadsProcessor extends HistoryApi { (Seq(headerId) ++ headerIdsAtHeight(height).filterNot(_ sameElements headerId)).flatten.toArray - private def putToHistory(payload: Payload): ProgressInfo = - //historyStorage.insertObjects(Seq(payload)) - ProgressInfo(none, Seq.empty, Seq.empty, none) - private def isBetterChain(id: ModifierId): Boolean = (for { bestFullBlockId <- getBestBlockId @@ -154,15 +150,14 @@ trait HistoryPayloadsProcessor extends HistoryApi { bestFullHeaderId: ModifierId, updateHeaderInfo: Boolean = false, additionalIndexes: List[(Array[Byte], Array[Byte])] = List.empty - ): AwaitingAppendToHistory = { + ): HistoryUpdateInfoAcc = { val indicesToInsert: Seq[(Array[Byte], Array[Byte])] = if (updateHeaderInfo) Seq(BestBlockKey -> bestFullHeaderId, BestHeaderKey -> bestFullHeaderId) else Seq(BestBlockKey -> bestFullHeaderId) - AwaitingAppendToHistory((indicesToInsert ++ additionalIndexes).map { + HistoryUpdateInfoAcc((indicesToInsert ++ additionalIndexes).map { case (bytes, bytes1) => StorageKey @@ bytes -> StorageValue @@ bytes1 - }, newModRow) - //historyStorage.bulkInsert(newModRow.id, indicesToInsert ++ additionalIndexes, Seq(newModRow)) + }, newModRow, insertToObjectStore = false) } private def isValidFirstBlock(header: Header): Boolean = From 3f3d64dfc1546b38eaf46f1c2beb085ed7a79acf Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 23 Mar 2020 14:35:11 +0300 Subject: [PATCH 133/177] add reader to ProgressInfoForState --- src/main/scala/encry/nvg/IntermediaryNVHView.scala | 2 +- src/main/scala/encry/nvg/NVHHistory.scala | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index 1df19c4e67..d91ddd0fa4 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -70,7 +70,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, isProcessingModifierByHistory = false getNextModifier() - case ProgressInfoForState(pi, flag, isFullChainSynced) => //todo work with state starts here + case ProgressInfoForState(pi, flag, isFullChainSynced, reader) => //todo work with state starts here case msg: StateAction.ApplyFailed => historyRef ! msg case msg: StateAction.ModifierApplied => historyRef ! msg } diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index b8d0e8da54..5051530572 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -127,7 +127,10 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends A } object NVHHistory { - final case class ProgressInfoForState(pi: ProgressInfo, saveRootNodeFlag: Boolean, isFullChainSynced: Boolean) + final case class ProgressInfoForState(pi: ProgressInfo, + saveRootNodeFlag: Boolean, + isFullChainSynced: Boolean, + reader: HistoryReader) case object ModifierAppliedToHistory final case object InsertNewUpdates } From 688618fa4c00cd60b97bc9747e487f19bd5d8605 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 23 Mar 2020 15:07:23 +0300 Subject: [PATCH 134/177] improved history holder one more time --- .../scala/encry/nvg/IntermediaryNVHView.scala | 40 ++++++++----------- src/main/scala/encry/nvg/NVHHistory.scala | 34 ++++++++++++---- src/main/scala/encry/nvg/NodeViewHolder.scala | 33 --------------- 3 files changed, 43 insertions(+), 64 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index d91ddd0fa4..5e9de18e5b 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -3,10 +3,11 @@ package encry.nvg import akka.actor.{ Actor, ActorRef, Props } import com.typesafe.scalalogging.StrictLogging import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{ RegisterHistory, RegisterState } -import encry.nvg.IntermediaryNVHView.ModifierToAppend +import encry.nvg.IntermediaryNVHView.{ InitGenesisHistory, ModifierToAppend } import encry.nvg.ModifiersValidator.ValidatedModifier import encry.nvg.NVHHistory.{ ModifierAppliedToHistory, ProgressInfoForState } import encry.nvg.NVHState.StateAction +import encry.nvg.NodeViewHolder.SyntacticallyFailedModification import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader @@ -22,9 +23,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, val historyRef: ActorRef = ActorRef.noSender - var isProcessingModifierByHistory: Boolean = false - - var isProcessingModifierByState: Boolean = false + var isModifierProcessingInProgress: Boolean = false override def receive: Receive = awaitingViewActors() @@ -36,9 +35,10 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, context.actorOf( NVHState .restoreConsistentStateProps(settings, reader, influx) - .getOrElse( + .getOrElse { + historyRef ! InitGenesisHistory NVHState.genesisProps(settings, influx) - ) + } ) case RegisterHistory(_) => context.become(viewReceive(sender(), state.get)) @@ -50,29 +50,20 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, def viewReceive(history: ActorRef, state: ActorRef): Receive = { case ValidatedModifier(modifier: PersistentModifier) => - val wrappedKey: mutable.WrappedArray.ofByte = NodeViewHolder.toKey(modifier.id) - val isInHistory: Boolean = historyReader.isModifierDefined(modifier.id) - - val isInCache: Boolean = ModifiersCache.contains(wrappedKey) - + val isInCache: Boolean = ModifiersCache.contains(NodeViewHolder.toKey(modifier.id)) if (isInHistory || isInCache) logger.info( s"Modifier ${modifier.encodedId} can't be placed into the cache cause: " + s"contains in cache: $isInCache, contains in history: $isInHistory." ) - else ModifiersCache.put(wrappedKey, modifier, historyReader, settings) - - if (!isProcessingModifierByHistory && !isProcessingModifierByState) - getNextModifier() - - case ModifierAppliedToHistory => - isProcessingModifierByHistory = false - getNextModifier() - - case ProgressInfoForState(pi, flag, isFullChainSynced, reader) => //todo work with state starts here - case msg: StateAction.ApplyFailed => historyRef ! msg - case msg: StateAction.ModifierApplied => historyRef ! msg + else ModifiersCache.put(NodeViewHolder.toKey(modifier.id), modifier, historyReader, settings) + if (!isModifierProcessingInProgress) getNextModifier() + case ModifierAppliedToHistory => isModifierProcessingInProgress = false; getNextModifier() + case msg: ProgressInfoForState => //todo work with state starts here + case msg: StateAction.ApplyFailed => historyRef ! msg + case msg: StateAction.ModifierApplied => historyRef ! msg + case msg: SyntacticallyFailedModification => context.parent ! msg } def awaitingHistoryBranchPoint(history: ActorRef): Receive = ??? @@ -81,7 +72,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, ModifiersCache .popCandidate(historyReader, settings) .foreach { mod: PersistentModifier => - isProcessingModifierByHistory = true + isModifierProcessingInProgress = true logger.info(s"Got new modifiers in compute application function: ${mod.encodedId}.") historyRef ! ModifierToAppend(mod, isLocallyGenerated = false) } @@ -97,6 +88,7 @@ object IntermediaryNVHView { } final case class ModifierToAppend(modifier: PersistentModifier, isLocallyGenerated: Boolean) + case object InitGenesisHistory def props(settings: EncryAppSettings, ntp: NetworkTimeProvider, influxRef: Option[ActorRef]): Props = Props(new IntermediaryNVHView(settings, ntp, influxRef)) diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index 5051530572..827d6f9f46 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -5,10 +5,11 @@ import java.io.File import akka.actor.Actor import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging +import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.RegisterHistory -import encry.nvg.IntermediaryNVHView.ModifierToAppend +import encry.nvg.IntermediaryNVHView.{ InitGenesisHistory, ModifierToAppend } import encry.nvg.NVHHistory.{ ModifierAppliedToHistory, ProgressInfoForState } import encry.nvg.NVHState.StateAction import encry.nvg.NodeViewHolder.{ @@ -17,14 +18,14 @@ import encry.nvg.NodeViewHolder.{ SyntacticallyFailedModification } import encry.settings.EncryAppSettings -import encry.stats.StatsSender.{ ModifierAppendedToHistory, StartApplyingModifier } +import encry.stats.StatsSender._ import encry.utils.NetworkTimeProvider import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.history.History.HistoryUpdateInfoAcc import encry.view.history.{ History, HistoryReader } import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Header, Payload } +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } @@ -32,6 +33,8 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends A var history: History = initializeHistory + var lastProgressInfo: ProgressInfo = ProgressInfo(none, Seq.empty, Seq.empty, none) + context.parent ! RegisterHistory(HistoryReader(history)) override def receive: Receive = { @@ -52,6 +55,9 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends A s"Modifier ${mod.encodedId} of type ${mod.modifierTypeId} processed successfully by history. " + s"Time of processing is: ${(System.currentTimeMillis() - startProcessingTime) / 1000}s." ) + history.insertUpdateInfo(newUpdateInformation) + if (mod.modifierTypeId == Header.modifierTypeId) history.updateIdsForSyncInfo() + context.parent ! EndOfApplyingModifier(mod.id) context.parent ! ModifierAppendedToHistory(mod match { case _: Header => true case _: Payload => false @@ -61,9 +67,10 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends A context.parent ! ProgressInfoForState( progressInfo, (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2, - history.isFullChainSynced + history.isFullChainSynced, + HistoryReader(history) ) - history.insertUpdateInfo(newUpdateInformation) + lastProgressInfo = progressInfo if (!isLocallyGenerated) progressInfo.toApply.foreach { case header: Header => requestDownloads(progressInfo, header.id.some) case _ => requestDownloads(progressInfo, none) @@ -71,6 +78,7 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends A } else { logger.info(s"Progress info contains an empty toApply. Going to form request download.") if (!isLocallyGenerated) requestDownloads(progressInfo, mod.id.some) + context.parent ! HeightStatistics(history.getBestHeaderHeight, -1) //todo incorrect state height context.parent ! SemanticallySuccessfulModifier(mod) } } @@ -82,7 +90,16 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends A case StateAction.ModifierApplied(mod: PersistentModifier) => history = history.reportModifierIsValid(mod) context.parent ! HistoryReader(history) - context.parent ! ModifierAppliedToHistory + context.parent ! BlockAndHeaderInfo(history.getBestHeader, history.getBestBlock) + context.parent ! SemanticallySuccessfulModifier(mod) + if (history.getBestHeaderId.exists(besId => history.getBestBlockId.exists(_.sameElements(besId)))) + history.isFullChainSynced = true + context.parent ! HeightStatistics(history.getBestHeaderHeight, -1) //todo incorrect state height + if (mod match { + case _: Block => true + case _: Payload => true + case _ => false + }) context.parent ! ModifierAppendedToState(success = true) case StateAction.ApplyFailed(mod, e) => val (newHistory: History, progressInfo: ProgressInfo) = history.reportModifierIsInvalid(mod) @@ -90,10 +107,13 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends A context.parent ! ProgressInfoForState( progressInfo, (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2, - history.isFullChainSynced + history.isFullChainSynced, + HistoryReader(history) ) + lastProgressInfo = progressInfo history = newHistory + case InitGenesisHistory => history = initializeHistory } def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = none): Unit = diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 9e96d00e6a..b0fa32e731 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -207,10 +207,6 @@ class NodeViewHolder( isLocallyGenerated: Boolean = false ): (History, UtxoState, Seq[PersistentModifier]) = { logger.info(s"Starting updating state in updateState function!") -// if (!isLocallyGenerated) progressInfo.toApply.foreach { -// case header: Header => requestDownloads(progressInfo, header.id.some) -// case _ => requestDownloads(progressInfo, none) -// } val branchingPointOpt: Option[VersionTag] = progressInfo.branchPoint.map(VersionTag !@@ _) val (stateToApplyTry: Try[UtxoState], suffixTrimmed: IndexedSeq[PersistentModifier] @unchecked) = if (progressInfo.chainSwitchingNeeded) { @@ -284,17 +280,6 @@ class NodeViewHolder( } if (settings.node.mining && progressInfo.chainSwitchingNeeded) context.parent ! StartMining - context.parent ! SemanticallySuccessfulModifier(modToApply) - if (newHis.getBestHeaderId.exists( - bestHeaderId => - newHis.getBestBlockId.exists(bId => ByteArrayWrapper(bId) == ByteArrayWrapper(bestHeaderId)) - )) newHis.isFullChainSynced = true - context.parent ! HeightStatistics(newHis.getBestHeaderHeight, stateAfterApply.height) - if (modToApply match { - case _: Block => true - case _: Payload => true - case _ => false - }) context.parent ! ModifierAppendedToState(success = true) UpdateInformation(newHis, stateAfterApply, none, none, u.suffix :+ modToApply) case Left(e: List[ModifierApplyError]) => logger.info(s"Application to state failed cause $e") @@ -325,20 +310,6 @@ class NodeViewHolder( context.parent ! StartApplyingModifier(modifier.id, modifier.modifierTypeId, System.currentTimeMillis()) nodeView.history.append(modifier) match { case Right((historyBeforeStUpdate, progressInfo)) => - logger.info( - s"Successfully applied modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} to the history. " + - s"Time of applying is: ${(System.currentTimeMillis() - startApplicationToTheHistory) / 1000}s." - ) - if (modifier.modifierTypeId == Header.modifierTypeId) historyBeforeStUpdate.updateIdsForSyncInfo() - context.parent ! EndOfApplyingModifier(modifier.id) - context.parent ! ModifierAppendedToHistory(modifier match { - case _: Header => true - case _: Payload => false - }, success = true) - logger.info( - s"Going to apply modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} to the state. " + - s"Progress info is: $progressInfo." - ) if (progressInfo.toApply.nonEmpty) { val startPoint: Long = System.currentTimeMillis() logger.info(s"Progress info is non empty. To apply is: ${progressInfo.toApply.map(_.encodedId)}.") @@ -359,10 +330,6 @@ class NodeViewHolder( } updateNodeView(newHistory.some, newState.some, nodeView.wallet.some) } else { - logger.info(s"Progress info is empty.") - context.parent ! HeightStatistics(historyBeforeStUpdate.getBestHeaderHeight, nodeView.state.height) - if (!isLocallyGenerated) requestDownloads(progressInfo, modifier.id.some) - context.parent ! SemanticallySuccessfulModifier(modifier) updateNodeView(updatedHistory = historyBeforeStUpdate.some) } case Left(e: Throwable) => From 1f3aaa5a278cf49e3a3936deb504a8f0a1e50648 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 23 Mar 2020 15:22:15 +0300 Subject: [PATCH 135/177] added work with lgm --- .../scala/encry/nvg/IntermediaryNVHView.scala | 39 +++++++++++++------ src/main/scala/encry/nvg/ModifiersCache.scala | 38 ++++++++++-------- 2 files changed, 50 insertions(+), 27 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index 5e9de18e5b..9e43b5b072 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -1,20 +1,19 @@ package encry.nvg -import akka.actor.{ Actor, ActorRef, Props } +import akka.actor.{Actor, ActorRef, Props} import com.typesafe.scalalogging.StrictLogging -import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{ RegisterHistory, RegisterState } -import encry.nvg.IntermediaryNVHView.{ InitGenesisHistory, ModifierToAppend } +import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{RegisterHistory, RegisterState} +import encry.nvg.IntermediaryNVHView.{InitGenesisHistory, ModifierToAppend} import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NVHHistory.{ ModifierAppliedToHistory, ProgressInfoForState } +import encry.nvg.NVHHistory.{ModifierAppliedToHistory, ProgressInfoForState} import encry.nvg.NVHState.StateAction +import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier import encry.nvg.NodeViewHolder.SyntacticallyFailedModification import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier -import scala.collection.mutable - class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, influx: Option[ActorRef]) extends Actor with StrictLogging { @@ -49,6 +48,16 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, } def viewReceive(history: ActorRef, state: ActorRef): Receive = { + case LocallyGeneratedModifier(modifier: PersistentModifier) => + ModifiersCache.put( + NodeViewHolder.toKey(modifier.id), + modifier, + historyReader, + settings, + isLocallyGenerated = true + ) + if (!isModifierProcessingInProgress) getNextModifier() + case ValidatedModifier(modifier: PersistentModifier) => val isInHistory: Boolean = historyReader.isModifierDefined(modifier.id) val isInCache: Boolean = ModifiersCache.contains(NodeViewHolder.toKey(modifier.id)) @@ -57,7 +66,14 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, s"Modifier ${modifier.encodedId} can't be placed into the cache cause: " + s"contains in cache: $isInCache, contains in history: $isInHistory." ) - else ModifiersCache.put(NodeViewHolder.toKey(modifier.id), modifier, historyReader, settings) + else + ModifiersCache.put( + NodeViewHolder.toKey(modifier.id), + modifier, + historyReader, + settings, + isLocallyGenerated = false + ) if (!isModifierProcessingInProgress) getNextModifier() case ModifierAppliedToHistory => isModifierProcessingInProgress = false; getNextModifier() case msg: ProgressInfoForState => //todo work with state starts here @@ -71,10 +87,11 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, def getNextModifier(): Unit = ModifiersCache .popCandidate(historyReader, settings) - .foreach { mod: PersistentModifier => - isModifierProcessingInProgress = true - logger.info(s"Got new modifiers in compute application function: ${mod.encodedId}.") - historyRef ! ModifierToAppend(mod, isLocallyGenerated = false) + .foreach { + case (mod: PersistentModifier, isLocallyGenerated) => + isModifierProcessingInProgress = true + logger.info(s"Got new modifiers in compute application function: ${mod.encodedId}.") + historyRef ! ModifierToAppend(mod, isLocallyGenerated) } } diff --git a/src/main/scala/encry/nvg/ModifiersCache.scala b/src/main/scala/encry/nvg/ModifiersCache.scala index aeb3229040..26747abc52 100644 --- a/src/main/scala/encry/nvg/ModifiersCache.scala +++ b/src/main/scala/encry/nvg/ModifiersCache.scala @@ -8,7 +8,6 @@ import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.Header import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId - import scala.collection.concurrent.TrieMap import scala.collection.immutable.SortedMap import scala.collection.mutable @@ -17,7 +16,7 @@ object ModifiersCache extends StrictLogging { private type Key = mutable.WrappedArray[Byte] - val cache: TrieMap[Key, PersistentModifier] = TrieMap.empty + val cache: TrieMap[Key, (PersistentModifier, Boolean)] = TrieMap.empty private var headersCollection: SortedMap[Int, List[ModifierId]] = SortedMap.empty[Int, List[ModifierId]] @@ -31,10 +30,16 @@ object ModifiersCache extends StrictLogging { def contains(key: Key): Boolean = cache.contains(key) - def put(key: Key, value: PersistentModifier, history: HistoryReader, settings: EncryAppSettings): Unit = + def put( + key: Key, + value: PersistentModifier, + history: HistoryReader, + settings: EncryAppSettings, + isLocallyGenerated: Boolean + ): Unit = if (!contains(key)) { logger.debug(s"Put ${value.encodedId} of type ${value.modifierTypeId} to cache.") - cache.put(key, value) + cache.put(key, value -> isLocallyGenerated) value match { case header: Header => val possibleHeadersAtCurrentHeight: List[ModifierId] = headersCollection.getOrElse(header.height, List()) @@ -50,7 +55,7 @@ object ModifiersCache extends StrictLogging { } if (size > settings.node.modifiersCacheSize) cache.find { - case (_, modifier) => + case (_, (modifier, _)) => history.testApplicable(modifier) match { case Right(_) | Left(_: NonFatalValidationError) => false case _ => true @@ -58,14 +63,15 @@ object ModifiersCache extends StrictLogging { }.map(mod => remove(mod._1)) } - def remove(key: Key): Option[PersistentModifier] = { + def remove(key: Key): Option[(PersistentModifier, Boolean)] = { logger.debug(s"Going to delete ${Algos.encode(key.toArray)}. Cache contains: ${cache.get(key).isDefined}.") cache.remove(key) } - def popCandidate(history: HistoryReader, settings: EncryAppSettings): List[PersistentModifier] = synchronized { - findCandidateKey(history, settings).take(1).flatMap(k => remove(k)) - } + def popCandidate(history: HistoryReader, settings: EncryAppSettings): List[(PersistentModifier, Boolean)] = + synchronized { + findCandidateKey(history, settings).take(1).flatMap(k => remove(k)) + } override def toString: String = cache.keys.map(key => Algos.encode(key.toArray)).mkString(",") @@ -74,14 +80,14 @@ object ModifiersCache extends StrictLogging { def isApplicable(key: Key): Boolean = cache .get(key) - .exists( - modifier => + .exists { + case (modifier, _) => history.testApplicable(modifier) match { case Left(_: FatalValidationError) => remove(key); false case Right(_) => true case Left(_) => false - } - ) + } + } def getHeadersKeysAtHeight(height: Int): List[Key] = headersCollection.get(height) match { @@ -96,8 +102,8 @@ object ModifiersCache extends StrictLogging { def exhaustiveSearch: List[Key] = List(cache.find { case (k, v) => - v match { - case _: Header if history.getBestHeaderId.exists(_ sameElements v.parentId) => true + v._1 match { + case _: Header if history.getBestHeaderId.exists(_ sameElements v._1.parentId) => true case _ => val isApplicableMod: Boolean = isApplicable(k) isApplicableMod @@ -111,7 +117,7 @@ object ModifiersCache extends StrictLogging { logger.debug(s"HeadersCollection size is: ${headersCollection.size}") logger.debug(s"Drop height ${history.getBestHeaderHeight + 1} in HeadersCollection") val res = value.map(cache.get(_)).collect { - case Some(v: Header) + case Some((v: Header, _)) if (history.getBestHeaderHeight == settings.constants.PreGenesisHeight && (v.parentId sameElements Header.GenesisParentId) || history.getHeaderById(v.parentId).nonEmpty) && isApplicable(new mutable.WrappedArray.ofByte(v.id)) => From f8f0fd761cfa4acdfa27887c8a0dc63221c4c775 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 23 Mar 2020 15:23:35 +0300 Subject: [PATCH 136/177] history holder actor creation added --- .../scala/encry/nvg/IntermediaryNVHView.scala | 10 +++++----- src/main/scala/encry/nvg/NVHHistory.scala | 19 ++++++++----------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index 9e43b5b072..ec527c8a66 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -1,11 +1,11 @@ package encry.nvg -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{ Actor, ActorRef, Props } import com.typesafe.scalalogging.StrictLogging -import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{RegisterHistory, RegisterState} -import encry.nvg.IntermediaryNVHView.{InitGenesisHistory, ModifierToAppend} +import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{ RegisterHistory, RegisterState } +import encry.nvg.IntermediaryNVHView.{ InitGenesisHistory, ModifierToAppend } import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NVHHistory.{ModifierAppliedToHistory, ProgressInfoForState} +import encry.nvg.NVHHistory.{ ModifierAppliedToHistory, ProgressInfoForState } import encry.nvg.NVHState.StateAction import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier import encry.nvg.NodeViewHolder.SyntacticallyFailedModification @@ -20,7 +20,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, var historyReader: HistoryReader = HistoryReader.empty - val historyRef: ActorRef = ActorRef.noSender + val historyRef: ActorRef = context.actorOf(NVHHistory.props(ntp, settings)) var isModifierProcessingInProgress: Boolean = false diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index 827d6f9f46..2dbcf55b41 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -2,32 +2,28 @@ package encry.nvg import java.io.File -import akka.actor.Actor +import akka.actor.{Actor, Props} import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.RegisterHistory -import encry.nvg.IntermediaryNVHView.{ InitGenesisHistory, ModifierToAppend } -import encry.nvg.NVHHistory.{ ModifierAppliedToHistory, ProgressInfoForState } +import encry.nvg.IntermediaryNVHView.{InitGenesisHistory, ModifierToAppend} +import encry.nvg.NVHHistory.{ModifierAppliedToHistory, ProgressInfoForState} import encry.nvg.NVHState.StateAction -import encry.nvg.NodeViewHolder.{ - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification -} +import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.NetworkTimeProvider import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.history.History.HistoryUpdateInfoAcc -import encry.view.history.{ History, HistoryReader } +import encry.view.history.{History, HistoryReader} import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } +import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends Actor with StrictLogging { @@ -153,4 +149,5 @@ object NVHHistory { reader: HistoryReader) case object ModifierAppliedToHistory final case object InsertNewUpdates + def props(ntp: NetworkTimeProvider, settings: EncryAppSettings): Props = Props(new NVHHistory(settings, ntp)) } From 0cbd92477967647aca57e4b3a0e03d45033d61f5 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 23 Mar 2020 15:38:34 +0300 Subject: [PATCH 137/177] separate progress info --- src/main/scala/encry/nvg/IntermediaryNVHView.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index ec527c8a66..46c4b3a2ff 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -76,7 +76,8 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, ) if (!isModifierProcessingInProgress) getNextModifier() case ModifierAppliedToHistory => isModifierProcessingInProgress = false; getNextModifier() - case msg: ProgressInfoForState => //todo work with state starts here + case msg: ProgressInfoForState if msg.pi.chainSwitchingNeeded => //todo work with state starts here + case msg: ProgressInfoForState => //todo work with state starts here case msg: StateAction.ApplyFailed => historyRef ! msg case msg: StateAction.ModifierApplied => historyRef ! msg case msg: SyntacticallyFailedModification => context.parent ! msg From bb529488d4bbe6cf78b625d20bb845615f22e041 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 23 Mar 2020 15:40:25 +0300 Subject: [PATCH 138/177] added messages forwarding through nvh middle actor --- .../scala/encry/nvg/IntermediaryNVHView.scala | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index ec527c8a66..347eb6cf2e 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -2,14 +2,21 @@ package encry.nvg import akka.actor.{ Actor, ActorRef, Props } import com.typesafe.scalalogging.StrictLogging +import encry.api.http.DataHolderForApi.BlockAndHeaderInfo +import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{ RegisterHistory, RegisterState } import encry.nvg.IntermediaryNVHView.{ InitGenesisHistory, ModifierToAppend } import encry.nvg.ModifiersValidator.ValidatedModifier import encry.nvg.NVHHistory.{ ModifierAppliedToHistory, ProgressInfoForState } import encry.nvg.NVHState.StateAction import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier -import encry.nvg.NodeViewHolder.SyntacticallyFailedModification +import encry.nvg.NodeViewHolder.{ + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification +} import encry.settings.EncryAppSettings +import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader import org.encryfoundation.common.modifiers.PersistentModifier @@ -80,6 +87,12 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, case msg: StateAction.ApplyFailed => historyRef ! msg case msg: StateAction.ModifierApplied => historyRef ! msg case msg: SyntacticallyFailedModification => context.parent ! msg + case msg: StatsSenderMessage => context.parent ! msg + case msg: RequestFromLocal => context.parent ! msg + case msg: SemanticallyFailedModification => context.parent ! msg + case msg: SemanticallySuccessfulModifier => context.parent ! msg + case msg: BlockAndHeaderInfo => context.parent ! msg + case msg: HistoryReader => historyReader = msg; context.parent ! msg } def awaitingHistoryBranchPoint(history: ActorRef): Receive = ??? From e6189c78c88d6680fecba4198dc875c50bc5003d Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 23 Mar 2020 15:40:59 +0300 Subject: [PATCH 139/177] added messages forwarding through nvh middle actor --- src/main/scala/encry/nvg/IntermediaryNVHView.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index d8666d4ff1..9d3f738ac6 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -82,6 +82,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, isLocallyGenerated = false ) if (!isModifierProcessingInProgress) getNextModifier() + case ModifierAppliedToHistory => isModifierProcessingInProgress = false; getNextModifier() case msg: ProgressInfoForState if msg.pi.chainSwitchingNeeded => //todo work with state starts here case msg: ProgressInfoForState => //todo work with state starts here From 7eed136ffc9d9d71875dd6cbaa683aaa319f4322 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 23 Mar 2020 16:11:59 +0300 Subject: [PATCH 140/177] add utxo state reader --- .../scala/encry/nvg/IntermediaryNVHView.scala | 17 +++++---- src/main/scala/encry/nvg/NVHState.scala | 27 ++++++-------- .../scala/encry/view/state/UtxoState.scala | 21 +++++++++-- .../encry/view/state/UtxoStateReader.scala | 37 +++++++++++++------ 4 files changed, 64 insertions(+), 38 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index 46c4b3a2ff..b640452f1c 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -1,17 +1,18 @@ package encry.nvg -import akka.actor.{ Actor, ActorRef, Props } +import akka.actor.{Actor, ActorRef, Props} import com.typesafe.scalalogging.StrictLogging -import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{ RegisterHistory, RegisterState } -import encry.nvg.IntermediaryNVHView.{ InitGenesisHistory, ModifierToAppend } +import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{RegisterHistory, RegisterState} +import encry.nvg.IntermediaryNVHView.{InitGenesisHistory, ModifierToAppend} import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NVHHistory.{ ModifierAppliedToHistory, ProgressInfoForState } +import encry.nvg.NVHHistory.{ModifierAppliedToHistory, ProgressInfoForState} import encry.nvg.NVHState.StateAction import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier import encry.nvg.NodeViewHolder.SyntacticallyFailedModification import encry.settings.EncryAppSettings import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader +import encry.view.state.UtxoStateReader import org.encryfoundation.common.modifiers.PersistentModifier class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, influx: Option[ActorRef]) @@ -41,7 +42,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, ) case RegisterHistory(_) => context.become(viewReceive(sender(), state.get)) - case RegisterState if history.isEmpty => + case RegisterState(_) if history.isEmpty => context.become(awaitingViewActors(history, Some(sender())), discardOld = true) case RegisterHistory => context.become(viewReceive(history.get, sender())) @@ -75,9 +76,11 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, isLocallyGenerated = false ) if (!isModifierProcessingInProgress) getNextModifier() - case ModifierAppliedToHistory => isModifierProcessingInProgress = false; getNextModifier() + case ModifierAppliedToHistory => isModifierProcessingInProgress = false; getNextModifier() case msg: ProgressInfoForState if msg.pi.chainSwitchingNeeded => //todo work with state starts here + case msg: ProgressInfoForState => //todo work with state starts here + case msg: StateAction.ApplyFailed => historyRef ! msg case msg: StateAction.ModifierApplied => historyRef ! msg case msg: SyntacticallyFailedModification => context.parent ! msg @@ -102,7 +105,7 @@ object IntermediaryNVHView { sealed trait IntermediaryNVHViewActions object IntermediaryNVHViewActions { case class RegisterHistory(historyReader: HistoryReader) extends IntermediaryNVHViewActions - case object RegisterState extends IntermediaryNVHViewActions + case class RegisterState(stateReader: UtxoStateReader) extends IntermediaryNVHViewActions } final case class ModifierToAppend(modifier: PersistentModifier, isLocallyGenerated: Boolean) diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index 3e1c193f68..891241b4a5 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -5,40 +5,35 @@ import java.io.File import akka.actor.{Actor, ActorRef, Props} import cats.syntax.option.none import com.typesafe.scalalogging.StrictLogging -import encry.api.http.DataHolderForApi.BlockAndHeaderInfo -import encry.consensus.HistoryConsensus.ProgressInfo import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.RegisterState import encry.nvg.NVHState.StateAction.{ApplyFailed, ApplyModifier, CreateTreeChunks, ModifierApplied} -import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier, UpdateInformation} -import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId -import encry.nvg.fast.sync.SnapshotProcessor.{SnapshotChunk, TreeChunks} -import encry.settings.{EncryAppSettings, NodeSettings} -import encry.stats.StatsSender.{HeightStatistics, ModifierAppendedToState, TransactionsInBlock} -import encry.storage.{RootNodesStorage, VersionalStorage} +import encry.nvg.fast.sync.SnapshotProcessor.SnapshotChunk +import encry.settings.EncryAppSettings +import encry.stats.StatsSender.TransactionsInBlock import encry.storage.VersionalStorage.{StorageKey, StorageValue} import encry.storage.iodb.versionalIODB.IODBWrapper import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, VersionalLevelDBCompanion} +import encry.storage.{RootNodesStorage, VersionalStorage} import encry.utils.CoreTaggedTypes.VersionTag import encry.view.NodeViewErrors.ModifierApplyError -import encry.view.history.{History, HistoryReader} -import encry.view.state.UtxoState -import encry.view.state.UtxoState.logger +import encry.view.history.HistoryReader +import encry.view.state.{UtxoState, UtxoStateReader} import encry.view.state.avlTree.AvlTree -import io.iohk.iodb.{ByteArrayWrapper, LSMStore} +import encry.view.state.avlTree.utils.implicits.Instances._ +import io.iohk.iodb.LSMStore import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{Block, Header} import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId} import org.encryfoundation.common.utils.constants.Constants import org.iq80.leveldb.Options -import encry.view.state.avlTree.utils.implicits.Instances._ import scala.util.Try class NVHState(influxRef: Option[ActorRef], var state: UtxoState, settings: EncryAppSettings) extends Actor with StrictLogging with AutoCloseable { - override def preStart(): Unit = context.parent ! RegisterState + override def preStart(): Unit = context.parent ! RegisterState(UtxoStateReader(state)) override def receive: Receive = { case ApplyModifier(modifier: PersistentModifier, @@ -48,7 +43,7 @@ class NVHState(influxRef: Option[ActorRef], var state: UtxoState, settings: Encr case Right(stateAfterApply) => modifier match { case b: Block if isFullChainSynced => context.parent ! TransactionsInBlock(b.payload.txs.size) - case _ => + case _ => } state = stateAfterApply logger.info(s"Successfully apply modifier: ${Algos.encode(modifier.id)} of type ${modifier.modifierTypeId}") diff --git a/src/main/scala/encry/view/state/UtxoState.scala b/src/main/scala/encry/view/state/UtxoState.scala index c37e81bb0b..20f2236b50 100644 --- a/src/main/scala/encry/view/state/UtxoState.scala +++ b/src/main/scala/encry/view/state/UtxoState.scala @@ -1,7 +1,6 @@ package encry.view.state import java.io.File - import akka.actor.ActorRef import cats.data.Validated import cats.instances.list._ @@ -35,12 +34,11 @@ import org.encryfoundation.common.modifiers.state.box.Box.Amount import org.encryfoundation.common.modifiers.state.box.TokenIssuingBox.TokenId import org.encryfoundation.common.modifiers.state.box.{AssetBox, EncryBaseBox, EncryProposition} import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, Height} +import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ADKey, Height} import org.encryfoundation.common.utils.constants.Constants import org.encryfoundation.common.validation.ValidationResult.Invalid import org.encryfoundation.common.validation.{MalformedModifierError, ValidationResult} import org.iq80.leveldb.Options - import scala.util.Try final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], @@ -208,6 +206,23 @@ final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], .map(err => Invalid(Seq(err)).asLeft[Transaction]) .getOrElse(tx.asRight[ValidationResult]) + override def version: VersionTag = VersionTag !@@ tree.avlStorage.currentVersion + + override def stateSafePointHeight: Unit = tree.rootNodesStorage.safePointHeight + + override def boxById(boxId: ADKey): Option[EncryBaseBox] = tree.get(StorageKey !@@ boxId) + .map(bytes => StateModifierSerializer.parseBytes(bytes, boxId.head)).flatMap(_.toOption) + + override def boxesByIds(ids: Seq[ADKey]): Seq[EncryBaseBox] = + ids.foldLeft(Seq.empty[EncryBaseBox])((acc, id) => + boxById(id).map(bx => acc :+ bx).getOrElse(acc) + ) + + override def typedBoxById[B <: EncryBaseBox](boxId: ADKey): Option[EncryBaseBox] = + boxById(boxId) match { + case Some(bx: B@unchecked) if bx.isInstanceOf[B] => Some(bx) + case _ => None + } def close(): Unit = tree.close() } diff --git a/src/main/scala/encry/view/state/UtxoStateReader.scala b/src/main/scala/encry/view/state/UtxoStateReader.scala index 9bdcf4bc1f..7b1c967ae7 100644 --- a/src/main/scala/encry/view/state/UtxoStateReader.scala +++ b/src/main/scala/encry/view/state/UtxoStateReader.scala @@ -13,21 +13,34 @@ trait UtxoStateReader { implicit val hf: Algos.HF = Algos.hash - val tree: AvlTree[StorageKey, StorageValue] + //val tree: AvlTree[StorageKey, StorageValue] - def version: VersionTag = VersionTag !@@ tree.avlStorage.currentVersion + def version: VersionTag// = VersionTag !@@ tree.avlStorage.currentVersion - def stateSafePointHeight = tree.rootNodesStorage.safePointHeight + def stateSafePointHeight// = tree.rootNodesStorage.safePointHeight - def boxById(boxId: ADKey): Option[EncryBaseBox] = tree.get(StorageKey !@@ boxId) - .map(bytes => StateModifierSerializer.parseBytes(bytes, boxId.head)).flatMap(_.toOption) + def boxById(boxId: ADKey): Option[EncryBaseBox]// = +// tree.get(StorageKey !@@ boxId) +// .map(bytes => StateModifierSerializer.parseBytes(bytes, boxId.head)).flatMap(_.toOption) - def boxesByIds(ids: Seq[ADKey]): Seq[EncryBaseBox] = ids.foldLeft(Seq[EncryBaseBox]())((acc, id) => - boxById(id).map(bx => acc :+ bx).getOrElse(acc)) + def boxesByIds(ids: Seq[ADKey]): Seq[EncryBaseBox] //= +// ids.foldLeft(Seq[EncryBaseBox]())((acc, id) => +// boxById(id).map(bx => acc :+ bx).getOrElse(acc)) - def typedBoxById[B <: EncryBaseBox](boxId: ADKey): Option[EncryBaseBox] = - boxById(boxId) match { - case Some(bx: B@unchecked) if bx.isInstanceOf[B] => Some(bx) - case _ => None - } + def typedBoxById[B <: EncryBaseBox](boxId: ADKey): Option[EncryBaseBox] //= +// boxById(boxId) match { +// case Some(bx: B@unchecked) if bx.isInstanceOf[B] => Some(bx) +// case _ => None +// } +} + +object UtxoStateReader { + + def apply(state: UtxoState): UtxoStateReader = new UtxoStateReader { + override def version: VersionTag = state.version + override def stateSafePointHeight: Unit = state.safePointHeight + override def boxById(boxId: ADKey): Option[EncryBaseBox] = state.boxById(boxId) + override def boxesByIds(ids: Seq[ADKey]): Seq[EncryBaseBox] = state.boxesByIds(ids) + override def typedBoxById[B <: EncryBaseBox](boxId: ADKey): Option[EncryBaseBox] = state.typedBoxById() + } } \ No newline at end of file From 76376d0ad5a429e1bba859249967e61b93f784e6 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 23 Mar 2020 17:07:52 +0300 Subject: [PATCH 141/177] add rollback logic to interNvhView --- .../scala/encry/nvg/IntermediaryNVHView.scala | 49 +++++++++++++------ src/main/scala/encry/nvg/NVHState.scala | 14 +++--- .../encry/view/state/UtxoStateReader.scala | 24 +++------ 3 files changed, 50 insertions(+), 37 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index e421fdf6ec..6b44f82e96 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -4,23 +4,25 @@ import akka.actor.{Actor, ActorRef, Props} import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.network.Messages.MessageToNetwork.RequestFromLocal -import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{ RegisterHistory, RegisterState } -import encry.nvg.IntermediaryNVHView.{ InitGenesisHistory, ModifierToAppend } +import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{RegisterHistory, RegisterState} +import encry.nvg.IntermediaryNVHView.{InitGenesisHistory, ModifierToAppend} import encry.nvg.ModifiersValidator.ValidatedModifier import encry.nvg.NVHHistory.{ModifierAppliedToHistory, ProgressInfoForState} import encry.nvg.NVHState.StateAction import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier -import encry.nvg.NodeViewHolder.{ - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification -} +import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification} import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage +import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader import encry.view.state.UtxoStateReader import org.encryfoundation.common.modifiers.PersistentModifier +import org.encryfoundation.common.modifiers.history.{Block, Header} +import org.encryfoundation.common.utils.TaggedTypes.ModifierId + +import scala.collection.IndexedSeq +import scala.util.{Failure, Success} class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, influx: Option[ActorRef]) extends Actor @@ -34,7 +36,9 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, override def receive: Receive = awaitingViewActors() - def awaitingViewActors(history: Option[ActorRef] = None, state: Option[ActorRef] = None): Receive = { + def awaitingViewActors(history: Option[ActorRef] = None, + state: Option[ActorRef] = None, + stateReader: Option[UtxoStateReader] = None): Receive = { case RegisterHistory(reader) if state.isEmpty => historyReader = reader logger.info(s"NodeViewParent actor got init history. Going to init state actor.") @@ -48,14 +52,17 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, } ) case RegisterHistory(_) => - context.become(viewReceive(sender(), state.get)) - case RegisterState(_) if history.isEmpty => - context.become(awaitingViewActors(history, Some(sender())), discardOld = true) + context.become(viewReceive(sender(), state.get, stateReader.get)) + case RegisterState(reader) if history.isEmpty => + context.become(awaitingViewActors(history, Some(sender()), Some(reader)), discardOld = true) case RegisterHistory => - context.become(viewReceive(history.get, sender())) + context.become(viewReceive(history.get, sender(), stateReader.get)) } - def viewReceive(history: ActorRef, state: ActorRef): Receive = { + def viewReceive(history: ActorRef, state: ActorRef, stateReader: UtxoStateReader): Receive = { + + case RegisterState(reader) => context.become(viewReceive(history, sender(), reader)) + case LocallyGeneratedModifier(modifier: PersistentModifier) => ModifiersCache.put( NodeViewHolder.toKey(modifier.id), @@ -84,7 +91,21 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, ) if (!isModifierProcessingInProgress) getNextModifier() case ModifierAppliedToHistory => isModifierProcessingInProgress = false; getNextModifier() - case msg: ProgressInfoForState if msg.pi.chainSwitchingNeeded => //todo work with state starts here + case msg: ProgressInfoForState if msg.pi.chainSwitchingNeeded => + context.become(viewReceive( + history, + context.actorOf( + NVHState.rollbackProps( + settings, + influx, + stateReader.safePointHeight, + msg.pi.branchPoint.get, + historyReader, + settings.constants + ) + ), + stateReader + )) case msg: ProgressInfoForState => //todo work with state starts here case msg: StateAction.ApplyFailed => historyRef ! msg case msg: StateAction.ModifierApplied => historyRef ! msg diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index 891241b4a5..6f6ce5209c 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -116,12 +116,12 @@ object NVHState extends StrictLogging { } //rollback - def props(settings: EncryAppSettings, - influxRef: Option[ActorRef], - safePointHeight: Int, - branchPoint: VersionTag, - historyReader: HistoryReader, - constants: Constants): Props = { + def rollbackProps(settings: EncryAppSettings, + influxRef: Option[ActorRef], + safePointHeight: Int, + branchPoint: ModifierId, + historyReader: HistoryReader, + constants: Constants): Props = { val branchPointHeight: Int = historyReader.getHeaderById(ModifierId !@@ branchPoint).get.height val additionalBlocks: List[Block] = (safePointHeight + 1 to branchPointHeight).foldLeft(List.empty[Block]) { @@ -151,7 +151,7 @@ object NVHState extends StrictLogging { RootNodesStorage[StorageKey, StorageValue](levelDBInit, settings.constants.MaxRollbackDepth, rootsDir) } val state: UtxoState = UtxoState.rollbackTo( - branchPoint, + VersionTag !@@ branchPoint, additionalBlocks, versionalStorage, rootStorage, diff --git a/src/main/scala/encry/view/state/UtxoStateReader.scala b/src/main/scala/encry/view/state/UtxoStateReader.scala index 7b1c967ae7..cf2f2f265c 100644 --- a/src/main/scala/encry/view/state/UtxoStateReader.scala +++ b/src/main/scala/encry/view/state/UtxoStateReader.scala @@ -7,38 +7,30 @@ import encry.view.state.avlTree.utils.implicits.Instances._ import org.encryfoundation.common.modifiers.state.StateModifierSerializer import org.encryfoundation.common.modifiers.state.box.EncryBaseBox import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.ADKey +import org.encryfoundation.common.utils.TaggedTypes.{ADKey, Height} trait UtxoStateReader { implicit val hf: Algos.HF = Algos.hash - //val tree: AvlTree[StorageKey, StorageValue] + def version: VersionTag - def version: VersionTag// = VersionTag !@@ tree.avlStorage.currentVersion + def stateSafePointHeight: Height - def stateSafePointHeight// = tree.rootNodesStorage.safePointHeight + def boxById(boxId: ADKey): Option[EncryBaseBox] - def boxById(boxId: ADKey): Option[EncryBaseBox]// = -// tree.get(StorageKey !@@ boxId) -// .map(bytes => StateModifierSerializer.parseBytes(bytes, boxId.head)).flatMap(_.toOption) + def boxesByIds(ids: Seq[ADKey]): Seq[EncryBaseBox] - def boxesByIds(ids: Seq[ADKey]): Seq[EncryBaseBox] //= -// ids.foldLeft(Seq[EncryBaseBox]())((acc, id) => -// boxById(id).map(bx => acc :+ bx).getOrElse(acc)) + def typedBoxById[B <: EncryBaseBox](boxId: ADKey): Option[EncryBaseBox] - def typedBoxById[B <: EncryBaseBox](boxId: ADKey): Option[EncryBaseBox] //= -// boxById(boxId) match { -// case Some(bx: B@unchecked) if bx.isInstanceOf[B] => Some(bx) -// case _ => None -// } + def safePointHeight: Height } object UtxoStateReader { def apply(state: UtxoState): UtxoStateReader = new UtxoStateReader { override def version: VersionTag = state.version - override def stateSafePointHeight: Unit = state.safePointHeight + override def stateSafePointHeight: Height = state.safePointHeight override def boxById(boxId: ADKey): Option[EncryBaseBox] = state.boxById(boxId) override def boxesByIds(ids: Seq[ADKey]): Seq[EncryBaseBox] = state.boxesByIds(ids) override def typedBoxById[B <: EncryBaseBox](boxId: ADKey): Option[EncryBaseBox] = state.typedBoxById() From 699fa80e0e77b53a9996f35db24eb80ceaacbba2 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 24 Mar 2020 12:16:15 +0300 Subject: [PATCH 142/177] add autocloseable to storages, comment tests --- .../scala/encry/cli/commands/AddPeer.scala | 3 +- .../scala/encry/nvg/IntermediaryNVH.scala | 2 +- .../scala/encry/nvg/IntermediaryNVHView.scala | 13 +- src/main/scala/encry/nvg/NVHHistory.scala | 5 +- src/main/scala/encry/nvg/NVHState.scala | 19 +- src/main/scala/encry/nvg/NodeViewHolder.scala | 833 +++++----- .../encry/storage/RootNodesStorage.scala | 7 +- .../iodb/versionalIODB/IODBWrapper.scala | 2 +- .../versionalLevelDB/VLDBWrapper.scala | 2 +- .../history/HistoryModifiersValidator.scala | 3 +- .../scala/encry/view/state/UtxoState.scala | 12 +- .../encry/view/state/UtxoStateReader.scala | 16 +- .../scala/encry/view/wallet/EncryWallet.scala | 4 +- src/test/scala/encry/nvg/NVHStateTest.scala | 206 +-- .../encry/nvg/NodeViewNMProcessorTests.scala | 1399 +++++++++-------- src/test/scala/encry/nvg/PipelinesTests.scala | 1108 ++++++------- src/test/scala/encry/nvg/ValidatorTests.scala | 368 ++--- src/test/scala/encry/view/QWE.scala | 196 +-- .../scala/encry/view/wallet/WalletSpec.scala | 124 +- 19 files changed, 2189 insertions(+), 2133 deletions(-) diff --git a/src/main/scala/encry/cli/commands/AddPeer.scala b/src/main/scala/encry/cli/commands/AddPeer.scala index a3e198b630..d20b4dee51 100644 --- a/src/main/scala/encry/cli/commands/AddPeer.scala +++ b/src/main/scala/encry/cli/commands/AddPeer.scala @@ -10,7 +10,8 @@ import scala.concurrent.Future /** * Command "peer addPeer -host= -port=" - * Example: peer addPeer -host='172.16.10.57' -port=9040 + * 10.101.0.18:59954 + * Example: peer addPeer -host='10.101.0.18' -port=59954 */ object AddPeer extends Command { override def execute(args: Command.Args, diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index f82d227e89..846188f071 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -60,7 +60,7 @@ class IntermediaryNVH( val networkMessagesProcessor: ActorRef = context.actorOf(NodeViewNMProcessor.props(settings), name = "Network-messages-processor") val nodeViewHolder: ActorRef = - context.actorOf(NodeViewHolder.props(settings, timeProvider, influxRef), name = "Node-view-holder") + context.actorOf(IntermediaryNVHView.props(settings, timeProvider, influxRef), name = "Node-view-holder") val modifiersValidatorRouter: ActorRef = context.actorOf( BalancingPool(5) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index 6b44f82e96..e36dd2d1b3 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -9,6 +9,7 @@ import encry.nvg.IntermediaryNVHView.{InitGenesisHistory, ModifierToAppend} import encry.nvg.ModifiersValidator.ValidatedModifier import encry.nvg.NVHHistory.{ModifierAppliedToHistory, ProgressInfoForState} import encry.nvg.NVHState.StateAction +import encry.nvg.NVHState.StateAction.ApplyModifier import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification} import encry.settings.EncryAppSettings @@ -17,6 +18,7 @@ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.history.HistoryReader import encry.view.state.UtxoStateReader +import io.iohk.iodb.ByteArrayWrapper import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{Block, Header} import org.encryfoundation.common.utils.TaggedTypes.ModifierId @@ -34,6 +36,8 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, var isModifierProcessingInProgress: Boolean = false + var toApply = Set.empty[ByteArrayWrapper] + override def receive: Receive = awaitingViewActors() def awaitingViewActors(history: Option[ActorRef] = None, @@ -91,7 +95,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, ) if (!isModifierProcessingInProgress) getNextModifier() case ModifierAppliedToHistory => isModifierProcessingInProgress = false; getNextModifier() - case msg: ProgressInfoForState if msg.pi.chainSwitchingNeeded => + case msg: ProgressInfoForState if msg.pi.chainSwitchingNeeded && msg.pi.branchPoint.exists(point => !stateReader.version.sameElements(point))=> context.become(viewReceive( history, context.actorOf( @@ -106,9 +110,12 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, ), stateReader )) - case msg: ProgressInfoForState => //todo work with state starts here + case msg: ProgressInfoForState => + toApply = msg.pi.toApply.map(mod => ByteArrayWrapper(mod.id)).toSet + msg.pi.toApply.foreach(mod => state ! StateAction.ApplyModifier(mod, msg.saveRootNodeFlag, msg.isFullChainSynced)) case msg: StateAction.ApplyFailed => historyRef ! msg - case msg: StateAction.ModifierApplied => historyRef ! msg + case msg: StateAction.ModifierApplied => + historyRef ! msg case msg: SyntacticallyFailedModification => context.parent ! msg case msg: StatsSenderMessage => context.parent ! msg case msg: RequestFromLocal => context.parent ! msg diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index 2dbcf55b41..32398aedb3 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -27,6 +27,7 @@ import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends Actor with StrictLogging { + logger.info("start here!") var history: History = initializeHistory var lastProgressInfo: ProgressInfo = ProgressInfo(none, Seq.empty, Seq.empty, none) @@ -109,7 +110,9 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends A lastProgressInfo = progressInfo history = newHistory - case InitGenesisHistory => history = initializeHistory + case InitGenesisHistory => + logger.info("Init in InitGenesisHistory") + history = initializeHistory } def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = none): Unit = diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index 6f6ce5209c..04ca2df2e1 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -28,7 +28,7 @@ import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId} import org.encryfoundation.common.utils.constants.Constants import org.iq80.leveldb.Options -import scala.util.Try +import scala.util.{Failure, Success, Try} class NVHState(influxRef: Option[ActorRef], var state: UtxoState, settings: EncryAppSettings) extends Actor with StrictLogging with AutoCloseable { @@ -84,14 +84,21 @@ object NVHState extends StrictLogging { case class TreeChunks(chunks: List[SnapshotChunk]) extends StateAction } + def restoreProps(settings: EncryAppSettings, + historyReader: HistoryReader, + influxRef: Option[ActorRef]): Props = { + + } + //genesis state def genesisProps(settings: EncryAppSettings, influxRef: Option[ActorRef]): Props = { + logger.info("Init genesis!") val stateDir: File = UtxoState.getStateDir(settings) - stateDir.mkdir() stateDir.listFiles.foreach(_.delete()) + stateDir.mkdir() val rootsDir: File = UtxoState.getRootsDir(settings) - rootsDir.mkdir() rootsDir.listFiles.foreach(_.delete()) + rootsDir.mkdir() val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) Props(new NVHState(influxRef, state, settings)) } @@ -105,6 +112,7 @@ object NVHState extends StrictLogging { stateDir.mkdirs() val rootsDir: File = UtxoState.getRootsDir(settings) rootsDir.mkdir() + logger.info("init dirs") val state: UtxoState = restoreConsistentState ( UtxoState.create(stateDir, rootsDir, settings, influxRef), historyReader, @@ -112,6 +120,11 @@ object NVHState extends StrictLogging { settings ) Props(new NVHState(influxRef, state, settings)) + } match { + case a: Success[Props] => a + case err: Failure[Props] => + logger.info(s"Err: ${err.exception}") + err } } diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index b0fa32e731..030445f836 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -43,419 +43,420 @@ import scala.collection.{ mutable, IndexedSeq, Seq } import scala.concurrent.Future import scala.concurrent.duration._ import scala.util.{ Failure, Success, Try } - -class NodeViewHolder( - settings: EncryAppSettings, - ntp: NetworkTimeProvider, - influxRef: Option[ActorRef] -) extends Actor - with StrictLogging - with AutoCloseable { - - import context.dispatcher - - var nodeView: NodeView = restoreState().getOrElse(genesisState) - - var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] - - context.parent ! UpdateHistoryReader(HistoryReader(nodeView.history)) - context.parent ! BlockAndHeaderInfo(nodeView.history.getBestHeader, nodeView.history.getBestBlock) - - context.system.scheduler.schedule(1.seconds, 10.seconds) { - logger.info( - s"\n\n History best header id is: ${nodeView.history.getBestHeaderId.map(Algos.encode)}.\n " + - s"History best header height is: ${nodeView.history.getBestHeaderHeight}.\n " + - s"History best block id is: ${nodeView.history.getBestBlockId.map(Algos.encode)}.\n " + - s"History best block height is: ${nodeView.history.getBestBlockHeight}.\n " + - s"History best block header is: ${nodeView.history.getHeaderOfBestBlock.map(_.encodedId)}.\n " + - s"State height is: ${nodeView.state.height}.\n " + - s"Cache size is: ${ModifiersCache.size}.\n " - ) - logger.info( - s"Cache elements are: ${ModifiersCache.cache.keys.toList.map(key => Algos.encode(key.toArray)).mkString(",")}." - ) - } - - override def receive: Receive = { - case ValidatedModifier(modifier: PersistentModifier) => - val startTime: Long = System.currentTimeMillis() - val wrappedKey: mutable.WrappedArray.ofByte = NodeViewHolder.toKey(modifier.id) - val isInHistory: Boolean = nodeView.history.isModifierDefined(modifier.id) - val isInCache: Boolean = ModifiersCache.contains(wrappedKey) - if (isInHistory || isInCache) - logger.info( - s"Modifier ${modifier.encodedId} can't be placed into the cache cause: " + - s"contains in cache: $isInCache, contains in history: $isInHistory." - ) - else ModifiersCache.put(wrappedKey, modifier, nodeView.history) - computeApplications() - logger.debug( - s"Time of processing validated modifier with id: ${modifier.encodedId} " + - s"is: ${(System.currentTimeMillis() - startTime) / 1000}s." - ) - - case LocallyGeneratedModifier(modifier: PersistentModifier) => - val startTime: Long = System.currentTimeMillis() - logger.info(s"Got locally generated modifier ${modifier.encodedId}.") - modifier match { - case block: Block => - applyModifier(block.header, isLocallyGenerated = true) - applyModifier(block.payload, isLocallyGenerated = true) - } - logger.debug( - s"Time of processing locally generated modifier with id: ${modifier.encodedId} " + - s"is ${(System.currentTimeMillis() - startTime) / 1000}s." - ) - - case GetDataFromCurrentView(f) => - logger.info("Receive GetDataFromCurrentView on nvh") - f(CurrentView(nodeView.history, nodeView.state, nodeView.wallet)) match { - case resultFuture: Future[_] => resultFuture.pipeTo(sender) - case result => sender ! result - } - - case FastSyncFinished(state: UtxoState, wallet: EncryWallet) => - val startTime: Long = System.currentTimeMillis() - logger.info(s"Got a signal about finishing fast sync process.") - nodeView.state.tree.avlStorage.close() - nodeView.wallet.close() - FileUtils.deleteDirectory(new File(s"${settings.directory}/tmpDirState")) - FileUtils.deleteDirectory(new File(s"${settings.directory}/keysTmp")) - FileUtils.deleteDirectory(new File(s"${settings.directory}/walletTmp")) - val newHistory = new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { - override val settings: EncryAppSettings = settings //todo incorrect - override var isFullChainSynced: Boolean = settings.node.offlineGeneration - override val timeProvider: NetworkTimeProvider = ntp - override val historyStorage: HistoryStorage = nodeView.history.historyStorage - } - newHistory.fastSyncInProgress.fastSyncVal = false - newHistory.blockDownloadProcessor.updateMinimalBlockHeightVar( - nodeView.history.blockDownloadProcessor.minimalBlockHeight - ) - newHistory.isHeadersChainSyncedVar = true - updateNodeView( - updatedHistory = newHistory.some, - updatedState = state.some, - updatedVault = wallet.some - ) - context.parent ! FastSyncDone - logger.debug(s"Time of processing FastSyncDone message is: ${(System.currentTimeMillis() - startTime) / 1000}s.") - - case RemoveRedundantManifestIds => potentialManifestIds = List.empty - - case CreateAccountManagerFromSeed(seed) => - val newAccount: Either[String, EncryWallet] = - nodeView.wallet.addAccount(seed, settings.wallet.map(_.password).get, nodeView.state) - updateNodeView(updatedVault = newAccount.toOption) - sender() ! newAccount - - } - - //todo refactor loop - def computeApplications(): Unit = { - val modifiers: List[PersistentModifier] = ModifiersCache.popCandidate(nodeView.history) - if (modifiers.nonEmpty) { - logger.info(s"Got new modifiers in compute application function: ${modifiers.map(_.encodedId)}.") - modifiers.foreach(applyModifier(_)) - computeApplications() - } else () - } - - def updateNodeView( - updatedHistory: Option[History] = none, - updatedState: Option[UtxoState] = none, - updatedVault: Option[EncryWallet] = none - ): Unit = { - val newNodeView: NodeView = NodeView( - updatedHistory.getOrElse(nodeView.history), - updatedState.getOrElse(nodeView.state), - updatedVault.getOrElse(nodeView.wallet) - ) - if (updatedHistory.nonEmpty) context.parent ! UpdateHistoryReader(HistoryReader(newNodeView.history)) - nodeView = newNodeView - } - - def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = none): Unit = - pi.toDownload.foreach { - case (tid: ModifierTypeId, id: ModifierId) => - logger.info( - s"Node view holder created download request for modifier ${Algos.encode(id)} of type $tid. " + - s"Previous modifier is ${previousModifier.map(Algos.encode)}." - ) - if (tid != Payload.modifierTypeId || (nodeView.history.isFullChainSynced && tid == Payload.modifierTypeId)) - context.parent ! RequestFromLocal(none, tid, List(id)) - else - logger.info( - s"Ignore sending download request for modifier ${Algos.encode(id)} because full chain is not synced." - ) - } - - def trimChainSuffix( - suffix: IndexedSeq[PersistentModifier], - rollbackPoint: ModifierId - ): IndexedSeq[PersistentModifier] = { - val idx: Int = suffix.indexWhere(_.id.sameElements(rollbackPoint)) - if (idx == -1) IndexedSeq.empty else suffix.drop(idx) - } - - @scala.annotation.tailrec - private def updateState( - history: History, - state: UtxoState, - progressInfo: ProgressInfo, - suffixApplied: IndexedSeq[PersistentModifier], - isLocallyGenerated: Boolean = false - ): (History, UtxoState, Seq[PersistentModifier]) = { - logger.info(s"Starting updating state in updateState function!") - val branchingPointOpt: Option[VersionTag] = progressInfo.branchPoint.map(VersionTag !@@ _) - val (stateToApplyTry: Try[UtxoState], suffixTrimmed: IndexedSeq[PersistentModifier] @unchecked) = - if (progressInfo.chainSwitchingNeeded) { - branchingPointOpt.map { branchPoint: VersionTag => - if (!state.version.sameElements(branchPoint)) { - val branchPointHeight: Int = history.getHeaderById(ModifierId !@@ branchPoint).get.height - val additionalBlocks: List[Block] = - (state.safePointHeight + 1 to branchPointHeight).foldLeft(List.empty[Block]) { - case (blocks: List[Block], height: Int) => - val headerAtHeight: Header = history.getBestHeaderAtHeight(height).get - val blockAtHeight: Block = history.getBlockByHeader(headerAtHeight).get - blocks :+ blockAtHeight - } - context.parent ! DisableMining - state.rollbackTo(branchPoint, additionalBlocks) -> trimChainSuffix(suffixApplied, - ModifierId !@@ branchPoint) - } else Success(state) -> IndexedSeq.empty - }.getOrElse(Failure(new Exception("Trying to rollback when branchPoint is empty."))) - } else Success(state) -> suffixApplied - stateToApplyTry match { - case Success(stateToApply: UtxoState) => - context.parent ! RollbackSucceed(branchingPointOpt) - if (settings.node.mining && nodeView.history.isFullChainSynced && progressInfo.chainSwitchingNeeded) - context.parent ! EnableMining - val u0: UpdateInformation = UpdateInformation(history, stateToApply, none, none, suffixTrimmed) - val uf: UpdateInformation = progressInfo.toApply.foldLeft(u0) { - case (u: UpdateInformation, modToApply: PersistentModifier) => - val saveRootNodesFlag: Boolean = - (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2 - if (u.failedMod.isEmpty) u.state.applyModifier(modToApply, saveRootNodesFlag) match { - case Right(stateAfterApply) => - modToApply match { - case b: Block if history.isFullChainSynced => context.parent ! TransactionsInBlock(b.payload.txs.size) - case _ => - } - val newHis: History = history.reportModifierIsValid(modToApply) - context.parent ! BlockAndHeaderInfo(newHis.getBestHeader, newHis.getBestBlock) - modToApply match { - case header: Header => - val requiredHeight: Int = header.height - settings.constants.MaxRollbackDepth - if (requiredHeight % settings.constants.SnapshotCreationHeight == 0) { - newHis.lastAvailableManifestHeight = requiredHeight - logger.info(s"heightOfLastAvailablePayloadForRequest -> ${newHis.lastAvailableManifestHeight}") - } - case _ => - } - newHis.getHeaderOfBestBlock.foreach { header: Header => - val potentialManifestId: Array[Byte] = Algos.hash(stateAfterApply.tree.rootHash ++ header.id) - val isManifestExists: Boolean = potentialManifestIds.exists(_.sameElements(potentialManifestId)) - val isCorrectCreationHeight: Boolean = - header.height % settings.constants.SnapshotCreationHeight == 0 - val isGenesisHeader: Boolean = header.height == settings.constants.GenesisHeight - if (settings.snapshotSettings.enableSnapshotCreation && newHis.isFullChainSynced && - !isManifestExists && isCorrectCreationHeight && !isGenesisHeader) { - val startTime = System.currentTimeMillis() - logger.info(s"Start chunks creation for new snapshot") - import encry.view.state.avlTree.utils.implicits.Instances._ - val chunks: List[SnapshotChunk] = - AvlTree.getChunks( - stateAfterApply.tree.rootNode, - currentChunkHeight = settings.snapshotSettings.chunkDepth, - stateAfterApply.tree.avlStorage - ) - context.parent ! TreeChunks(chunks, potentialManifestId) - potentialManifestIds = ManifestId @@ potentialManifestId :: potentialManifestIds - logger.info( - s"State tree successfully processed for snapshot. " + - s"Processing time is: ${(System.currentTimeMillis() - startTime) / 1000}s." - ) - } - } - if (settings.node.mining && progressInfo.chainSwitchingNeeded) - context.parent ! StartMining - UpdateInformation(newHis, stateAfterApply, none, none, u.suffix :+ modToApply) - case Left(e: List[ModifierApplyError]) => - logger.info(s"Application to state failed cause $e") - val (newHis: History, newProgressInfo: ProgressInfo) = - history.reportModifierIsInvalid(modToApply) - context.parent ! SemanticallyFailedModification(modToApply, e) - UpdateInformation(newHis, u.state, modToApply.some, newProgressInfo.some, u.suffix) - } else u - } - uf.failedMod match { - case Some(_) => - uf.history.updateIdsForSyncInfo() - updateState(uf.history, uf.state, uf.alternativeProgressInfo.get, uf.suffix, isLocallyGenerated) - case None => (uf.history, uf.state, uf.suffix) - } - case Failure(e) => - context.parent ! RollbackFailed(branchingPointOpt) - EncryApp.forceStopApplication(500, s"Rollback failed: $e") - } - } - - def applyModifier(modifier: PersistentModifier, isLocallyGenerated: Boolean = false): Unit = - if (!nodeView.history.isModifierDefined(modifier.id)) { - logger.debug( - s"Start modifier ${modifier.encodedId} application of type ${modifier.modifierTypeId} to the history." - ) - val startApplicationToTheHistory: Long = System.currentTimeMillis() - context.parent ! StartApplyingModifier(modifier.id, modifier.modifierTypeId, System.currentTimeMillis()) - nodeView.history.append(modifier) match { - case Right((historyBeforeStUpdate, progressInfo)) => - if (progressInfo.toApply.nonEmpty) { - val startPoint: Long = System.currentTimeMillis() - logger.info(s"Progress info is non empty. To apply is: ${progressInfo.toApply.map(_.encodedId)}.") - val (newHistory: History, newState: UtxoState, blocksApplied: Seq[PersistentModifier]) = - updateState(historyBeforeStUpdate, nodeView.state, progressInfo, IndexedSeq(), isLocallyGenerated) - if (newHistory.isHeadersChainSynced) context.parent ! HeaderChainIsSynced - context.parent ! StateUpdating(System.currentTimeMillis() - startPoint) - sendUpdatedInfoToMemoryPool(progressInfo.toRemove, progressInfo.toApply) - if (progressInfo.chainSwitchingNeeded) - nodeView.wallet.rollback(VersionTag !@@ progressInfo.branchPoint.get).get - blocksApplied.foreach(nodeView.wallet.scanPersistent) - logger.debug(s"Persistent modifier ${modifier.encodedId} was applied successfully.") - newHistory.getBestHeader.foreach(context.parent ! BestHeaderInChain(_)) - if (newHistory.isFullChainSynced) { - logger.info(s"BlockChain is synced on nvh at the height ${newHistory.getBestHeaderHeight}.") - ModifiersCache.setChainSynced() - context.parent ! FullBlockChainIsSynced - } - updateNodeView(newHistory.some, newState.some, nodeView.wallet.some) - } else { - updateNodeView(updatedHistory = historyBeforeStUpdate.some) - } - case Left(e: Throwable) => - logger.debug(s"Can't apply modifier ${modifier.encodedId}, contents: $modifier to history cause $e.") - context.parent ! SyntacticallyFailedModification(modifier, List(HistoryApplyError(e.getMessage))) - } - } - else logger.info(s"Trying to apply modifier ${modifier.encodedId} that's already in history.") - - def sendUpdatedInfoToMemoryPool(toRemove: Seq[PersistentModifier], toApply: Seq[PersistentModifier]): Unit = { - val toRemoveTxs: IndexedSeq[Transaction] = toRemove - .flatMap(extractTransactions) - .toIndexedSeq - val toApplyTxs: Vector[String] = toApply - .flatMap(extractTransactions) - .toVector - .map(_.encodedId) - val resultedTxs: IndexedSeq[Transaction] = toRemoveTxs.filterNot(tx => toApplyTxs.contains(tx.encodedId)) - if (resultedTxs.nonEmpty) context.parent ! RolledBackTransactions(resultedTxs) - } - - def extractTransactions(mod: PersistentModifier): Seq[Transaction] = mod match { - case b: Block => b.payload.txs - case p: Payload => p.txs - case _ => Seq.empty[Transaction] - } - - def genesisState: NodeView = { - val stateDir: File = UtxoState.getStateDir(settings) - stateDir.mkdir() - val rootsDir: File = UtxoState.getRootsDir(settings) - rootsDir.mkdir() - assert(stateDir.listFiles().isEmpty, s"Genesis directory $stateDir should always be empty.") - val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) - val history: History = History.readOrGenerate(settings, ntp) - val wallet: EncryWallet = - EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) - NodeView(history, state, wallet) - } - - def restoreState(influxRef: Option[ActorRef] = none): Option[NodeView] = - if (History.getHistoryIndexDir(settings).listFiles.nonEmpty) - try { - val stateDir: File = UtxoState.getStateDir(settings) - stateDir.mkdirs() - val rootsDir: File = UtxoState.getRootsDir(settings) - rootsDir.mkdir() - val history: History = History.readOrGenerate(settings, ntp) - val wallet: EncryWallet = - EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) - val state: UtxoState = restoreConsistentState( - UtxoState.create(stateDir, rootsDir, settings, influxRef), - history, - influxRef - ) - history.updateIdsForSyncInfo() - logger.info(s"History best block height: ${history.getBestBlockHeight}") - logger.info(s"History best header height: ${history.getBestHeaderHeight}") - NodeView(history, state, wallet).some - } catch { - case ex: Throwable => - logger.info(s"${ex.getMessage} during state restore. Recover from Modifiers holder!") - new File(settings.directory).listFiles.foreach(dir => FileUtils.cleanDirectory(dir)) - genesisState.some - } else { - none - } - - def getRecreatedState( - version: Option[VersionTag] = none, - digest: Option[ADDigest] = none, - influxRef: Option[ActorRef] - ): UtxoState = { - val dir: File = UtxoState.getStateDir(settings) - dir.mkdirs() - dir.listFiles.foreach(_.delete()) - val stateDir: File = UtxoState.getStateDir(settings) - stateDir.mkdirs() - val rootsDir: File = UtxoState.getRootsDir(settings) - rootsDir.mkdir() - UtxoState.create(stateDir, rootsDir, settings, influxRef) - } - - def restoreConsistentState( - stateIn: UtxoState, - history: History, - influxRefActor: Option[ActorRef] - ): UtxoState = - (stateIn.version, history.getBestBlock, stateIn, stateIn.safePointHeight) match { - case (stateId, None, _, _) if stateId sameElements Array.emptyByteArray => - logger.info(s"State and history are both empty on startup") - stateIn - case (_, None, _, _) => - logger.info( - s"State and history are inconsistent." + - s" History is empty on startup, rollback state to genesis." - ) - getRecreatedState(influxRef = influxRefActor) - case (_, Some(historyBestBlock), state: UtxoState, safePointHeight) => - val headerAtSafePointHeight = history.getBestHeaderAtHeight(safePointHeight) - val (rollbackId, newChain) = history.getChainToHeader(headerAtSafePointHeight, historyBestBlock.header) - logger.info( - s"State and history are inconsistent." + - s" Going to rollback to ${rollbackId.map(Algos.encode)} and " + - s"apply ${newChain.length} modifiers. State safe point: ${safePointHeight}. ${newChain.headers.head.height}. ${newChain.headers.last.height}" - ) - val additionalBlocks = - (state.safePointHeight + 1 to historyBestBlock.header.height).foldLeft(List.empty[Block]) { - case (blocks, height) => - val headerAtHeight = history.getBestHeaderAtHeight(height).get - val blockAtHeight = history.getBlockByHeader(headerAtHeight).get - blocks :+ blockAtHeight - } - logger.info(s"Qty of additional blocks: ${additionalBlocks.length}") - rollbackId - .map(_ => state.restore(additionalBlocks).get) - .getOrElse(getRecreatedState(influxRef = influxRefActor)) - } - - override def close(): Unit = { - nodeView.history.close() - nodeView.state.close() - nodeView.wallet.close() - } -} - +// +//class NodeViewHolder( +// settings: EncryAppSettings, +// ntp: NetworkTimeProvider, +// influxRef: Option[ActorRef] +//) extends Actor +// with StrictLogging +// with AutoCloseable { +// +// import context.dispatcher +// +// var nodeView: NodeView = restoreState().getOrElse(genesisState) +// +// var potentialManifestIds: List[ManifestId] = List.empty[ManifestId] +// +// context.parent ! UpdateHistoryReader(HistoryReader(nodeView.history)) +// context.parent ! BlockAndHeaderInfo(nodeView.history.getBestHeader, nodeView.history.getBestBlock) +// +// context.system.scheduler.schedule(1.seconds, 10.seconds) { +// logger.info( +// s"\n\n History best header id is: ${nodeView.history.getBestHeaderId.map(Algos.encode)}.\n " + +// s"History best header height is: ${nodeView.history.getBestHeaderHeight}.\n " + +// s"History best block id is: ${nodeView.history.getBestBlockId.map(Algos.encode)}.\n " + +// s"History best block height is: ${nodeView.history.getBestBlockHeight}.\n " + +// s"History best block header is: ${nodeView.history.getHeaderOfBestBlock.map(_.encodedId)}.\n " + +// s"State height is: ${nodeView.state.height}.\n " + +// s"Cache size is: ${ModifiersCache.size}.\n " +// ) +// logger.info( +// s"Cache elements are: ${ModifiersCache.cache.keys.toList.map(key => Algos.encode(key.toArray)).mkString(",")}." +// ) +// } +// +// override def receive: Receive = { +// case ValidatedModifier(modifier: PersistentModifier) => +// val startTime: Long = System.currentTimeMillis() +// val wrappedKey: mutable.WrappedArray.ofByte = NodeViewHolder.toKey(modifier.id) +// val isInHistory: Boolean = nodeView.history.isModifierDefined(modifier.id) +// val isInCache: Boolean = ModifiersCache.contains(wrappedKey) +// if (isInHistory || isInCache) +// logger.info( +// s"Modifier ${modifier.encodedId} can't be placed into the cache cause: " + +// s"contains in cache: $isInCache, contains in history: $isInHistory." +// ) +// else ModifiersCache.put(wrappedKey, modifier, nodeView.history) +// computeApplications() +// logger.debug( +// s"Time of processing validated modifier with id: ${modifier.encodedId} " + +// s"is: ${(System.currentTimeMillis() - startTime) / 1000}s." +// ) +// +// case LocallyGeneratedModifier(modifier: PersistentModifier) => +// val startTime: Long = System.currentTimeMillis() +// logger.info(s"Got locally generated modifier ${modifier.encodedId}.") +// modifier match { +// case block: Block => +// applyModifier(block.header, isLocallyGenerated = true) +// applyModifier(block.payload, isLocallyGenerated = true) +// } +// logger.debug( +// s"Time of processing locally generated modifier with id: ${modifier.encodedId} " + +// s"is ${(System.currentTimeMillis() - startTime) / 1000}s." +// ) +// +// case GetDataFromCurrentView(f) => +// logger.info("Receive GetDataFromCurrentView on nvh") +// f(CurrentView(nodeView.history, nodeView.state, nodeView.wallet)) match { +// case resultFuture: Future[_] => resultFuture.pipeTo(sender) +// case result => sender ! result +// } +// +// case FastSyncFinished(state: UtxoState, wallet: EncryWallet) => +// val startTime: Long = System.currentTimeMillis() +// logger.info(s"Got a signal about finishing fast sync process.") +// nodeView.state.tree.avlStorage.close() +// nodeView.wallet.close() +// FileUtils.deleteDirectory(new File(s"${settings.directory}/tmpDirState")) +// FileUtils.deleteDirectory(new File(s"${settings.directory}/keysTmp")) +// FileUtils.deleteDirectory(new File(s"${settings.directory}/walletTmp")) +// val newHistory = new History with HistoryHeadersProcessor with HistoryPayloadsProcessor { +// override val settings: EncryAppSettings = settings //todo incorrect +// override var isFullChainSynced: Boolean = settings.node.offlineGeneration +// override val timeProvider: NetworkTimeProvider = ntp +// override val historyStorage: HistoryStorage = nodeView.history.historyStorage +// } +// newHistory.fastSyncInProgress.fastSyncVal = false +// newHistory.blockDownloadProcessor.updateMinimalBlockHeightVar( +// nodeView.history.blockDownloadProcessor.minimalBlockHeight +// ) +// newHistory.isHeadersChainSyncedVar = true +// updateNodeView( +// updatedHistory = newHistory.some, +// updatedState = state.some, +// updatedVault = wallet.some +// ) +// context.parent ! FastSyncDone +// logger.debug(s"Time of processing FastSyncDone message is: ${(System.currentTimeMillis() - startTime) / 1000}s.") +// +// case RemoveRedundantManifestIds => potentialManifestIds = List.empty +// +// case CreateAccountManagerFromSeed(seed) => +// val newAccount: Either[String, EncryWallet] = +// nodeView.wallet.addAccount(seed, settings.wallet.map(_.password).get, nodeView.state) +// updateNodeView(updatedVault = newAccount.toOption) +// sender() ! newAccount +// +// } +// +// //todo refactor loop +// def computeApplications(): Unit = { +// val modifiers: List[PersistentModifier] = ModifiersCache.popCandidate(nodeView.history) +// if (modifiers.nonEmpty) { +// logger.info(s"Got new modifiers in compute application function: ${modifiers.map(_.encodedId)}.") +// modifiers.foreach(applyModifier(_)) +// computeApplications() +// } else () +// } +// +// def updateNodeView( +// updatedHistory: Option[History] = none, +// updatedState: Option[UtxoState] = none, +// updatedVault: Option[EncryWallet] = none +// ): Unit = { +// val newNodeView: NodeView = NodeView( +// updatedHistory.getOrElse(nodeView.history), +// updatedState.getOrElse(nodeView.state), +// updatedVault.getOrElse(nodeView.wallet) +// ) +// if (updatedHistory.nonEmpty) context.parent ! UpdateHistoryReader(HistoryReader(newNodeView.history)) +// nodeView = newNodeView +// } +// +// def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = none): Unit = +// pi.toDownload.foreach { +// case (tid: ModifierTypeId, id: ModifierId) => +// logger.info( +// s"Node view holder created download request for modifier ${Algos.encode(id)} of type $tid. " + +// s"Previous modifier is ${previousModifier.map(Algos.encode)}." +// ) +// if (tid != Payload.modifierTypeId || (nodeView.history.isFullChainSynced && tid == Payload.modifierTypeId)) +// context.parent ! RequestFromLocal(none, tid, List(id)) +// else +// logger.info( +// s"Ignore sending download request for modifier ${Algos.encode(id)} because full chain is not synced." +// ) +// } +// +// def trimChainSuffix( +// suffix: IndexedSeq[PersistentModifier], +// rollbackPoint: ModifierId +// ): IndexedSeq[PersistentModifier] = { +// val idx: Int = suffix.indexWhere(_.id.sameElements(rollbackPoint)) +// if (idx == -1) IndexedSeq.empty else suffix.drop(idx) +// } +// +// @scala.annotation.tailrec +// private def updateState( +// history: History, +// state: UtxoState, +// progressInfo: ProgressInfo, +// suffixApplied: IndexedSeq[PersistentModifier], +// isLocallyGenerated: Boolean = false +// ): (History, UtxoState, Seq[PersistentModifier]) = { +// logger.info(s"Starting updating state in updateState function!") +// val branchingPointOpt: Option[VersionTag] = progressInfo.branchPoint.map(VersionTag !@@ _) +// val (stateToApplyTry: Try[UtxoState], suffixTrimmed: IndexedSeq[PersistentModifier] @unchecked) = +// if (progressInfo.chainSwitchingNeeded) { +// branchingPointOpt.map { branchPoint: VersionTag => +// if (!state.version.sameElements(branchPoint)) { +// val branchPointHeight: Int = history.getHeaderById(ModifierId !@@ branchPoint).get.height +// val additionalBlocks: List[Block] = +// (state.safePointHeight + 1 to branchPointHeight).foldLeft(List.empty[Block]) { +// case (blocks: List[Block], height: Int) => +// val headerAtHeight: Header = history.getBestHeaderAtHeight(height).get +// val blockAtHeight: Block = history.getBlockByHeader(headerAtHeight).get +// blocks :+ blockAtHeight +// } +// context.parent ! DisableMining +// state.rollbackTo(branchPoint, additionalBlocks) -> trimChainSuffix(suffixApplied, +// ModifierId !@@ branchPoint) +// } else Success(state) -> IndexedSeq.empty +// }.getOrElse(Failure(new Exception("Trying to rollback when branchPoint is empty."))) +// } else Success(state) -> suffixApplied +// stateToApplyTry match { +// case Success(stateToApply: UtxoState) => +// context.parent ! RollbackSucceed(branchingPointOpt) +// if (settings.node.mining && nodeView.history.isFullChainSynced && progressInfo.chainSwitchingNeeded) +// context.parent ! EnableMining +// val u0: UpdateInformation = UpdateInformation(history, stateToApply, none, none, suffixTrimmed) +// val uf: UpdateInformation = progressInfo.toApply.foldLeft(u0) { +// case (u: UpdateInformation, modToApply: PersistentModifier) => +// val saveRootNodesFlag: Boolean = +// (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2 +// if (u.failedMod.isEmpty) u.state.applyModifier(modToApply, saveRootNodesFlag) match { +// case Right(stateAfterApply) => +// modToApply match { +// case b: Block if history.isFullChainSynced => context.parent ! TransactionsInBlock(b.payload.txs.size) +// case _ => +// } +// val newHis: History = history.reportModifierIsValid(modToApply) +// context.parent ! BlockAndHeaderInfo(newHis.getBestHeader, newHis.getBestBlock) +// modToApply match { +// case header: Header => +// val requiredHeight: Int = header.height - settings.constants.MaxRollbackDepth +// if (requiredHeight % settings.constants.SnapshotCreationHeight == 0) { +// newHis.lastAvailableManifestHeight = requiredHeight +// logger.info(s"heightOfLastAvailablePayloadForRequest -> ${newHis.lastAvailableManifestHeight}") +// } +// case _ => +// } +// newHis.getHeaderOfBestBlock.foreach { header: Header => +// val potentialManifestId: Array[Byte] = Algos.hash(stateAfterApply.tree.rootHash ++ header.id) +// val isManifestExists: Boolean = potentialManifestIds.exists(_.sameElements(potentialManifestId)) +// val isCorrectCreationHeight: Boolean = +// header.height % settings.constants.SnapshotCreationHeight == 0 +// val isGenesisHeader: Boolean = header.height == settings.constants.GenesisHeight +// if (settings.snapshotSettings.enableSnapshotCreation && newHis.isFullChainSynced && +// !isManifestExists && isCorrectCreationHeight && !isGenesisHeader) { +// val startTime = System.currentTimeMillis() +// logger.info(s"Start chunks creation for new snapshot") +// import encry.view.state.avlTree.utils.implicits.Instances._ +// val chunks: List[SnapshotChunk] = +// AvlTree.getChunks( +// stateAfterApply.tree.rootNode, +// currentChunkHeight = settings.snapshotSettings.chunkDepth, +// stateAfterApply.tree.avlStorage +// ) +// context.parent ! TreeChunks(chunks, potentialManifestId) +// potentialManifestIds = ManifestId @@ potentialManifestId :: potentialManifestIds +// logger.info( +// s"State tree successfully processed for snapshot. " + +// s"Processing time is: ${(System.currentTimeMillis() - startTime) / 1000}s." +// ) +// } +// } +// if (settings.node.mining && progressInfo.chainSwitchingNeeded) +// context.parent ! StartMining +// UpdateInformation(newHis, stateAfterApply, none, none, u.suffix :+ modToApply) +// case Left(e: List[ModifierApplyError]) => +// logger.info(s"Application to state failed cause $e") +// val (newHis: History, newProgressInfo: ProgressInfo) = +// history.reportModifierIsInvalid(modToApply) +// context.parent ! SemanticallyFailedModification(modToApply, e) +// UpdateInformation(newHis, u.state, modToApply.some, newProgressInfo.some, u.suffix) +// } else u +// } +// uf.failedMod match { +// case Some(_) => +// uf.history.updateIdsForSyncInfo() +// updateState(uf.history, uf.state, uf.alternativeProgressInfo.get, uf.suffix, isLocallyGenerated) +// case None => (uf.history, uf.state, uf.suffix) +// } +// case Failure(e) => +// context.parent ! RollbackFailed(branchingPointOpt) +// EncryApp.forceStopApplication(500, s"Rollback failed: $e") +// } +// } +// +// def applyModifier(modifier: PersistentModifier, isLocallyGenerated: Boolean = false): Unit = +// if (!nodeView.history.isModifierDefined(modifier.id)) { +// logger.debug( +// s"Start modifier ${modifier.encodedId} application of type ${modifier.modifierTypeId} to the history." +// ) +// val startApplicationToTheHistory: Long = System.currentTimeMillis() +// context.parent ! StartApplyingModifier(modifier.id, modifier.modifierTypeId, System.currentTimeMillis()) +// nodeView.history.append(modifier) match { +// case Right((historyBeforeStUpdate, progressInfo)) => +// if (progressInfo.toApply.nonEmpty) { +// val startPoint: Long = System.currentTimeMillis() +// logger.info(s"Progress info is non empty. To apply is: ${progressInfo.toApply.map(_.encodedId)}.") +// val (newHistory: History, newState: UtxoState, blocksApplied: Seq[PersistentModifier]) = +// updateState(historyBeforeStUpdate, nodeView.state, progressInfo, IndexedSeq(), isLocallyGenerated) +// if (newHistory.isHeadersChainSynced) context.parent ! HeaderChainIsSynced +// context.parent ! StateUpdating(System.currentTimeMillis() - startPoint) +// sendUpdatedInfoToMemoryPool(progressInfo.toRemove, progressInfo.toApply) +// if (progressInfo.chainSwitchingNeeded) +// nodeView.wallet.rollback(VersionTag !@@ progressInfo.branchPoint.get).get +// blocksApplied.foreach(nodeView.wallet.scanPersistent) +// logger.debug(s"Persistent modifier ${modifier.encodedId} was applied successfully.") +// newHistory.getBestHeader.foreach(context.parent ! BestHeaderInChain(_)) +// if (newHistory.isFullChainSynced) { +// logger.info(s"BlockChain is synced on nvh at the height ${newHistory.getBestHeaderHeight}.") +// ModifiersCache.setChainSynced() +// context.parent ! FullBlockChainIsSynced +// } +// updateNodeView(newHistory.some, newState.some, nodeView.wallet.some) +// } else { +// updateNodeView(updatedHistory = historyBeforeStUpdate.some) +// } +// case Left(e: Throwable) => +// logger.debug(s"Can't apply modifier ${modifier.encodedId}, contents: $modifier to history cause $e.") +// context.parent ! SyntacticallyFailedModification(modifier, List(HistoryApplyError(e.getMessage))) +// } +// } +// else logger.info(s"Trying to apply modifier ${modifier.encodedId} that's already in history.") +// +// def sendUpdatedInfoToMemoryPool(toRemove: Seq[PersistentModifier], toApply: Seq[PersistentModifier]): Unit = { +// val toRemoveTxs: IndexedSeq[Transaction] = toRemove +// .flatMap(extractTransactions) +// .toIndexedSeq +// val toApplyTxs: Vector[String] = toApply +// .flatMap(extractTransactions) +// .toVector +// .map(_.encodedId) +// val resultedTxs: IndexedSeq[Transaction] = toRemoveTxs.filterNot(tx => toApplyTxs.contains(tx.encodedId)) +// if (resultedTxs.nonEmpty) context.parent ! RolledBackTransactions(resultedTxs) +// } +// +// def extractTransactions(mod: PersistentModifier): Seq[Transaction] = mod match { +// case b: Block => b.payload.txs +// case p: Payload => p.txs +// case _ => Seq.empty[Transaction] +// } +// +// def genesisState: NodeView = { +// val stateDir: File = UtxoState.getStateDir(settings) +// stateDir.mkdir() +// val rootsDir: File = UtxoState.getRootsDir(settings) +// rootsDir.mkdir() +// assert(stateDir.listFiles().isEmpty, s"Genesis directory $stateDir should always be empty.") +// val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) +// val history: History = History.readOrGenerate(settings, ntp) +// val wallet: EncryWallet = +// EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) +// NodeView(history, state, wallet) +// } +// +// def restoreState(influxRef: Option[ActorRef] = none): Option[NodeView] = +// if (History.getHistoryIndexDir(settings).listFiles.nonEmpty) +// try { +// val stateDir: File = UtxoState.getStateDir(settings) +// stateDir.mkdirs() +// val rootsDir: File = UtxoState.getRootsDir(settings) +// rootsDir.mkdir() +// val history: History = History.readOrGenerate(settings, ntp) +// val wallet: EncryWallet = +// EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) +// val state: UtxoState = restoreConsistentState( +// UtxoState.create(stateDir, rootsDir, settings, influxRef), +// history, +// influxRef +// ) +// history.updateIdsForSyncInfo() +// logger.info(s"History best block height: ${history.getBestBlockHeight}") +// logger.info(s"History best header height: ${history.getBestHeaderHeight}") +// NodeView(history, state, wallet).some +// } catch { +// case ex: Throwable => +// logger.info(s"${ex.getMessage} during state restore. Recover from Modifiers holder!") +// new File(settings.directory).listFiles.foreach(dir => FileUtils.cleanDirectory(dir)) +// genesisState.some +// } else { +// none +// } +// +// def getRecreatedState( +// version: Option[VersionTag] = none, +// digest: Option[ADDigest] = none, +// influxRef: Option[ActorRef] +// ): UtxoState = { +// val dir: File = UtxoState.getStateDir(settings) +// dir.mkdirs() +// dir.listFiles.foreach(_.delete()) +// val stateDir: File = UtxoState.getStateDir(settings) +// stateDir.mkdirs() +// val rootsDir: File = UtxoState.getRootsDir(settings) +// rootsDir.mkdir() +// UtxoState.create(stateDir, rootsDir, settings, influxRef) +// } +// +// def restoreConsistentState( +// stateIn: UtxoState, +// history: History, +// influxRefActor: Option[ActorRef] +// ): UtxoState = +// (stateIn.version, history.getBestBlock, stateIn, stateIn.safePointHeight) match { +// case (stateId, None, _, _) if stateId sameElements Array.emptyByteArray => +// logger.info(s"State and history are both empty on startup") +// stateIn +// case (_, None, _, _) => +// logger.info( +// s"State and history are inconsistent." + +// s" History is empty on startup, rollback state to genesis." +// ) +// getRecreatedState(influxRef = influxRefActor) +// case (_, Some(historyBestBlock), state: UtxoState, safePointHeight) => +// val headerAtSafePointHeight = history.getBestHeaderAtHeight(safePointHeight) +// val (rollbackId, newChain) = history.getChainToHeader(headerAtSafePointHeight, historyBestBlock.header) +// logger.info( +// s"State and history are inconsistent." + +// s" Going to rollback to ${rollbackId.map(Algos.encode)} and " + +// s"apply ${newChain.length} modifiers. State safe point: ${safePointHeight}. ${newChain.headers.head.height}. ${newChain.headers.last.height}" +// ) +// val additionalBlocks = +// (state.safePointHeight + 1 to historyBestBlock.header.height).foldLeft(List.empty[Block]) { +// case (blocks, height) => +// val headerAtHeight = history.getBestHeaderAtHeight(height).get +// val blockAtHeight = history.getBlockByHeader(headerAtHeight).get +// blocks :+ blockAtHeight +// } +// logger.info(s"Qty of additional blocks: ${additionalBlocks.length}") +// rollbackId +// .map(_ => state.restore(additionalBlocks).get) +// .getOrElse(getRecreatedState(influxRef = influxRefActor)) +// } +// +// override def close(): Unit = { +// logger.info("Close nvh!") +// nodeView.history.close() +// nodeView.state.close() +// nodeView.wallet.close() +// } +//} +// object NodeViewHolder { def toKey(id: ModifierId): mutable.WrappedArray.ofByte = new mutable.WrappedArray.ofByte(id) @@ -501,10 +502,4 @@ object NodeViewHolder { alternativeProgressInfo: Option[ProgressInfo], suffix: IndexedSeq[PersistentModifier] ) - - def props( - settings: EncryAppSettings, - timeProvider: NetworkTimeProvider, - influxRef: Option[ActorRef] - ): Props = Props(new NodeViewHolder(settings, timeProvider, influxRef)) } diff --git a/src/main/scala/encry/storage/RootNodesStorage.scala b/src/main/scala/encry/storage/RootNodesStorage.scala index 66528341e1..8a1705a68a 100644 --- a/src/main/scala/encry/storage/RootNodesStorage.scala +++ b/src/main/scala/encry/storage/RootNodesStorage.scala @@ -32,7 +32,7 @@ object RootNodesStorage { def apply[K: Serializer: Monoid: Hashable: Order, V: Serializer: Monoid: Hashable](storage: DB, rollbackDepth: Int, - rootsPath: File): RootNodesStorage[K, V] = new RootNodesStorage[K, V] with AutoCloseable with StrictLogging { + rootsPath: File): RootNodesStorage[K, V] = new RootNodesStorage[K, V] with StrictLogging { private def atHeightKey(height: Height): Array[Byte] = Ints.toByteArray(height) @@ -99,7 +99,10 @@ object RootNodesStorage { } } - override def close(): Unit = storage.close() + override def close(): Unit = { + storage.close() + logger.info("Close tree storage") + } } def emptyRootStorage[K: Serializer: Monoid, V: Serializer: Monoid]: RootNodesStorage[K, V] = diff --git a/src/main/scala/encry/storage/iodb/versionalIODB/IODBWrapper.scala b/src/main/scala/encry/storage/iodb/versionalIODB/IODBWrapper.scala index ccf0b72b87..8ccb0c271d 100644 --- a/src/main/scala/encry/storage/iodb/versionalIODB/IODBWrapper.scala +++ b/src/main/scala/encry/storage/iodb/versionalIODB/IODBWrapper.scala @@ -13,7 +13,7 @@ import scala.collection.mutable * Wrapper, which extends VersionalStorage trait * @param store */ -case class IODBWrapper(store: Store) extends VersionalStorage with StrictLogging { +case class IODBWrapper(store: Store) extends VersionalStorage with StrictLogging with AutoCloseable { override def get(key: StorageKey): Option[StorageValue] = store.get(ByteArrayWrapper(key)).map(StorageValue @@ _.data) diff --git a/src/main/scala/encry/storage/levelDb/versionalLevelDB/VLDBWrapper.scala b/src/main/scala/encry/storage/levelDb/versionalLevelDB/VLDBWrapper.scala index f024267fea..05047cc29d 100644 --- a/src/main/scala/encry/storage/levelDb/versionalLevelDB/VLDBWrapper.scala +++ b/src/main/scala/encry/storage/levelDb/versionalLevelDB/VLDBWrapper.scala @@ -4,7 +4,7 @@ import encry.storage.VersionalStorage import encry.storage.VersionalStorage.{StorageKey, StorageValue, StorageVersion} import encry.storage.levelDb.versionalLevelDB.VersionalLevelDBCompanion.{LevelDBVersion, VersionalLevelDbKey, VersionalLevelDbValue} -case class VLDBWrapper(vldb: VersionalLevelDB) extends VersionalStorage { +case class VLDBWrapper(vldb: VersionalLevelDB) extends VersionalStorage with AutoCloseable { override def get(key: StorageKey): Option[StorageValue] = vldb.get(VersionalLevelDbKey @@ key.untag(StorageKey)).map(StorageValue @@ _.untag(VersionalLevelDbValue)) diff --git a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala index 3a80a90dde..38350bff64 100644 --- a/src/main/scala/encry/view/history/HistoryModifiersValidator.scala +++ b/src/main/scala/encry/view/history/HistoryModifiersValidator.scala @@ -6,6 +6,7 @@ import encry.view.history.ValidationError.FatalValidationError._ import encry.view.history.ValidationError.NonFatalValidationError._ import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{Header, Payload} +import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, ModifierId} import org.encryfoundation.common.validation.ModifierSemanticValidity @@ -28,7 +29,7 @@ trait HistoryModifiersValidator extends HistoryApi { if (h.isGenesis) genesisBlockHeaderValidator(h) else getHeaderById(h.parentId) .map(p => headerValidator(h, p)) - .getOrElse(HeaderNonFatalValidationError(s"Header's ${h.encodedId} parent doesn't contain in history").asLeft[Header]) + .getOrElse(HeaderNonFatalValidationError(s"Header's ${h.encodedId} at height ${h.height}, parent (${Algos.encode(h.parentId)} at height ${h.height - 1}) doesn't contain in history").asLeft[Header]) private def validatePayload(mod: Payload): Either[ValidationError, PersistentModifier] = getHeaderById(mod.headerId) .map(header => payloadValidator(mod, header, blockDownloadProcessor.minimalBlockHeight)) diff --git a/src/main/scala/encry/view/state/UtxoState.scala b/src/main/scala/encry/view/state/UtxoState.scala index 20f2236b50..d3b7aa6734 100644 --- a/src/main/scala/encry/view/state/UtxoState.scala +++ b/src/main/scala/encry/view/state/UtxoState.scala @@ -1,6 +1,7 @@ package encry.view.state import java.io.File + import akka.actor.ActorRef import cats.data.Validated import cats.instances.list._ @@ -23,7 +24,7 @@ import encry.utils.implicits.Validation._ import encry.view.NodeViewErrors.ModifierApplyError import encry.view.NodeViewErrors.ModifierApplyError.StateModifierApplyError import encry.view.state.UtxoState.StateChange -import encry.view.state.avlTree.AvlTree +import encry.view.state.avlTree.{AvlTree, Node} import encry.view.state.avlTree.utils.implicits.Instances._ import io.iohk.iodb.LSMStore import org.encryfoundation.common.modifiers.PersistentModifier @@ -39,6 +40,7 @@ import org.encryfoundation.common.utils.constants.Constants import org.encryfoundation.common.validation.ValidationResult.Invalid import org.encryfoundation.common.validation.{MalformedModifierError, ValidationResult} import org.iq80.leveldb.Options + import scala.util.Try final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], @@ -208,7 +210,7 @@ final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], override def version: VersionTag = VersionTag !@@ tree.avlStorage.currentVersion - override def stateSafePointHeight: Unit = tree.rootNodesStorage.safePointHeight + override def stateSafePointHeight: Height = tree.rootNodesStorage.safePointHeight override def boxById(boxId: ADKey): Option[EncryBaseBox] = tree.get(StorageKey !@@ boxId) .map(bytes => StateModifierSerializer.parseBytes(bytes, boxId.head)).flatMap(_.toOption) @@ -225,6 +227,12 @@ final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], } def close(): Unit = tree.close() + + override def rootNode: Node[StorageKey, StorageValue] = tree.rootNode + + override def avlStorage: VersionalStorage = tree.avlStorage + + override def rootHash: Array[Byte] = tree.rootHash } object UtxoState extends StrictLogging { diff --git a/src/main/scala/encry/view/state/UtxoStateReader.scala b/src/main/scala/encry/view/state/UtxoStateReader.scala index cf2f2f265c..42c8c15655 100644 --- a/src/main/scala/encry/view/state/UtxoStateReader.scala +++ b/src/main/scala/encry/view/state/UtxoStateReader.scala @@ -1,8 +1,9 @@ package encry.view.state +import encry.storage.VersionalStorage import encry.storage.VersionalStorage.{StorageKey, StorageValue} import encry.utils.CoreTaggedTypes.VersionTag -import encry.view.state.avlTree.AvlTree +import encry.view.state.avlTree.{AvlTree, Node} import encry.view.state.avlTree.utils.implicits.Instances._ import org.encryfoundation.common.modifiers.state.StateModifierSerializer import org.encryfoundation.common.modifiers.state.box.EncryBaseBox @@ -15,6 +16,13 @@ trait UtxoStateReader { def version: VersionTag + def rootNode: Node[StorageKey, StorageValue] + + //todo remove + def avlStorage: VersionalStorage + + def rootHash: Array[Byte] + def stateSafePointHeight: Height def boxById(boxId: ADKey): Option[EncryBaseBox] @@ -33,6 +41,10 @@ object UtxoStateReader { override def stateSafePointHeight: Height = state.safePointHeight override def boxById(boxId: ADKey): Option[EncryBaseBox] = state.boxById(boxId) override def boxesByIds(ids: Seq[ADKey]): Seq[EncryBaseBox] = state.boxesByIds(ids) - override def typedBoxById[B <: EncryBaseBox](boxId: ADKey): Option[EncryBaseBox] = state.typedBoxById() + override def typedBoxById[B <: EncryBaseBox](boxId: ADKey): Option[EncryBaseBox] = state.typedBoxById[B](boxId) + override def rootNode: Node[StorageKey, StorageValue] = state.rootNode + override def avlStorage: VersionalStorage = state.avlStorage + override def rootHash: Array[Byte] = state.rootHash + override def safePointHeight: Height = state.safePointHeight } } \ No newline at end of file diff --git a/src/main/scala/encry/view/wallet/EncryWallet.scala b/src/main/scala/encry/view/wallet/EncryWallet.scala index 604a3a187e..adcdbe5229 100644 --- a/src/main/scala/encry/view/wallet/EncryWallet.scala +++ b/src/main/scala/encry/view/wallet/EncryWallet.scala @@ -74,10 +74,10 @@ case class EncryWallet(walletStorage: WalletVersionalLevelDB, accountManagers: S } def scanWalletFromUtxo(state: UtxoStateReader, props: Set[EncryProposition]): EncryWallet = { - val bxsToAdd: Seq[EncryBaseBox] = EncryWallet.scanTree(state.tree.rootNode, state.tree.avlStorage, props) + val bxsToAdd: Seq[EncryBaseBox] = EncryWallet.scanTree(state.rootNode, state.avlStorage, props) if (bxsToAdd.nonEmpty) walletStorage.updateWallet( - ModifierId !@@ state.tree.avlStorage.currentVersion, + ModifierId !@@ state.avlStorage.currentVersion, bxsToAdd, List.empty, settings.constants.IntrinsicTokenId diff --git a/src/test/scala/encry/nvg/NVHStateTest.scala b/src/test/scala/encry/nvg/NVHStateTest.scala index 8f09d88cfb..995fe24c3d 100644 --- a/src/test/scala/encry/nvg/NVHStateTest.scala +++ b/src/test/scala/encry/nvg/NVHStateTest.scala @@ -1,103 +1,103 @@ -package encry.nvg - -import akka.actor.ActorSystem -import akka.testkit.{TestActorRef, TestKit, TestProbe} -import com.typesafe.scalalogging.StrictLogging -import encry.consensus.HistoryConsensus -import encry.consensus.HistoryConsensus.Equal -import encry.modifiers.InstanceFactory -import encry.modifiers.history.HeaderChain -import encry.nvg.NVHState.StateAction.ApplyModifier -import encry.utils.FileHelper -import encry.view.history.HistoryReader -import org.encryfoundation.common.modifiers.history.{Block, Header} -import org.encryfoundation.common.network.SyncInfo -import org.encryfoundation.common.utils.TaggedTypes.ModifierId -import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} - -class NVHStateTest - extends TestKit(ActorSystem("Tested-Akka-System")) - with WordSpecLike - with Matchers - with BeforeAndAfterAll - with InstanceFactory - with OneInstancePerTest - with StrictLogging { - - "Nvh state" should { - "correctly init genesis state" in { - val stateDir = FileHelper.getRandomTempDir - val actor = TestActorRef[NVHState](NVHState.genesisProps(settings.copy(directory = stateDir.getAbsolutePath), None)) - } - "correctly recover state" in { - - val ((_, _, blocks), _) = PipelinesTests.genForOn(3) - - val historyReader = new HistoryReader { - - override def getBestHeaderHeight: Int = blocks.last.header.height - - override def getBestBlockHeight: Int = blocks.last.header.height - - override def getBestHeaderAtHeight(h: Int): Option[Header] = Some(blocks(h).header) - - override def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty - - override def compare(si: SyncInfo): HistoryConsensus.HistoryComparisonResult = Equal - - override def getHeaderById(id: ModifierId): Option[Header] = blocks.find(_.id sameElements id).map(_.header) - - override def getChainToHeader(fromHeaderOpt: Option[Header], - toHeader: Header): (Option[ModifierId], HeaderChain) = - fromHeaderOpt.map(header => - Some(header.id) -> HeaderChain(blocks.dropWhile(block => !(block.id sameElements header.id)).map(_.header)) - ).getOrElse(None -> HeaderChain.empty) - - override def getBlockByHeaderId(id: ModifierId): Option[Block] = blocks.find(_.id sameElements id) - - override def getBlockByHeader(header: Header): Option[Block] = blocks.find(_.id sameElements header.id) - - override var isFullChainSynced: Boolean = true - - override def isModifierDefined(id: ModifierId): Boolean = blocks.exists(_.id sameElements id) - - override def headerIdsAtHeight(height: Int): List[ModifierId] = List(blocks(height).id) - - override def modifierBytesById(id: ModifierId): Option[Array[Byte]] = blocks.find(_.id sameElements id).map(_.bytes) - - override def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty - - override def lastHeaders(count: Int): HeaderChain = HeaderChain.empty - - override def syncInfo: SyncInfo = SyncInfo(List.empty) - - override def isFastSyncInProcess: Boolean = false - - override def getBestHeader: Option[Header] = Some(blocks.last.header) - - override def getBestBlock: Option[Block] = Some(blocks.last) - - } - - val stateDir = FileHelper.getRandomTempDir - val emptyHistoryReader = HistoryReader.empty - val parent = TestProbe() - val actor = TestActorRef[NVHState]( - NVHState.genesisProps(settings.copy(directory = stateDir.getAbsolutePath), None), parent.ref - ) - blocks.sortBy(_.header.height).foreach { block => - actor ! ApplyModifier(block.header, saveRootNodesFlag = true, isFullChainSynced = true) - actor ! ApplyModifier(block, saveRootNodesFlag = true, isFullChainSynced = true) - } - Thread.sleep(5000) - actor.stop() - val recreatedActor = TestActorRef[NVHState](NVHState.restoreConsistentStateProps( - settings.copy(directory = stateDir.getAbsolutePath), - historyReader, - Some(TestProbe().ref) - ).get) - recreatedActor.underlyingActor.state.height shouldEqual blocks.last.header.height - recreatedActor.underlyingActor.state.tree.avlStorage.currentVersion shouldEqual blocks.last.id - } - } -} +//package encry.nvg +// +//import akka.actor.ActorSystem +//import akka.testkit.{TestActorRef, TestKit, TestProbe} +//import com.typesafe.scalalogging.StrictLogging +//import encry.consensus.HistoryConsensus +//import encry.consensus.HistoryConsensus.Equal +//import encry.modifiers.InstanceFactory +//import encry.modifiers.history.HeaderChain +//import encry.nvg.NVHState.StateAction.ApplyModifier +//import encry.utils.FileHelper +//import encry.view.history.HistoryReader +//import org.encryfoundation.common.modifiers.history.{Block, Header} +//import org.encryfoundation.common.network.SyncInfo +//import org.encryfoundation.common.utils.TaggedTypes.ModifierId +//import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +// +//class NVHStateTest +// extends TestKit(ActorSystem("Tested-Akka-System")) +// with WordSpecLike +// with Matchers +// with BeforeAndAfterAll +// with InstanceFactory +// with OneInstancePerTest +// with StrictLogging { +// +// "Nvh state" should { +// "correctly init genesis state" in { +// val stateDir = FileHelper.getRandomTempDir +// val actor = TestActorRef[NVHState](NVHState.genesisProps(settings.copy(directory = stateDir.getAbsolutePath), None)) +// } +// "correctly recover state" in { +// +// val ((_, _, blocks), _) = PipelinesTests.genForOn(3) +// +// val historyReader = new HistoryReader { +// +// override def getBestHeaderHeight: Int = blocks.last.header.height +// +// override def getBestBlockHeight: Int = blocks.last.header.height +// +// override def getBestHeaderAtHeight(h: Int): Option[Header] = Some(blocks(h).header) +// +// override def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = Seq.empty +// +// override def compare(si: SyncInfo): HistoryConsensus.HistoryComparisonResult = Equal +// +// override def getHeaderById(id: ModifierId): Option[Header] = blocks.find(_.id sameElements id).map(_.header) +// +// override def getChainToHeader(fromHeaderOpt: Option[Header], +// toHeader: Header): (Option[ModifierId], HeaderChain) = +// fromHeaderOpt.map(header => +// Some(header.id) -> HeaderChain(blocks.dropWhile(block => !(block.id sameElements header.id)).map(_.header)) +// ).getOrElse(None -> HeaderChain.empty) +// +// override def getBlockByHeaderId(id: ModifierId): Option[Block] = blocks.find(_.id sameElements id) +// +// override def getBlockByHeader(header: Header): Option[Block] = blocks.find(_.id sameElements header.id) +// +// override var isFullChainSynced: Boolean = true +// +// override def isModifierDefined(id: ModifierId): Boolean = blocks.exists(_.id sameElements id) +// +// override def headerIdsAtHeight(height: Int): List[ModifierId] = List(blocks(height).id) +// +// override def modifierBytesById(id: ModifierId): Option[Array[Byte]] = blocks.find(_.id sameElements id).map(_.bytes) +// +// override def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = Seq.empty +// +// override def lastHeaders(count: Int): HeaderChain = HeaderChain.empty +// +// override def syncInfo: SyncInfo = SyncInfo(List.empty) +// +// override def isFastSyncInProcess: Boolean = false +// +// override def getBestHeader: Option[Header] = Some(blocks.last.header) +// +// override def getBestBlock: Option[Block] = Some(blocks.last) +// +// } +// +// val stateDir = FileHelper.getRandomTempDir +// val emptyHistoryReader = HistoryReader.empty +// val parent = TestProbe() +// val actor = TestActorRef[NVHState]( +// NVHState.genesisProps(settings.copy(directory = stateDir.getAbsolutePath), None), parent.ref +// ) +// blocks.sortBy(_.header.height).foreach { block => +// actor ! ApplyModifier(block.header, saveRootNodesFlag = true, isFullChainSynced = true) +// actor ! ApplyModifier(block, saveRootNodesFlag = true, isFullChainSynced = true) +// } +// Thread.sleep(5000) +// actor.stop() +// val recreatedActor = TestActorRef[NVHState](NVHState.restoreConsistentStateProps( +// settings.copy(directory = stateDir.getAbsolutePath), +// historyReader, +// Some(TestProbe().ref) +// ).get) +// recreatedActor.underlyingActor.state.height shouldEqual blocks.last.header.height +// recreatedActor.underlyingActor.state.tree.avlStorage.currentVersion shouldEqual blocks.last.id +// } +// } +//} diff --git a/src/test/scala/encry/nvg/NodeViewNMProcessorTests.scala b/src/test/scala/encry/nvg/NodeViewNMProcessorTests.scala index ddaa238e6e..7ccebf1848 100644 --- a/src/test/scala/encry/nvg/NodeViewNMProcessorTests.scala +++ b/src/test/scala/encry/nvg/NodeViewNMProcessorTests.scala @@ -30,696 +30,709 @@ import scorex.utils.Random import scala.collection.immutable import scala.concurrent.duration._ -class NodeViewNMProcessorTests - extends TestKit(ActorSystem("Tested-Akka-System")) - with WordSpecLike - with Matchers - with BeforeAndAfterAll - with InstanceFactory - with OneInstancePerTest { - - override def afterAll(): Unit = system.terminate() - - "Network messages processor" should { - "process sync info message correctly" should { - "determine older extension" in { - val parentActor = TestProbe() - - val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - - val (historyMain: History, historyOlder: History) = - NodeViewNMProcessorTests.formYoungerActorState(10, 10) - - val historyReader: HistoryReader = HistoryReader(historyMain) - - networkProcessor ! UpdateHistoryReader(historyReader) - - val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(historyOlder.syncInfo) - - val (dataFromPeerMsg, address) = - NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) - - val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Older) - - networkProcessor ! dataFromPeerMsg - - parentActor.expectMsgPF() { - case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true - case CheckPayloadsToDownload => - } - } - "determine younger extension" in { - val parentActor = TestProbe() - - val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - - val (historyMain: History, historyYounger: History) = - NodeViewNMProcessorTests.formOlderActorState(10, 10) - - val historyReader: HistoryReader = HistoryReader(historyMain) - - networkProcessor ! UpdateHistoryReader(historyReader) - - val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(historyYounger.syncInfo) - - val (dataFromPeerMsg, address) = - NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) - - val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Younger) - - networkProcessor ! dataFromPeerMsg - - parentActor.expectMsgPF() { - case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true - case CheckPayloadsToDownload => - } - } - "determine equals extension" in { - val parentActor = TestProbe() - - val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - - val (history1: History, history2: History) = - NodeViewNMProcessorTests.formEqualActorState(10, 10) - - val historyReader: HistoryReader = HistoryReader(history1) - - networkProcessor ! UpdateHistoryReader(historyReader) - - val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(history2.syncInfo) - - val (dataFromPeerMsg, address) = - NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) - - val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Equal) - - networkProcessor ! dataFromPeerMsg - - parentActor.expectMsgPF() { - case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true - case CheckPayloadsToDownload => - } - } - "determine unknown extension" in { - val parentActor = TestProbe() - - val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - - val (h1: History, h2: History) = NodeViewNMProcessorTests.formUnknownActorState - - val historyReader: HistoryReader = HistoryReader(h1) - - networkProcessor ! UpdateHistoryReader(historyReader) - - val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(h2.syncInfo) - - val (dataFromPeerMsg, address) = - NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) - - val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Unknown) - - networkProcessor ! dataFromPeerMsg - - parentActor.expectMsgPF() { - case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true - case CheckPayloadsToDownload => - } - } - "determine fork extension" in { - val parentActor = TestProbe() - - val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - - val (h1: History, h2: History) = - NodeViewNMProcessorTests.formForkActorState(10, 20, 5) - - val historyReader: HistoryReader = HistoryReader(h1) - - networkProcessor ! UpdateHistoryReader(historyReader) - - val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(h2.syncInfo) - - val (dataFromPeerMsg, address) = - NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) - - val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Fork) - - networkProcessor ! dataFromPeerMsg - - parentActor.expectMsgPF() { - case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true - case CheckPayloadsToDownload => - } - } - } - "process inv message correctly" should { - "not process inv for payload while full chain is not synced" in { - val parentActor = TestProbe() - - val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - - val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) - - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) - - networkProcessor ! DataFromPeer(InvNetworkMessage(Payload.modifierTypeId -> ids), address) - - parentActor.expectNoMsg() - } - "not create response from local for payloads if header's chain is not synced" in { - val parentActor = TestProbe() - - val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - - val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) - - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) - - val historyReader = HistoryReader.empty - - historyReader.isHeadersChainSyncedVar = false - - historyReader.isFullChainSynced = true - - networkProcessor ! UpdateHistoryReader(historyReader) - - networkProcessor ! DataFromPeer(InvNetworkMessage(Payload.modifierTypeId -> ids), address) - - parentActor.expectNoMsg() - } - "create response from local for payloads if header's chain is synced" in { - val parentActor = TestProbe() - - val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - - val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) - - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) - - val historyReader = HistoryReader.empty - - historyReader.isHeadersChainSyncedVar = true - - historyReader.isFullChainSynced = true - - networkProcessor ! UpdateHistoryReader(historyReader) - - networkProcessor ! DataFromPeer(InvNetworkMessage(Header.modifierTypeId -> ids), address) - - val requiredRequestFromLocal = RequestFromLocal(address.some, Header.modifierTypeId, ids.toList) - - parentActor.expectMsgPF() { - case CheckPayloadsToDownload => - case msg: RequestFromLocal => (msg eqv requiredRequestFromLocal) shouldBe true - } - } - "request only unique new modifiers" in { - val parentActor = TestProbe() - - val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - - val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) - - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) - - val (history, blocks) = NodeViewNMProcessorTests.formHistory - - val historyReader = HistoryReader(history) - - historyReader.isHeadersChainSyncedVar = true - - historyReader.isFullChainSynced = true - - networkProcessor ! UpdateHistoryReader(historyReader) - - networkProcessor ! DataFromPeer(InvNetworkMessage(Header.modifierTypeId -> (ids ++ blocks.map(_.id))), address) - - val requiredRequestFromLocal = RequestFromLocal(address.some, Header.modifierTypeId, ids.toList) - - parentActor.expectMsgPF() { - case CheckPayloadsToDownload => - case msg: RequestFromLocal => (msg eqv requiredRequestFromLocal) shouldBe true - } - } - } - "process semantically successful modifier correctly" should { - "update local cache with last semantically successful modifier" in { - val networkMessagesProcessor: TestActorRef[NodeViewNMProcessor] = - NodeViewNMProcessorTests.initActorState(settings) - - val reader = HistoryReader.empty - - reader.isFullChainSynced = true - - networkMessagesProcessor ! UpdateHistoryReader(reader) - - val block = generateGenesisBlock(Height @@ 1) - - networkMessagesProcessor ! SemanticallySuccessfulModifier(block) - - networkMessagesProcessor.underlyingActor.modifiersRequestCache.size shouldBe 2 - networkMessagesProcessor.underlyingActor.modifiersRequestCache.get(block.encodedId).nonEmpty shouldBe true - networkMessagesProcessor.underlyingActor.modifiersRequestCache - .get(block.payload.encodedId) - .nonEmpty shouldBe true - - val block2 = generateGenesisBlock(Height @@ 2) - - networkMessagesProcessor ! SemanticallySuccessfulModifier(block2) - - networkMessagesProcessor.underlyingActor.modifiersRequestCache.size shouldBe 2 - networkMessagesProcessor.underlyingActor.modifiersRequestCache.get(block.encodedId).nonEmpty shouldBe false - networkMessagesProcessor.underlyingActor.modifiersRequestCache - .get(block.payload.encodedId) - .nonEmpty shouldBe false - - networkMessagesProcessor.underlyingActor.modifiersRequestCache.get(block2.encodedId).nonEmpty shouldBe true - networkMessagesProcessor.underlyingActor.modifiersRequestCache - .get(block2.payload.encodedId) - .nonEmpty shouldBe true - } - "send broadcast message for new modifier" in { - val parentActor = TestProbe() - - val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - - val reader = HistoryReader.empty - - reader.isFullChainSynced = true - - networkProcessor ! UpdateHistoryReader(reader) - - val block = generateGenesisBlock(Height @@ 1) - - networkProcessor ! SemanticallySuccessfulModifier(block) - - parentActor.expectMsgPF() { - case CheckPayloadsToDownload => - case BroadcastModifier(modType, id) if modType == Header.modifierTypeId => - id.sameElements(block.id) shouldBe true - case BroadcastModifier(modType, id) if modType == Payload.modifierTypeId => - id.sameElements(block.payload.id) shouldBe true - } - } - } - "process request for modifier correctly" should { - "response for modifiers which are in cache by using this cache" in { - val parentActor = TestProbe() - - val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - - val (history, blocks) = NodeViewNMProcessorTests.formHistory - - val historyReader = HistoryReader(history) - - historyReader.isFullChainSynced = true - - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) - - networkProcessor ! UpdateHistoryReader(historyReader) - - val block = generateGenesisBlock(Height @@ 1) - - networkProcessor ! SemanticallySuccessfulModifier(block) - - networkProcessor ! DataFromPeer( - RequestModifiersNetworkMessage(Header.modifierTypeId -> (blocks.headOption.get.id :: block.id :: Nil)), - address - ) - - parentActor.expectMsgPF() { - case ResponseFromLocal(_, _, ids) if ids.size == 1 => - ids.forall { - case (id, _) => - id.sameElements(block.id) - } shouldBe true - case ResponseFromLocal(_, _, ids) => - ids.forall { - case (id, _) => - id.sameElements(blocks.headOption.get.id) - } shouldBe true - case CheckPayloadsToDownload => - case _: BroadcastModifier => - } - } - "response for headers in 1 message for all headers" in { - val parentActor = TestProbe() - - val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - - val (history, blocks) = NodeViewNMProcessorTests.formHistory - - val historyReader = HistoryReader(history) - - historyReader.isFullChainSynced = true - - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) - - networkProcessor ! UpdateHistoryReader(historyReader) - - val block = generateGenesisBlock(Height @@ 1) - - networkProcessor ! SemanticallySuccessfulModifier(block) - - networkProcessor ! DataFromPeer( - RequestModifiersNetworkMessage(Header.modifierTypeId -> blocks.take(2).map(_.id)), - address - ) - - parentActor.expectMsgPF() { - case ResponseFromLocal(_, _, ids) if ids.size == 2 => - ids.keys.toList.zip(blocks.take(2).map(_.id)).forall { case (id, id1) => id.sameElements(id1) } shouldBe true - case CheckPayloadsToDownload => - case _: BroadcastModifier => - } - } - "response for payloads in 1 message for 1 payload" in { - val parentActor = TestProbe() - - val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - - val (history, blocks) = NodeViewNMProcessorTests.formHistory - - val historyReader = HistoryReader(history) - - historyReader.isFullChainSynced = true - - val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) - - networkProcessor ! UpdateHistoryReader(historyReader) - - val block = generateGenesisBlock(Height @@ 1) - - networkProcessor ! SemanticallySuccessfulModifier(block) - - networkProcessor ! DataFromPeer( - RequestModifiersNetworkMessage(Payload.modifierTypeId -> blocks.take(2).map(_.payload.id)), - address - ) - - parentActor.expectMsgPF() { - case ResponseFromLocal(_, _, ids) if ids.size == 1 => - ids.keys.toList.forall { id => - id.sameElements(blocks(1).payload.id) || id.sameElements(blocks.head.payload.id) - } shouldBe true - case CheckPayloadsToDownload => - case _: BroadcastModifier => - } - } - } - "have correct logic with check payloads to download" should { - "request required modifiers" in { - val parentActor = TestProbe() - - val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - - val (_, blocks) = NodeViewNMProcessorTests.formHistory - - val history = blocks.take(5).foldLeft(generateDummyHistory(settings)) { - case (h, block) => - h.append(block.header).right.get._1.reportModifierIsValid(block.header) - } - - history.isFullChainSynced = true - - val historyReader = HistoryReader(history) - - networkProcessor ! UpdateHistoryReader(historyReader) - - val requiredResponse = RequestFromLocal(none, Payload.modifierTypeId, blocks.take(5).map(_.payload.id)) - - parentActor.expectMsgPF(settings.network.syncInterval + 1.seconds) { - case _: SendSyncInfo => - case msg: RequestFromLocal => msg.eqv(requiredResponse) shouldBe true - } - - parentActor.expectMsgPF(settings.network.syncInterval + 1.seconds) { - case _: SendSyncInfo => - case msg: RequestFromLocal => msg.eqv(requiredResponse) shouldBe true - } - - val updHistory = blocks.drop(5).foldLeft(history) { - case (h, block) => - h.append(block.header).right.get._1.reportModifierIsValid(block.header) - } - - val updHistory2 = blocks.take(3).foldLeft(updHistory) { - case (h, block) => - h.append(block.payload).right.get._1.reportModifierIsValid(block.payload).reportModifierIsValid(block) - } - - val historyReader1 = HistoryReader(updHistory2) - - networkProcessor ! UpdateHistoryReader(historyReader1) - - val requiredResponse1 = RequestFromLocal(none, Payload.modifierTypeId, blocks.drop(3).map(_.payload.id)) - - parentActor.expectMsgPF(settings.network.syncInterval + 1.seconds) { - case _: SendSyncInfo => - case msg: RequestFromLocal => msg.eqv(requiredResponse1) shouldBe true - } - } - "not request if headers chain is not synced" in { - val parentActor = TestProbe() - - val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) - - val (_, blocks) = NodeViewNMProcessorTests.formHistory - - val history = blocks.foldLeft(generateDummyHistory(settings)) { - case (h, block) => - h.append(block.header).right.get._1.reportModifierIsValid(block.header) - } - - history.isFullChainSynced = false - - val historyReader = HistoryReader(history) - - networkProcessor ! UpdateHistoryReader(historyReader) - - parentActor.expectMsgPF(settings.network.syncInterval + 2.seconds) { - case _: SendSyncInfo => - } - } - } - } -} - -object NodeViewNMProcessorTests extends InstanceFactory { - - def initActorState(settings: EncryAppSettings)(implicit AS: ActorSystem): TestActorRef[NodeViewNMProcessor] = { - val networkProcessor: TestActorRef[NodeViewNMProcessor] = - TestActorRef[NodeViewNMProcessor](NodeViewNMProcessor.props(settings)) - networkProcessor - } - - def formDataFromPeerMessage(innerMessage: NetworkMessage, host: String, port: Int)( - implicit AS: ActorSystem - ): (DataFromPeer, InetSocketAddress) = { - val address = new InetSocketAddress(host, port) - DataFromPeer(innerMessage, address) -> address - } - - def formYoungerActorState(blocksQty: Int, olderBlocksQty: Int): (History, History) = { - val (hMain, hOlder, blocks) = (0 until blocksQty).foldLeft( - generateDummyHistory(settings), - generateDummyHistory(settings), - List.empty[Block] - ) { - case ((historyMain, historyOlder, blocks: List[Block]), _) => - val block: Block = generateNextBlock(historyMain) - val hMain: History = historyMain - .append(block.header) - .right - .get - ._1 - .append(block.payload) - .right - .get - ._1 - .reportModifierIsValid(block) - - val hOlder = historyOlder - .append(block.header) - .right - .get - ._1 - .append(block.payload) - .right - .get - ._1 - .reportModifierIsValid(block) - (hMain, hOlder, block +: blocks) - } - val (hOlderUpdated, _) = (0 until olderBlocksQty).foldLeft(hOlder, List.empty[Block]) { - case ((historyOlder, blocks: List[Block]), _) => - val block: Block = generateNextBlock(historyOlder) - val hOlder: History = historyOlder - .append(block.header) - .right - .get - ._1 - .append(block.payload) - .right - .get - ._1 - .reportModifierIsValid(block) - (hOlder, block +: blocks) - } - (hMain, hOlderUpdated) - } - - def formOlderActorState(blocksQty: Int, olderBlocksQty: Int): (History, History) = { - val (historyYounger, historyOlder) = formYoungerActorState(blocksQty, olderBlocksQty) - (historyOlder, historyYounger) - } - - def formEqualActorState(blocksQty: Int, olderBlocksQty: Int): (History, History) = - (0 until blocksQty).foldLeft( - generateDummyHistory(settings), - generateDummyHistory(settings) - ) { - case ((historyMain, historyOlder), _) => - val block: Block = generateNextBlock(historyMain) - val hEq1: History = historyMain - .append(block.header) - .right - .get - ._1 - .append(block.payload) - .right - .get - ._1 - .reportModifierIsValid(block) - - val hEq2 = historyOlder - .append(block.header) - .right - .get - ._1 - .append(block.payload) - .right - .get - ._1 - .reportModifierIsValid(block) - (hEq1, hEq2) - } - - def formUnknownActorState: (History, History) = { - val history1 = (0 until 10).foldLeft( - generateDummyHistory(settings) - ) { - case (h1, _) => - val block1: Block = generateNextBlock(h1) - val hEq1: History = h1 - .append(block1.header) - .right - .get - ._1 - .append(block1.payload) - .right - .get - ._1 - .reportModifierIsValid(block1) - hEq1 - } - val history2 = (0 until 2).foldLeft( - generateDummyHistory(settings) - ) { - case (h2, _) => - val block1: Block = generateNextBlock(h2) - val hEq2: History = h2 - .append(block1.header) - .right - .get - ._1 - .append(block1.payload) - .right - .get - ._1 - .reportModifierIsValid(block1) - hEq2 - } - history1 -> history2 - } - - def formForkActorState(forkOn: Int, forkSize: Int, mainChainSize: Int): (History, History) = { - val (h1, h2) = (0 until forkOn).foldLeft( - generateDummyHistory(settings), - generateDummyHistory(settings) - ) { - case ((historyMain, historyOlder), _) => - val block: Block = generateNextBlock(historyMain) - val hEq1: History = historyMain - .append(block.header) - .right - .get - ._1 - .append(block.payload) - .right - .get - ._1 - .reportModifierIsValid(block) - - val hEq2 = historyOlder - .append(block.header) - .right - .get - ._1 - .append(block.payload) - .right - .get - ._1 - .reportModifierIsValid(block) - (hEq1, hEq2) - } - val fork = (0 until forkSize).foldLeft(h1) { - case (historyMain, _) => - val block: Block = generateNextBlock(historyMain) - val hEq1: History = historyMain - .append(block.header) - .right - .get - ._1 - .append(block.payload) - .right - .get - ._1 - .reportModifierIsValid(block) - hEq1 - } - val mch = (0 until mainChainSize).foldLeft(h2) { - case (historyMain, _) => - val block: Block = generateNextBlock(historyMain) - val hEq1: History = historyMain - .append(block.header) - .right - .get - ._1 - .append(block.payload) - .right - .get - ._1 - .reportModifierIsValid(block) - hEq1 - } - mch -> fork - } - - def formHistory: (History, List[Block]) = - (0 to 10).foldLeft(generateDummyHistory(settings), List.empty[Block]) { - case ((historyMain, blocks), _) => - val block: Block = generateNextBlock(historyMain) - val hEq1: History = historyMain - .append(block.header) - .right - .get - ._1 - .append(block.payload) - .right - .get - ._1 - .reportModifierIsValid(block) - hEq1 -> (blocks :+ block) - } -} +//class NodeViewNMProcessorTests +// extends TestKit(ActorSystem("Tested-Akka-System")) +// with WordSpecLike +// with Matchers +// with BeforeAndAfterAll +// with InstanceFactory +// with OneInstancePerTest { +// +// override def afterAll(): Unit = system.terminate() +// +// "Network messages processor" should { +// "process sync info message correctly" should { +// "determine older extension" in { +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) +// +// val (historyMain: History, historyOlder: History) = +// NodeViewNMProcessorTests.formYoungerActorState(10, 10) +// +// val historyReader: HistoryReader = HistoryReader(historyMain) +// +// networkProcessor ! UpdateHistoryReader(historyReader) +// +// val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(historyOlder.syncInfo) +// +// val (dataFromPeerMsg, address) = +// NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) +// +// val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Older) +// +// networkProcessor ! dataFromPeerMsg +// +// parentActor.expectMsgPF() { +// case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true +// case CheckPayloadsToDownload => +// } +// } +// "determine younger extension" in { +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) +// +// val (historyMain: History, historyYounger: History) = +// NodeViewNMProcessorTests.formOlderActorState(10, 10) +// +// val historyReader: HistoryReader = HistoryReader(historyMain) +// +// networkProcessor ! UpdateHistoryReader(historyReader) +// +// val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(historyYounger.syncInfo) +// +// val (dataFromPeerMsg, address) = +// NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) +// +// val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Younger) +// +// networkProcessor ! dataFromPeerMsg +// +// parentActor.expectMsgPF() { +// case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true +// case CheckPayloadsToDownload => +// } +// } +// "determine equals extension" in { +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) +// +// val (history1: History, history2: History) = +// NodeViewNMProcessorTests.formEqualActorState(10, 10) +// +// val historyReader: HistoryReader = HistoryReader(history1) +// +// networkProcessor ! UpdateHistoryReader(historyReader) +// +// val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(history2.syncInfo) +// +// val (dataFromPeerMsg, address) = +// NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) +// +// val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Equal) +// +// networkProcessor ! dataFromPeerMsg +// +// parentActor.expectMsgPF() { +// case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true +// case CheckPayloadsToDownload => +// } +// } +// "determine unknown extension" in { +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) +// +// val (h1: History, h2: History) = NodeViewNMProcessorTests.formUnknownActorState +// +// val historyReader: HistoryReader = HistoryReader(h1) +// +// networkProcessor ! UpdateHistoryReader(historyReader) +// +// val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(h2.syncInfo) +// +// val (dataFromPeerMsg, address) = +// NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) +// +// val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Unknown) +// +// networkProcessor ! dataFromPeerMsg +// +// parentActor.expectMsgPF() { +// case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true +// case CheckPayloadsToDownload => +// } +// } +// "determine fork extension" in { +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) +// +// val (h1: History, h2: History) = +// NodeViewNMProcessorTests.formForkActorState(10, 20, 5) +// +// val historyReader: HistoryReader = HistoryReader(h1) +// +// networkProcessor ! UpdateHistoryReader(historyReader) +// +// val syncInfoMessage: SyncInfoNetworkMessage = SyncInfoNetworkMessage(h2.syncInfo) +// +// val (dataFromPeerMsg, address) = +// NodeViewNMProcessorTests.formDataFromPeerMessage(syncInfoMessage, "0.0.0.0", 9001) +// +// val expectedResult: OtherNodeSyncingStatus = OtherNodeSyncingStatus(address, Fork) +// +// networkProcessor ! dataFromPeerMsg +// +// parentActor.expectMsgPF() { +// case msg: OtherNodeSyncingStatus => (msg eqv expectedResult) shouldBe true +// case CheckPayloadsToDownload => +// } +// } +// } +// "process inv message correctly" should { +// "not process inv for payload while full chain is not synced" in { +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) +// +// val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) +// +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) +// +// networkProcessor ! DataFromPeer(InvNetworkMessage(Payload.modifierTypeId -> ids), address) +// +// parentActor.expectNoMsg() +// } +// "not create response from local for payloads if header's chain is not synced" in { +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) +// +// val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) +// +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) +// +// val historyReader = HistoryReader.empty +// +// historyReader.isHeadersChainSyncedVar = false +// +// historyReader.isFullChainSynced = true +// +// networkProcessor ! UpdateHistoryReader(historyReader) +// +// networkProcessor ! DataFromPeer(InvNetworkMessage(Payload.modifierTypeId -> ids), address) +// +// parentActor.expectNoMsg() +// } +// "create response from local for payloads if header's chain is synced" in { +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) +// +// val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) +// +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) +// +// val historyReader = HistoryReader.empty +// +// historyReader.isHeadersChainSyncedVar = true +// +// historyReader.isFullChainSynced = true +// +// networkProcessor ! UpdateHistoryReader(historyReader) +// +// networkProcessor ! DataFromPeer(InvNetworkMessage(Header.modifierTypeId -> ids), address) +// +// val requiredRequestFromLocal = RequestFromLocal(address.some, Header.modifierTypeId, ids.toList) +// +// parentActor.expectMsgPF() { +// case CheckPayloadsToDownload => +// case msg: RequestFromLocal => (msg eqv requiredRequestFromLocal) shouldBe true +// } +// } +// "request only unique new modifiers" in { +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) +// +// val ids: immutable.IndexedSeq[ModifierId] = (0 to 10).map(_ => ModifierId @@ Random.randomBytes()) +// +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) +// +// val (history, blocks) = NodeViewNMProcessorTests.formHistory +// +// val historyReader = HistoryReader(history) +// +// historyReader.isHeadersChainSyncedVar = true +// +// historyReader.isFullChainSynced = true +// +// networkProcessor ! UpdateHistoryReader(historyReader) +// +// networkProcessor ! DataFromPeer(InvNetworkMessage(Header.modifierTypeId -> (ids ++ blocks.map(_.id))), address) +// +// val requiredRequestFromLocal = RequestFromLocal(address.some, Header.modifierTypeId, ids.toList) +// +// parentActor.expectMsgPF() { +// case CheckPayloadsToDownload => +// case msg: RequestFromLocal => (msg eqv requiredRequestFromLocal) shouldBe true +// } +// } +// } +// "process semantically successful modifier correctly" should { +// "update local cache with last semantically successful modifier" in { +// val networkMessagesProcessor: TestActorRef[NodeViewNMProcessor] = +// NodeViewNMProcessorTests.initActorState(settings) +// +// val reader = HistoryReader.empty +// +// reader.isFullChainSynced = true +// +// networkMessagesProcessor ! UpdateHistoryReader(reader) +// +// val block = generateGenesisBlock(Height @@ 1) +// +// networkMessagesProcessor ! SemanticallySuccessfulModifier(block) +// +// networkMessagesProcessor.underlyingActor.modifiersRequestCache.size shouldBe 2 +// networkMessagesProcessor.underlyingActor.modifiersRequestCache.get(block.encodedId).nonEmpty shouldBe true +// networkMessagesProcessor.underlyingActor.modifiersRequestCache +// .get(block.payload.encodedId) +// .nonEmpty shouldBe true +// +// val block2 = generateGenesisBlock(Height @@ 2) +// +// networkMessagesProcessor ! SemanticallySuccessfulModifier(block2) +// +// networkMessagesProcessor.underlyingActor.modifiersRequestCache.size shouldBe 2 +// networkMessagesProcessor.underlyingActor.modifiersRequestCache.get(block.encodedId).nonEmpty shouldBe false +// networkMessagesProcessor.underlyingActor.modifiersRequestCache +// .get(block.payload.encodedId) +// .nonEmpty shouldBe false +// +// networkMessagesProcessor.underlyingActor.modifiersRequestCache.get(block2.encodedId).nonEmpty shouldBe true +// networkMessagesProcessor.underlyingActor.modifiersRequestCache +// .get(block2.payload.encodedId) +// .nonEmpty shouldBe true +// } +// "send broadcast message for new modifier" in { +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) +// +// val reader = HistoryReader.empty +// +// reader.isFullChainSynced = true +// +// networkProcessor ! UpdateHistoryReader(reader) +// +// val block = generateGenesisBlock(Height @@ 1) +// +// networkProcessor ! SemanticallySuccessfulModifier(block) +// +// parentActor.expectMsgPF() { +// case CheckPayloadsToDownload => +// case BroadcastModifier(modType, id) if modType == Header.modifierTypeId => +// id.sameElements(block.id) shouldBe true +// case BroadcastModifier(modType, id) if modType == Payload.modifierTypeId => +// id.sameElements(block.payload.id) shouldBe true +// } +// } +// } +// "process request for modifier correctly" should { +// "response for modifiers which are in cache by using this cache" in { +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) +// +// val (history, blocks) = NodeViewNMProcessorTests.formHistory +// +// val historyReader = HistoryReader(history) +// +// historyReader.isFullChainSynced = true +// +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) +// +// networkProcessor ! UpdateHistoryReader(historyReader) +// +// val block = generateGenesisBlock(Height @@ 1) +// +// networkProcessor ! SemanticallySuccessfulModifier(block) +// +// networkProcessor ! DataFromPeer( +// RequestModifiersNetworkMessage(Header.modifierTypeId -> (blocks.headOption.get.id :: block.id :: Nil)), +// address +// ) +// +// parentActor.expectMsgPF() { +// case ResponseFromLocal(_, _, ids) if ids.size == 1 => +// ids.forall { +// case (id, _) => +// id.sameElements(block.id) +// } shouldBe true +// case ResponseFromLocal(_, _, ids) => +// ids.forall { +// case (id, _) => +// id.sameElements(blocks.headOption.get.id) +// } shouldBe true +// case CheckPayloadsToDownload => +// case _: BroadcastModifier => +// } +// } +// "response for headers in 1 message for all headers" in { +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) +// +// val (history, blocks) = NodeViewNMProcessorTests.formHistory +// +// val historyReader = HistoryReader(history) +// +// historyReader.isFullChainSynced = true +// +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) +// +// networkProcessor ! UpdateHistoryReader(historyReader) +// +// val block = generateGenesisBlock(Height @@ 1) +// +// networkProcessor ! SemanticallySuccessfulModifier(block) +// +// networkProcessor ! DataFromPeer( +// RequestModifiersNetworkMessage(Header.modifierTypeId -> blocks.take(2).map(_.id)), +// address +// ) +// +// parentActor.expectMsgPF() { +// case ResponseFromLocal(_, _, ids) if ids.size == 2 => +// ids.keys.toList.zip(blocks.take(2).map(_.id)).forall { case (id, id1) => id.sameElements(id1) } shouldBe true +// case CheckPayloadsToDownload => +// case _: BroadcastModifier => +// } +// } +// "response for payloads in 1 message for 1 payload" in { +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) +// +// val (history, blocks) = NodeViewNMProcessorTests.formHistory +// +// val historyReader = HistoryReader(history) +// +// historyReader.isFullChainSynced = true +// +// val address: InetSocketAddress = new InetSocketAddress("0.0.0.0", 9001) +// +// networkProcessor ! UpdateHistoryReader(historyReader) +// +// val block = generateGenesisBlock(Height @@ 1) +// +// networkProcessor ! SemanticallySuccessfulModifier(block) +// +// networkProcessor ! DataFromPeer( +// RequestModifiersNetworkMessage(Payload.modifierTypeId -> blocks.take(2).map(_.payload.id)), +// address +// ) +// +// parentActor.expectMsgPF() { +// case ResponseFromLocal(_, _, ids) if ids.size == 1 => +// ids.keys.toList.forall { id => +// id.sameElements(blocks(1).payload.id) || id.sameElements(blocks.head.payload.id) +// } shouldBe true +// case CheckPayloadsToDownload => +// case _: BroadcastModifier => +// } +// } +// } +// "have correct logic with check payloads to download" should { +// "request required modifiers" in { +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) +// +// val (_, blocks) = NodeViewNMProcessorTests.formHistory +// +// val history = blocks.take(5).foldLeft(generateDummyHistory(settings)) { +// case (h, block) => +// h.append(block.header).right.get._1 +// h.reportModifierIsValid(block.header) +// } +// +// history.isFullChainSynced = true +// +// val historyReader = HistoryReader(history) +// +// networkProcessor ! UpdateHistoryReader(historyReader) +// +// val requiredResponse = RequestFromLocal(none, Payload.modifierTypeId, blocks.take(5).map(_.payload.id)) +// +// parentActor.expectMsgPF(settings.network.syncInterval + 1.seconds) { +// case _: SendSyncInfo => +// case msg: RequestFromLocal => msg.eqv(requiredResponse) shouldBe true +// } +// +// parentActor.expectMsgPF(settings.network.syncInterval + 1.seconds) { +// case _: SendSyncInfo => +// case msg: RequestFromLocal => msg.eqv(requiredResponse) shouldBe true +// } +// +// val updHistory = blocks.drop(5).foldLeft(history) { +// case (h, block) => +// h.append(block.header).right.get._1 +// h.reportModifierIsValid(block.header) +// } +// +// val updHistory2 = blocks.take(3).foldLeft(updHistory) { +// case (h, block) => +// h.append(block.payload).right.get._1 +// h.reportModifierIsValid(block.payload).reportModifierIsValid(block) +// } +// +// val historyReader1 = HistoryReader(updHistory2) +// +// networkProcessor ! UpdateHistoryReader(historyReader1) +// +// val requiredResponse1 = RequestFromLocal(none, Payload.modifierTypeId, blocks.drop(3).map(_.payload.id)) +// +// parentActor.expectMsgPF(settings.network.syncInterval + 1.seconds) { +// case _: SendSyncInfo => +// case msg: RequestFromLocal => msg.eqv(requiredResponse1) shouldBe true +// } +// } +// "not request if headers chain is not synced" in { +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = parentActor.childActorOf(NodeViewNMProcessor.props(settings)) +// +// val (_, blocks) = NodeViewNMProcessorTests.formHistory +// +// val history = blocks.foldLeft(generateDummyHistory(settings)) { +// case (h, block) => +// h.append(block.header).right.get._1 +// h.reportModifierIsValid(block.header) +// } +// +// history.isFullChainSynced = false +// +// val historyReader = HistoryReader(history) +// +// networkProcessor ! UpdateHistoryReader(historyReader) +// +// parentActor.expectMsgPF(settings.network.syncInterval + 2.seconds) { +// case _: SendSyncInfo => +// } +// } +// } +// } +//} +// +//object NodeViewNMProcessorTests extends InstanceFactory { +// +// def initActorState(settings: EncryAppSettings)(implicit AS: ActorSystem): TestActorRef[NodeViewNMProcessor] = { +// val networkProcessor: TestActorRef[NodeViewNMProcessor] = +// TestActorRef[NodeViewNMProcessor](NodeViewNMProcessor.props(settings)) +// networkProcessor +// } +// +// def formDataFromPeerMessage(innerMessage: NetworkMessage, host: String, port: Int)( +// implicit AS: ActorSystem +// ): (DataFromPeer, InetSocketAddress) = { +// val address = new InetSocketAddress(host, port) +// DataFromPeer(innerMessage, address) -> address +// } +// +// def formYoungerActorState(blocksQty: Int, olderBlocksQty: Int): (History, History) = { +// val (hMain, hOlder, blocks) = (0 until blocksQty).foldLeft( +// generateDummyHistory(settings), +// generateDummyHistory(settings), +// List.empty[Block] +// ) { +// case ((historyMain, historyOlder, blocks: List[Block]), _) => +// val block: Block = generateNextBlock(historyMain) +// val hMain: History = { +// historyMain +// .append(block.header) +// .right +// .get +// ._1 +// +// historyMain.append(block.payload) +// .right +// .get +// ._1 +// +// historyMain.reportModifierIsValid(block) +// } +// +// val hOlder = { +// historyOlder.append(block.header) +// .right +// .get +// ._1 +// +// historyOlder.append(block.payload) +// .right +// .get +// ._1 +// +// historyOlder.reportModifierIsValid(block) +// } +// (hMain, hOlder, block +: blocks) +// } +// val (hOlderUpdated, _) = (0 until olderBlocksQty).foldLeft(hOlder, List.empty[Block]) { +// case ((historyOlder, blocks: List[Block]), _) => +// val block: Block = generateNextBlock(historyOlder) +// val hOlder: History = { +// historyOlder +// .append(block.header) +// .right +// .get +// ._1 +// historyOlder.append(block.payload) +// .right +// .get +// ._1 +// historyOlder.reportModifierIsValid(block) +// } +// (hOlder, block +: blocks) +// } +// (hMain, hOlderUpdated) +// } +// +// def formOlderActorState(blocksQty: Int, olderBlocksQty: Int): (History, History) = { +// val (historyYounger, historyOlder) = formYoungerActorState(blocksQty, olderBlocksQty) +// (historyOlder, historyYounger) +// } +// +// def formEqualActorState(blocksQty: Int, olderBlocksQty: Int): (History, History) = +// (0 until blocksQty).foldLeft( +// generateDummyHistory(settings), +// generateDummyHistory(settings) +// ) { +// case ((historyMain, historyOlder), _) => +// val block: Block = generateNextBlock(historyMain) +// val hEq1: History = historyMain +// .append(block.header) +// .right +// .get +// ._1 +// .append(block.payload) +// .right +// .get +// ._1 +// .reportModifierIsValid(block) +// +// val hEq2 = historyOlder +// .append(block.header) +// .right +// .get +// ._1 +// .append(block.payload) +// .right +// .get +// ._1 +// .reportModifierIsValid(block) +// (hEq1, hEq2) +// } +// +// def formUnknownActorState: (History, History) = { +// val history1 = (0 until 10).foldLeft( +// generateDummyHistory(settings) +// ) { +// case (h1, _) => +// val block1: Block = generateNextBlock(h1) +// val hEq1: History = h1 +// .append(block1.header) +// .right +// .get +// ._1 +// .append(block1.payload) +// .right +// .get +// ._1 +// .reportModifierIsValid(block1) +// hEq1 +// } +// val history2 = (0 until 2).foldLeft( +// generateDummyHistory(settings) +// ) { +// case (h2, _) => +// val block1: Block = generateNextBlock(h2) +// val hEq2: History = h2 +// .append(block1.header) +// .right +// .get +// ._1 +// .append(block1.payload) +// .right +// .get +// ._1 +// .reportModifierIsValid(block1) +// hEq2 +// } +// history1 -> history2 +// } +// +// def formForkActorState(forkOn: Int, forkSize: Int, mainChainSize: Int): (History, History) = { +// val (h1, h2) = (0 until forkOn).foldLeft( +// generateDummyHistory(settings), +// generateDummyHistory(settings) +// ) { +// case ((historyMain, historyOlder), _) => +// val block: Block = generateNextBlock(historyMain) +// val hEq1: History = historyMain +// .append(block.header) +// .right +// .get +// ._1 +// .append(block.payload) +// .right +// .get +// ._1 +// .reportModifierIsValid(block) +// +// val hEq2 = historyOlder +// .append(block.header) +// .right +// .get +// ._1 +// .append(block.payload) +// .right +// .get +// ._1 +// .reportModifierIsValid(block) +// (hEq1, hEq2) +// } +// val fork = (0 until forkSize).foldLeft(h1) { +// case (historyMain, _) => +// val block: Block = generateNextBlock(historyMain) +// val hEq1: History = historyMain +// .append(block.header) +// .right +// .get +// ._1 +// .append(block.payload) +// .right +// .get +// ._1 +// .reportModifierIsValid(block) +// hEq1 +// } +// val mch = (0 until mainChainSize).foldLeft(h2) { +// case (historyMain, _) => +// val block: Block = generateNextBlock(historyMain) +// val hEq1: History = historyMain +// .append(block.header) +// .right +// .get +// ._1 +// .append(block.payload) +// .right +// .get +// ._1 +// .reportModifierIsValid(block) +// hEq1 +// } +// mch -> fork +// } +// +// def formHistory: (History, List[Block]) = +// (0 to 10).foldLeft(generateDummyHistory(settings), List.empty[Block]) { +// case ((historyMain, blocks), _) => +// val block: Block = generateNextBlock(historyMain) +// val hEq1: History = historyMain +// .append(block.header) +// .right +// .get +// ._1 +// .append(block.payload) +// .right +// .get +// ._1 +// .reportModifierIsValid(block) +// hEq1 -> (blocks :+ block) +// } +//} diff --git a/src/test/scala/encry/nvg/PipelinesTests.scala b/src/test/scala/encry/nvg/PipelinesTests.scala index 541de76946..6e227594bd 100644 --- a/src/test/scala/encry/nvg/PipelinesTests.scala +++ b/src/test/scala/encry/nvg/PipelinesTests.scala @@ -1,554 +1,554 @@ -package encry.nvg - -import java.net.InetSocketAddress - -import akka.actor.{ActorRef, ActorSystem} -import akka.testkit.{TestActorRef, TestKit, TestProbe} -import com.typesafe.scalalogging.StrictLogging -import encry.EncryApp -import encry.consensus.EncrySupplyController -import encry.modifiers.InstanceFactory -import encry.modifiers.mempool.TransactionFactory -import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, SendSyncInfo} -import encry.network.NetworkController.ReceivableMessages.RegisterMessagesHandler -import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} -import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier -import encry.utils.implicits.UTXO.{combineAll, _} -import encry.utils.{FileHelper, Mnemonic, NetworkTimeProvider, TestHelper} -import encry.view.history.History -import encry.view.state.UtxoState -import encry.view.state.avlTree.utils.implicits.Instances._ -import encry.view.wallet.AccountManager -import org.encryfoundation.common.crypto.PrivateKey25519 -import org.encryfoundation.common.crypto.equihash.EquihashSolution -import org.encryfoundation.common.modifiers.history._ -import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.modifiers.state.box.AssetBox -import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height} -import org.encryfoundation.common.utils.constants.TestNetConstants -import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} - -import scala.concurrent.duration._ - -class PipelinesTests - extends TestKit(ActorSystem("Tested-Akka-System")) - with WordSpecLike - with Matchers - with BeforeAndAfterAll - with InstanceFactory - with OneInstancePerTest - with StrictLogging { - - override def afterAll(): Unit = system.terminate() - - "Node view pipelines" should { - "correct process modifier from the network async" in { - val tmpFile = FileHelper.getRandomTempDir - val path = tmpFile.getAbsolutePath - val settingsWithNewPath = - settings - .copy(directory = path) - .copy(wallet = settings.wallet.map(_.copy(password = "123"))) - .copy(node = settings.node.copy(isTestMod = true)) - AccountManager.init(Mnemonic.entropyToMnemonicCode(scorex.utils.Random.randomBytes(16)), - "123", - settingsWithNewPath) - - val intermediaryParent = TestProbe() - val networkIntermediary = TestProbe() - val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsWithNewPath.ntp) - - val intermediary: ActorRef = intermediaryParent.childActorOf( - IntermediaryNVH.props( - settingsWithNewPath, - networkIntermediary.ref, - timeProvider, - None, - TestProbe().ref, - TestProbe().ref - ) - ) - - val remote = new InetSocketAddress("0.0.0.0", 9001) - - val (_, _, blocks) = PipelinesTests.generateValidForHistoryAndStateBlocks(300, - generateDummyHistory(settings), - UtxoState.genesis( - FileHelper.getRandomTempDir, - FileHelper.getRandomTempDir, - settings, - None - )) - - blocks.reverse.foreach { block => - intermediary ! ModifierFromNetwork( - remote, - Header.modifierTypeId, - block.id, - HeaderProtoSerializer.toProto(block.header).toByteArray - ) - logger.info(s"Sent to nvh actor header ${block.encodedId}") - } - - Thread.sleep(8000) - - networkIntermediary.expectMsgPF(15.seconds) { - case SemanticallySuccessfulModifier(mod) => - blocks.exists(_.id.sameElements(mod.id)) shouldBe true - case msg @ SendSyncInfo(_) => - case msg @ BroadcastModifier(modifierTypeId, modifierId) => - blocks.exists(_.id.sameElements(modifierId)) shouldBe true - case RegisterMessagesHandler(_, _) => - case RegisterForModsHandling => - case RequestFromLocal(s, m, mods) => - mods.size shouldBe blocks.size - mods.zip(blocks).forall { - case (id, block) => id.sameElements(block.id) - } shouldBe true - } - - blocks.reverse.foreach { block => - intermediary ! ModifierFromNetwork( - remote, - Payload.modifierTypeId, - block.payload.id, - PayloadProtoSerializer.toProto(block.payload).toByteArray - ) - } - - Thread.sleep(10000) - - networkIntermediary.expectMsgPF(15.seconds) { - case SemanticallySuccessfulModifier(mod) => - blocks.exists(_.id.sameElements(mod.id)) shouldBe true - case msg @ SendSyncInfo(_) => - case RegisterForModsHandling => - case msg @ BroadcastModifier(modifierTypeId, modifierId) => - blocks.exists(_.id.sameElements(modifierId)) shouldBe true - case RegisterMessagesHandler(_, _) => - case RequestFromLocal(s, m, mods) => - } - - } - "correct process modifier from the network sync" in { - val tmpFile = FileHelper.getRandomTempDir - val path = tmpFile.getAbsolutePath - val settingsWithNewPath = - settings - .copy(directory = path) - .copy(wallet = settings.wallet.map(_.copy(password = "123"))) - .copy(node = settings.node.copy(isTestMod = true)) - AccountManager.init(Mnemonic.entropyToMnemonicCode(scorex.utils.Random.randomBytes(16)), - "123", - settingsWithNewPath) - - val networkIntermediary = TestProbe() - val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsWithNewPath.ntp) - - val intermediary = TestActorRef[IntermediaryNVH]( - IntermediaryNVH.props( - settingsWithNewPath, - networkIntermediary.ref, - timeProvider, - None, - TestProbe().ref, - TestProbe().ref - ) - ) - - val remote = new InetSocketAddress("0.0.0.0", 9001) - - val (_, _, blocks) = PipelinesTests.generateValidForHistoryAndStateBlocks(300, - generateDummyHistory(settings), - UtxoState.genesis( - FileHelper.getRandomTempDir, - FileHelper.getRandomTempDir, - settings, - None - )) - - blocks.reverse.foreach { block => - intermediary ! ModifierFromNetwork( - remote, - Header.modifierTypeId, - block.id, - HeaderProtoSerializer.toProto(block.header).toByteArray - ) - logger.info(s"Sent to nvh actor header ${block.encodedId}") - } - - Thread.sleep(8000) - - intermediary.underlyingActor.historyReader.getBestHeaderHeight shouldBe 300 - intermediary.underlyingActor.historyReader.getBestBlockHeight shouldBe -1 - - blocks.reverse.foreach { block => - intermediary ! ModifierFromNetwork( - remote, - Payload.modifierTypeId, - block.payload.id, - PayloadProtoSerializer.toProto(block.payload).toByteArray - ) - } - - Thread.sleep(10000) - - intermediary.underlyingActor.historyReader.getBestHeaderHeight shouldBe 300 - intermediary.underlyingActor.historyReader.getBestBlockHeight shouldBe 300 - } - "work with forks correctly" in { - logger.info("================ work with forks correctly start ====================") - val tmpFile = FileHelper.getRandomTempDir - val path = tmpFile.getAbsolutePath - val settingsWithNewPath = - settings - .copy(directory = path) - .copy(wallet = settings.wallet.map(_.copy(password = "123"))) - .copy(node = settings.node.copy(isTestMod = true)) - - AccountManager.init(Mnemonic.entropyToMnemonicCode(scorex.utils.Random.randomBytes(16)), - "123", - settingsWithNewPath) - - val networkIntermediary = TestProbe() - val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsWithNewPath.ntp) - - val intermediary = TestActorRef[IntermediaryNVH]( - IntermediaryNVH.props( - settingsWithNewPath, - networkIntermediary.ref, - timeProvider, - None, - TestProbe().ref, - TestProbe().ref - ) - ) - - val remote = new InetSocketAddress("0.0.0.0", 9001) - - val ((h1, s1, b1), (h2, s2, b2)) = PipelinesTests.genForOn(5) - - b1.foreach { block => - intermediary ! ModifierFromNetwork( - remote, - Header.modifierTypeId, - block.header.id, - HeaderProtoSerializer.toProto(block.header).toByteArray - ) - } - - Thread.sleep(3000) - - b1.foreach { block => - intermediary ! ModifierFromNetwork( - remote, - Payload.modifierTypeId, - block.payload.id, - PayloadProtoSerializer.toProto(block.payload).toByteArray - ) - } - - Thread.sleep(5000) - - b2.foreach { block => - intermediary ! ModifierFromNetwork( - remote, - Header.modifierTypeId, - block.header.id, - HeaderProtoSerializer.toProto(block.header).toByteArray - ) - } - - Thread.sleep(3000) - - b2.foreach { block => - intermediary ! ModifierFromNetwork( - remote, - Payload.modifierTypeId, - block.payload.id, - PayloadProtoSerializer.toProto(block.payload).toByteArray - ) - } - - Thread.sleep(5000) - - println("getBestBlock = " + Algos.encode(intermediary.underlyingActor.historyReader.getBestBlock.get.id)) - println("b2 = " + Algos.encode(b2.last.id)) - intermediary.underlyingActor.historyReader.getBestBlock.get.id.sameElements(b2.last.id) shouldBe true - intermediary.underlyingActor.historyReader.getBestHeader.get.id.sameElements(b2.last.id) shouldBe true - - } - } - -} - -object PipelinesTests extends InstanceFactory with StrictLogging { - - val key: PrivateKey25519 = TestHelper.genKeys(1).head - - def generateValidForHistoryAndStateBlocks( - blocksQty: Int, - history: History, - utxo: UtxoState, - from: Int = 0 - ): (History, UtxoState, List[Block]) = { - (from to from + blocksQty).foldLeft( - history, - utxo, - List.empty[Block] - ) { - case ((history, state, blocks), i) => - val blockNext: Block = - if (i > 0) { - val boxes: Seq[AssetBox] = - history.getBestBlock.get.payload.txs.flatMap(_.newBoxes.toList).take(30).collect { - case a: AssetBox if a.amount > 13 => a - } - val txs: Vector[Transaction] = - generatePaymentTransactions(key, boxes.toIndexedSeq, 1, 2) - val feesTotal = txs.map(_.fee).sum - val supplyTotal = EncrySupplyController.supplyAt(Height @@ i, settings.constants) - val coinbase: Transaction = TransactionFactory - .coinbaseTransactionScratch(key.publicImage, timestamp, supplyTotal, feesTotal, Height @@ i) - val resTxs = txs :+ coinbase - val difficulty: Difficulty = history.getBestHeader - .map( - parent => - history.requiredDifficultyAfter(parent) match { - case Right(value) => value - case Left(value) => EncryApp.forceStopApplication(999, value.toString) - } - ) - .getOrElse(TestNetConstants.InitialDifficulty) - val combinedStateChange: UtxoState.StateChange = combineAll(resTxs.map(UtxoState.tx2StateChange).toList) - val newStateRoot = state.tree - .getOperationsRootHash( - combinedStateChange.outputsToDb.toList, - combinedStateChange.inputsToDb.toList - ) - .get - - val header = - Header( - TestNetConstants.Version, - history.getBestHeaderId.get, - Payload.rootHash(resTxs.map(_.id)), - System.currentTimeMillis(), - i, - 1, - difficulty, - EquihashSolution(Seq(1, 3)), - newStateRoot - ) - val payload = Payload(header.id, resTxs) - val block = Block(header, payload) - block - } else { - val supplyTotal = EncrySupplyController.supplyAt(Height @@ i, settings.constants) - val coinbase: Transaction = TransactionFactory - .coinbaseTransactionScratch(key.publicImage, timestamp, supplyTotal, 0, Height @@ i) - val resTxs = List(coinbase) - val difficulty: Difficulty = history.getBestHeader - .map( - parent => - history.requiredDifficultyAfter(parent) match { - case Right(value) => value - case Left(value) => EncryApp.forceStopApplication(999, value.toString) - } - ) - .getOrElse(TestNetConstants.InitialDifficulty) - val combinedStateChange: UtxoState.StateChange = combineAll(resTxs.map(UtxoState.tx2StateChange).toList) - val newStateRoot = state.tree - .getOperationsRootHash( - combinedStateChange.outputsToDb.toList, - combinedStateChange.inputsToDb.toList - ) - .get - - val header = - Header( - 0: Byte, - Header.GenesisParentId, - Payload.rootHash(resTxs.map(_.id)), - System.currentTimeMillis(), - i, - 1, - difficulty, - EquihashSolution(Seq(1, 3)), - newStateRoot - ) - val payload = Payload(header.id, resTxs) - val block = Block(header, payload) - block - } - val h = history - .append(blockNext.header) - .right - .get - ._1 - .append(blockNext.payload) - .right - .get - ._1 - .reportModifierIsValid(blockNext) - - val s = state - .applyModifier(blockNext.header) - .right - .get - .applyModifier(blockNext) - .right - .get - (h, s, blocks :+ blockNext) - } - } - - def genForOn( - blocksQty: Int - ): ((History, UtxoState, List[Block]), (History, UtxoState, List[Block])) = { - val (h, s, h1, s1, b) = (0 to blocksQty).foldLeft( - generateDummyHistory(settings), - UtxoState.genesis( - FileHelper.getRandomTempDir, - FileHelper.getRandomTempDir, - settings, - None - ), - generateDummyHistory(settings), - UtxoState.genesis( - FileHelper.getRandomTempDir, - FileHelper.getRandomTempDir, - settings, - None - ), - List.empty[Block] - ) { - case ((history, state, h1, s1, blocks), i) => - val blockNext: Block = - if (i > 0) { - val boxes: Seq[AssetBox] = - history.getBestBlock.get.payload.txs.flatMap(_.newBoxes.toList).take(30).collect { - case a: AssetBox if a.amount > 13 => a - } - val txs: Vector[Transaction] = - generatePaymentTransactions(key, boxes.toIndexedSeq, 1, 2) - val feesTotal = txs.map(_.fee).sum - val supplyTotal = EncrySupplyController.supplyAt(Height @@ i, settings.constants) - val coinbase: Transaction = TransactionFactory - .coinbaseTransactionScratch(key.publicImage, timestamp, supplyTotal, feesTotal, Height @@ i) - val resTxs = txs :+ coinbase - val difficulty: Difficulty = history.getBestHeader - .map( - parent => - history.requiredDifficultyAfter(parent) match { - case Right(value) => value - case Left(value) => EncryApp.forceStopApplication(999, value.toString) - } - ) - .getOrElse(TestNetConstants.InitialDifficulty) - val combinedStateChange: UtxoState.StateChange = combineAll(resTxs.map(UtxoState.tx2StateChange).toList) - val newStateRoot = state.tree - .getOperationsRootHash( - combinedStateChange.outputsToDb.toList, - combinedStateChange.inputsToDb.toList - ) - .get - - val header = - Header( - TestNetConstants.Version, - history.getBestHeaderId.get, - Payload.rootHash(resTxs.map(_.id)), - System.currentTimeMillis(), - i, - 1, - difficulty, - EquihashSolution(Seq(1, 3)), - newStateRoot - ) - val payload = Payload(header.id, resTxs) - val block = Block(header, payload) - block - } else { - val supplyTotal = EncrySupplyController.supplyAt(Height @@ i, settings.constants) - val coinbase: Transaction = TransactionFactory - .coinbaseTransactionScratch(key.publicImage, timestamp, supplyTotal, 0, Height @@ i) - val resTxs = List(coinbase) - val difficulty: Difficulty = history.getBestHeader - .map( - parent => - history.requiredDifficultyAfter(parent) match { - case Right(value) => value - case Left(value) => EncryApp.forceStopApplication(999, value.toString) - } - ) - .getOrElse(TestNetConstants.InitialDifficulty) - val combinedStateChange: UtxoState.StateChange = combineAll(resTxs.map(UtxoState.tx2StateChange).toList) - val newStateRoot = state.tree - .getOperationsRootHash( - combinedStateChange.outputsToDb.toList, - combinedStateChange.inputsToDb.toList - ) - .get - - val header = - Header( - 0: Byte, - Header.GenesisParentId, - Payload.rootHash(resTxs.map(_.id)), - System.currentTimeMillis(), - i, - 1, - difficulty, - EquihashSolution(Seq(1, 3)), - newStateRoot - ) - val payload = Payload(header.id, resTxs) - val block = Block(header, payload) - block - } - val h = history - .append(blockNext.header) - .right - .get - ._1 - .append(blockNext.payload) - .right - .get - ._1 - .reportModifierIsValid(blockNext) - - val s = state - .applyModifier(blockNext.header) - .right - .get - .applyModifier(blockNext) - .right - .get - - val his1 = h1 - .append(blockNext.header) - .right - .get - ._1 - .append(blockNext.payload) - .right - .get - ._1 - .reportModifierIsValid(blockNext) - - val st1 = s1 - .applyModifier(blockNext.header) - .right - .get - .applyModifier(blockNext) - .right - .get - (h, s, his1, st1, blocks :+ blockNext) - } - val (h11, s11, b11) = generateValidForHistoryAndStateBlocks(5, h, s, h.getBestBlockHeight + 1) - val (h22, s22, b22) = generateValidForHistoryAndStateBlocks(10, h1, s1, h1.getBestBlockHeight + 1) - ((h11, s11, (b11 ++ b).sortBy(_.header.height)), (h22, s22, b22.sortBy(_.header.height))) - } -} +//package encry.nvg +// +//import java.net.InetSocketAddress +// +//import akka.actor.{ActorRef, ActorSystem} +//import akka.testkit.{TestActorRef, TestKit, TestProbe} +//import com.typesafe.scalalogging.StrictLogging +//import encry.EncryApp +//import encry.consensus.EncrySupplyController +//import encry.modifiers.InstanceFactory +//import encry.modifiers.mempool.TransactionFactory +//import encry.network.Messages.MessageToNetwork.{BroadcastModifier, RequestFromLocal, SendSyncInfo} +//import encry.network.NetworkController.ReceivableMessages.RegisterMessagesHandler +//import encry.network.NetworkRouter.{ModifierFromNetwork, RegisterForModsHandling} +//import encry.nvg.NodeViewHolder.SemanticallySuccessfulModifier +//import encry.utils.implicits.UTXO.{combineAll, _} +//import encry.utils.{FileHelper, Mnemonic, NetworkTimeProvider, TestHelper} +//import encry.view.history.History +//import encry.view.state.UtxoState +//import encry.view.state.avlTree.utils.implicits.Instances._ +//import encry.view.wallet.AccountManager +//import org.encryfoundation.common.crypto.PrivateKey25519 +//import org.encryfoundation.common.crypto.equihash.EquihashSolution +//import org.encryfoundation.common.modifiers.history._ +//import org.encryfoundation.common.modifiers.mempool.transaction.Transaction +//import org.encryfoundation.common.modifiers.state.box.AssetBox +//import org.encryfoundation.common.utils.Algos +//import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height} +//import org.encryfoundation.common.utils.constants.TestNetConstants +//import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} +// +//import scala.concurrent.duration._ +// +//class PipelinesTests +// extends TestKit(ActorSystem("Tested-Akka-System")) +// with WordSpecLike +// with Matchers +// with BeforeAndAfterAll +// with InstanceFactory +// with OneInstancePerTest +// with StrictLogging { +// +// override def afterAll(): Unit = system.terminate() +// +// "Node view pipelines" should { +// "correct process modifier from the network async" in { +// val tmpFile = FileHelper.getRandomTempDir +// val path = tmpFile.getAbsolutePath +// val settingsWithNewPath = +// settings +// .copy(directory = path) +// .copy(wallet = settings.wallet.map(_.copy(password = "123"))) +// .copy(node = settings.node.copy(isTestMod = true)) +// AccountManager.init(Mnemonic.entropyToMnemonicCode(scorex.utils.Random.randomBytes(16)), +// "123", +// settingsWithNewPath) +// +// val intermediaryParent = TestProbe() +// val networkIntermediary = TestProbe() +// val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsWithNewPath.ntp) +// +// val intermediary: ActorRef = intermediaryParent.childActorOf( +// IntermediaryNVH.props( +// settingsWithNewPath, +// networkIntermediary.ref, +// timeProvider, +// None, +// TestProbe().ref, +// TestProbe().ref +// ) +// ) +// +// val remote = new InetSocketAddress("0.0.0.0", 9001) +// +// val (_, _, blocks) = PipelinesTests.generateValidForHistoryAndStateBlocks(300, +// generateDummyHistory(settings), +// UtxoState.genesis( +// FileHelper.getRandomTempDir, +// FileHelper.getRandomTempDir, +// settings, +// None +// )) +// +// blocks.reverse.foreach { block => +// intermediary ! ModifierFromNetwork( +// remote, +// Header.modifierTypeId, +// block.id, +// HeaderProtoSerializer.toProto(block.header).toByteArray +// ) +// logger.info(s"Sent to nvh actor header ${block.encodedId}") +// } +// +// Thread.sleep(8000) +// +// networkIntermediary.expectMsgPF(15.seconds) { +// case SemanticallySuccessfulModifier(mod) => +// blocks.exists(_.id.sameElements(mod.id)) shouldBe true +// case msg @ SendSyncInfo(_) => +// case msg @ BroadcastModifier(modifierTypeId, modifierId) => +// blocks.exists(_.id.sameElements(modifierId)) shouldBe true +// case RegisterMessagesHandler(_, _) => +// case RegisterForModsHandling => +// case RequestFromLocal(s, m, mods) => +// mods.size shouldBe blocks.size +// mods.zip(blocks).forall { +// case (id, block) => id.sameElements(block.id) +// } shouldBe true +// } +// +// blocks.reverse.foreach { block => +// intermediary ! ModifierFromNetwork( +// remote, +// Payload.modifierTypeId, +// block.payload.id, +// PayloadProtoSerializer.toProto(block.payload).toByteArray +// ) +// } +// +// Thread.sleep(10000) +// +// networkIntermediary.expectMsgPF(15.seconds) { +// case SemanticallySuccessfulModifier(mod) => +// blocks.exists(_.id.sameElements(mod.id)) shouldBe true +// case msg @ SendSyncInfo(_) => +// case RegisterForModsHandling => +// case msg @ BroadcastModifier(modifierTypeId, modifierId) => +// blocks.exists(_.id.sameElements(modifierId)) shouldBe true +// case RegisterMessagesHandler(_, _) => +// case RequestFromLocal(s, m, mods) => +// } +// +// } +// "correct process modifier from the network sync" in { +// val tmpFile = FileHelper.getRandomTempDir +// val path = tmpFile.getAbsolutePath +// val settingsWithNewPath = +// settings +// .copy(directory = path) +// .copy(wallet = settings.wallet.map(_.copy(password = "123"))) +// .copy(node = settings.node.copy(isTestMod = true)) +// AccountManager.init(Mnemonic.entropyToMnemonicCode(scorex.utils.Random.randomBytes(16)), +// "123", +// settingsWithNewPath) +// +// val networkIntermediary = TestProbe() +// val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsWithNewPath.ntp) +// +// val intermediary = TestActorRef[IntermediaryNVH]( +// IntermediaryNVH.props( +// settingsWithNewPath, +// networkIntermediary.ref, +// timeProvider, +// None, +// TestProbe().ref, +// TestProbe().ref +// ) +// ) +// +// val remote = new InetSocketAddress("0.0.0.0", 9001) +// +// val (_, _, blocks) = PipelinesTests.generateValidForHistoryAndStateBlocks(300, +// generateDummyHistory(settings), +// UtxoState.genesis( +// FileHelper.getRandomTempDir, +// FileHelper.getRandomTempDir, +// settings, +// None +// )) +// +// blocks.reverse.foreach { block => +// intermediary ! ModifierFromNetwork( +// remote, +// Header.modifierTypeId, +// block.id, +// HeaderProtoSerializer.toProto(block.header).toByteArray +// ) +// logger.info(s"Sent to nvh actor header ${block.encodedId}") +// } +// +// Thread.sleep(8000) +// +// intermediary.underlyingActor.historyReader.getBestHeaderHeight shouldBe 300 +// intermediary.underlyingActor.historyReader.getBestBlockHeight shouldBe -1 +// +// blocks.reverse.foreach { block => +// intermediary ! ModifierFromNetwork( +// remote, +// Payload.modifierTypeId, +// block.payload.id, +// PayloadProtoSerializer.toProto(block.payload).toByteArray +// ) +// } +// +// Thread.sleep(10000) +// +// intermediary.underlyingActor.historyReader.getBestHeaderHeight shouldBe 300 +// intermediary.underlyingActor.historyReader.getBestBlockHeight shouldBe 300 +// } +// "work with forks correctly" in { +// logger.info("================ work with forks correctly start ====================") +// val tmpFile = FileHelper.getRandomTempDir +// val path = tmpFile.getAbsolutePath +// val settingsWithNewPath = +// settings +// .copy(directory = path) +// .copy(wallet = settings.wallet.map(_.copy(password = "123"))) +// .copy(node = settings.node.copy(isTestMod = true)) +// +// AccountManager.init(Mnemonic.entropyToMnemonicCode(scorex.utils.Random.randomBytes(16)), +// "123", +// settingsWithNewPath) +// +// val networkIntermediary = TestProbe() +// val timeProvider: NetworkTimeProvider = new NetworkTimeProvider(settingsWithNewPath.ntp) +// +// val intermediary = TestActorRef[IntermediaryNVH]( +// IntermediaryNVH.props( +// settingsWithNewPath, +// networkIntermediary.ref, +// timeProvider, +// None, +// TestProbe().ref, +// TestProbe().ref +// ) +// ) +// +// val remote = new InetSocketAddress("0.0.0.0", 9001) +// +// val ((h1, s1, b1), (h2, s2, b2)) = PipelinesTests.genForOn(5) +// +// b1.foreach { block => +// intermediary ! ModifierFromNetwork( +// remote, +// Header.modifierTypeId, +// block.header.id, +// HeaderProtoSerializer.toProto(block.header).toByteArray +// ) +// } +// +// Thread.sleep(3000) +// +// b1.foreach { block => +// intermediary ! ModifierFromNetwork( +// remote, +// Payload.modifierTypeId, +// block.payload.id, +// PayloadProtoSerializer.toProto(block.payload).toByteArray +// ) +// } +// +// Thread.sleep(5000) +// +// b2.foreach { block => +// intermediary ! ModifierFromNetwork( +// remote, +// Header.modifierTypeId, +// block.header.id, +// HeaderProtoSerializer.toProto(block.header).toByteArray +// ) +// } +// +// Thread.sleep(3000) +// +// b2.foreach { block => +// intermediary ! ModifierFromNetwork( +// remote, +// Payload.modifierTypeId, +// block.payload.id, +// PayloadProtoSerializer.toProto(block.payload).toByteArray +// ) +// } +// +// Thread.sleep(5000) +// +// println("getBestBlock = " + Algos.encode(intermediary.underlyingActor.historyReader.getBestBlock.get.id)) +// println("b2 = " + Algos.encode(b2.last.id)) +// intermediary.underlyingActor.historyReader.getBestBlock.get.id.sameElements(b2.last.id) shouldBe true +// intermediary.underlyingActor.historyReader.getBestHeader.get.id.sameElements(b2.last.id) shouldBe true +// +// } +// } +// +//} +// +//object PipelinesTests extends InstanceFactory with StrictLogging { +// +// val key: PrivateKey25519 = TestHelper.genKeys(1).head +// +// def generateValidForHistoryAndStateBlocks( +// blocksQty: Int, +// history: History, +// utxo: UtxoState, +// from: Int = 0 +// ): (History, UtxoState, List[Block]) = { +// (from to from + blocksQty).foldLeft( +// history, +// utxo, +// List.empty[Block] +// ) { +// case ((history, state, blocks), i) => +// val blockNext: Block = +// if (i > 0) { +// val boxes: Seq[AssetBox] = +// history.getBestBlock.get.payload.txs.flatMap(_.newBoxes.toList).take(30).collect { +// case a: AssetBox if a.amount > 13 => a +// } +// val txs: Vector[Transaction] = +// generatePaymentTransactions(key, boxes.toIndexedSeq, 1, 2) +// val feesTotal = txs.map(_.fee).sum +// val supplyTotal = EncrySupplyController.supplyAt(Height @@ i, settings.constants) +// val coinbase: Transaction = TransactionFactory +// .coinbaseTransactionScratch(key.publicImage, timestamp, supplyTotal, feesTotal, Height @@ i) +// val resTxs = txs :+ coinbase +// val difficulty: Difficulty = history.getBestHeader +// .map( +// parent => +// history.requiredDifficultyAfter(parent) match { +// case Right(value) => value +// case Left(value) => EncryApp.forceStopApplication(999, value.toString) +// } +// ) +// .getOrElse(TestNetConstants.InitialDifficulty) +// val combinedStateChange: UtxoState.StateChange = combineAll(resTxs.map(UtxoState.tx2StateChange).toList) +// val newStateRoot = state.tree +// .getOperationsRootHash( +// combinedStateChange.outputsToDb.toList, +// combinedStateChange.inputsToDb.toList +// ) +// .get +// +// val header = +// Header( +// TestNetConstants.Version, +// history.getBestHeaderId.get, +// Payload.rootHash(resTxs.map(_.id)), +// System.currentTimeMillis(), +// i, +// 1, +// difficulty, +// EquihashSolution(Seq(1, 3)), +// newStateRoot +// ) +// val payload = Payload(header.id, resTxs) +// val block = Block(header, payload) +// block +// } else { +// val supplyTotal = EncrySupplyController.supplyAt(Height @@ i, settings.constants) +// val coinbase: Transaction = TransactionFactory +// .coinbaseTransactionScratch(key.publicImage, timestamp, supplyTotal, 0, Height @@ i) +// val resTxs = List(coinbase) +// val difficulty: Difficulty = history.getBestHeader +// .map( +// parent => +// history.requiredDifficultyAfter(parent) match { +// case Right(value) => value +// case Left(value) => EncryApp.forceStopApplication(999, value.toString) +// } +// ) +// .getOrElse(TestNetConstants.InitialDifficulty) +// val combinedStateChange: UtxoState.StateChange = combineAll(resTxs.map(UtxoState.tx2StateChange).toList) +// val newStateRoot = state.tree +// .getOperationsRootHash( +// combinedStateChange.outputsToDb.toList, +// combinedStateChange.inputsToDb.toList +// ) +// .get +// +// val header = +// Header( +// 0: Byte, +// Header.GenesisParentId, +// Payload.rootHash(resTxs.map(_.id)), +// System.currentTimeMillis(), +// i, +// 1, +// difficulty, +// EquihashSolution(Seq(1, 3)), +// newStateRoot +// ) +// val payload = Payload(header.id, resTxs) +// val block = Block(header, payload) +// block +// } +// val h = history +// .append(blockNext.header) +// .right +// .get +// ._1 +// .append(blockNext.payload) +// .right +// .get +// ._1 +// .reportModifierIsValid(blockNext) +// +// val s = state +// .applyModifier(blockNext.header) +// .right +// .get +// .applyModifier(blockNext) +// .right +// .get +// (h, s, blocks :+ blockNext) +// } +// } +// +// def genForOn( +// blocksQty: Int +// ): ((History, UtxoState, List[Block]), (History, UtxoState, List[Block])) = { +// val (h, s, h1, s1, b) = (0 to blocksQty).foldLeft( +// generateDummyHistory(settings), +// UtxoState.genesis( +// FileHelper.getRandomTempDir, +// FileHelper.getRandomTempDir, +// settings, +// None +// ), +// generateDummyHistory(settings), +// UtxoState.genesis( +// FileHelper.getRandomTempDir, +// FileHelper.getRandomTempDir, +// settings, +// None +// ), +// List.empty[Block] +// ) { +// case ((history, state, h1, s1, blocks), i) => +// val blockNext: Block = +// if (i > 0) { +// val boxes: Seq[AssetBox] = +// history.getBestBlock.get.payload.txs.flatMap(_.newBoxes.toList).take(30).collect { +// case a: AssetBox if a.amount > 13 => a +// } +// val txs: Vector[Transaction] = +// generatePaymentTransactions(key, boxes.toIndexedSeq, 1, 2) +// val feesTotal = txs.map(_.fee).sum +// val supplyTotal = EncrySupplyController.supplyAt(Height @@ i, settings.constants) +// val coinbase: Transaction = TransactionFactory +// .coinbaseTransactionScratch(key.publicImage, timestamp, supplyTotal, feesTotal, Height @@ i) +// val resTxs = txs :+ coinbase +// val difficulty: Difficulty = history.getBestHeader +// .map( +// parent => +// history.requiredDifficultyAfter(parent) match { +// case Right(value) => value +// case Left(value) => EncryApp.forceStopApplication(999, value.toString) +// } +// ) +// .getOrElse(TestNetConstants.InitialDifficulty) +// val combinedStateChange: UtxoState.StateChange = combineAll(resTxs.map(UtxoState.tx2StateChange).toList) +// val newStateRoot = state.tree +// .getOperationsRootHash( +// combinedStateChange.outputsToDb.toList, +// combinedStateChange.inputsToDb.toList +// ) +// .get +// +// val header = +// Header( +// TestNetConstants.Version, +// history.getBestHeaderId.get, +// Payload.rootHash(resTxs.map(_.id)), +// System.currentTimeMillis(), +// i, +// 1, +// difficulty, +// EquihashSolution(Seq(1, 3)), +// newStateRoot +// ) +// val payload = Payload(header.id, resTxs) +// val block = Block(header, payload) +// block +// } else { +// val supplyTotal = EncrySupplyController.supplyAt(Height @@ i, settings.constants) +// val coinbase: Transaction = TransactionFactory +// .coinbaseTransactionScratch(key.publicImage, timestamp, supplyTotal, 0, Height @@ i) +// val resTxs = List(coinbase) +// val difficulty: Difficulty = history.getBestHeader +// .map( +// parent => +// history.requiredDifficultyAfter(parent) match { +// case Right(value) => value +// case Left(value) => EncryApp.forceStopApplication(999, value.toString) +// } +// ) +// .getOrElse(TestNetConstants.InitialDifficulty) +// val combinedStateChange: UtxoState.StateChange = combineAll(resTxs.map(UtxoState.tx2StateChange).toList) +// val newStateRoot = state.tree +// .getOperationsRootHash( +// combinedStateChange.outputsToDb.toList, +// combinedStateChange.inputsToDb.toList +// ) +// .get +// +// val header = +// Header( +// 0: Byte, +// Header.GenesisParentId, +// Payload.rootHash(resTxs.map(_.id)), +// System.currentTimeMillis(), +// i, +// 1, +// difficulty, +// EquihashSolution(Seq(1, 3)), +// newStateRoot +// ) +// val payload = Payload(header.id, resTxs) +// val block = Block(header, payload) +// block +// } +// val h = history +// .append(blockNext.header) +// .right +// .get +// ._1 +// .append(blockNext.payload) +// .right +// .get +// ._1 +// .reportModifierIsValid(blockNext) +// +// val s = state +// .applyModifier(blockNext.header) +// .right +// .get +// .applyModifier(blockNext) +// .right +// .get +// +// val his1 = h1 +// .append(blockNext.header) +// .right +// .get +// ._1 +// .append(blockNext.payload) +// .right +// .get +// ._1 +// .reportModifierIsValid(blockNext) +// +// val st1 = s1 +// .applyModifier(blockNext.header) +// .right +// .get +// .applyModifier(blockNext) +// .right +// .get +// (h, s, his1, st1, blocks :+ blockNext) +// } +// val (h11, s11, b11) = generateValidForHistoryAndStateBlocks(5, h, s, h.getBestBlockHeight + 1) +// val (h22, s22, b22) = generateValidForHistoryAndStateBlocks(10, h1, s1, h1.getBestBlockHeight + 1) +// ((h11, s11, (b11 ++ b).sortBy(_.header.height)), (h22, s22, b22.sortBy(_.header.height))) +// } +//} diff --git a/src/test/scala/encry/nvg/ValidatorTests.scala b/src/test/scala/encry/nvg/ValidatorTests.scala index 83af8da7f7..2cbf52e3de 100644 --- a/src/test/scala/encry/nvg/ValidatorTests.scala +++ b/src/test/scala/encry/nvg/ValidatorTests.scala @@ -1,184 +1,184 @@ -package encry.nvg - -import java.net.InetSocketAddress - -import akka.actor.{ ActorRef, ActorSystem } -import akka.testkit.{ TestKit, TestProbe } -import encry.modifiers.InstanceFactory -import encry.network.BlackList.BanReason.{ - CorruptedSerializedBytes, - ModifierIdInTheNetworkMessageIsNotTheSameAsIdOfModifierInThisMessage, - PreSemanticInvalidModifier, - SyntacticallyInvalidPersistentModifier -} -import encry.network.PeersKeeper.BanPeer -import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation, ValidatedModifier } -import encry.nvg.NodeViewHolder.SyntacticallyFailedModification -import encry.view.history.HistoryReader -import org.encryfoundation.common.modifiers.history.{ Block, Header, HeaderProtoSerializer } -import org.encryfoundation.common.utils.TaggedTypes.ModifierId -import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } -import scorex.utils.Random - -class ValidatorTests - extends TestKit(ActorSystem("Tested-Akka-System")) - with WordSpecLike - with Matchers - with BeforeAndAfterAll - with InstanceFactory - with OneInstancePerTest { - - override def afterAll(): Unit = system.terminate() - - "Modifiers validator" should { - "notify intermediary actor about modifier with invalid raw bytes" in { - val nvh = TestProbe() - - val intermediary = TestProbe() - - val parentActor = TestProbe() - - val networkProcessor: ActorRef = - parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) - - val (history, blocks) = NodeViewNMProcessorTests.formHistory - - val reader = HistoryReader(history) - - val corruptedBytes = Random.randomBytes(190) - - val remote = new InetSocketAddress("0.0.0.0", 9001) - - val randomId = ModifierId @@ Random.randomBytes() - - networkProcessor ! ModifierForValidation(reader, randomId, Header.modifierTypeId, corruptedBytes, remote) - - intermediary.expectMsgPF() { - case BanPeer(r, CorruptedSerializedBytes) => r shouldBe remote - case InvalidModifierBytes(id) => id.sameElements(blocks.head.id) shouldBe true - } - } - "notify intermediary actor about pre semantic invalid modifier" in { - val nvh = TestProbe() - - val intermediary = TestProbe() - - val parentActor = TestProbe() - - val networkProcessor: ActorRef = - parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) - - val (history, blocks) = NodeViewNMProcessorTests.formHistory - - val reader = HistoryReader(history) - - val corruptedBlock = blocks.head.copy( - header = blocks.head.header.copy(height = -1000) - ) - - val corruptedBytes = HeaderProtoSerializer.toProto(corruptedBlock.header).toByteArray - - val remote = new InetSocketAddress("0.0.0.0", 9001) - - networkProcessor ! ModifierForValidation(reader, - ModifierId @@ Random.randomBytes(), - Header.modifierTypeId, - corruptedBytes, - remote) - - intermediary.expectMsgPF() { - case BanPeer(r, PreSemanticInvalidModifier(_)) => r shouldBe remote - case SyntacticallyFailedModification(mod, _) => mod.id.sameElements(blocks.head.id) shouldBe true - } - } - "notify intermediary actor about syntactically invalid modifier" in { - val nvh = TestProbe() - - val intermediary = TestProbe() - - val parentActor = TestProbe() - - val networkProcessor: ActorRef = - parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) - - val (history, blocks) = NodeViewNMProcessorTests.formHistory - - val reader = HistoryReader(history) - - val corruptedBlock = blocks.head.copy( - header = blocks.head.header.copy(parentId = ModifierId @@ blocks.head.header.id.drop(2)) - ) - - val corruptedBytes = HeaderProtoSerializer.toProto(corruptedBlock.header).toByteArray - - val remote = new InetSocketAddress("0.0.0.0", 9001) - - networkProcessor ! ModifierForValidation(reader, - ModifierId @@ Random.randomBytes(), - Header.modifierTypeId, - corruptedBytes, - remote) - - intermediary.expectMsgPF() { - case BanPeer(r, SyntacticallyInvalidPersistentModifier) => r shouldBe remote - case SyntacticallyFailedModification(mod, _) => mod.id.sameElements(blocks.head.id) shouldBe true - } - } - "notify intermediary about incorrect modifier id" in { - val nvh = TestProbe() - - val intermediary = TestProbe() - - val parentActor = TestProbe() - - val networkProcessor: ActorRef = - parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) - - val (history, blocks) = NodeViewNMProcessorTests.formHistory - - val reader = HistoryReader(history) - - val corruptedBytes = HeaderProtoSerializer.toProto(blocks.head.header).toByteArray - - val remote = new InetSocketAddress("0.0.0.0", 9001) - - networkProcessor ! ModifierForValidation(reader, - ModifierId @@ Random.randomBytes(), - Header.modifierTypeId, - corruptedBytes, - remote) - - intermediary.expectMsgPF() { - case BanPeer(r, ModifierIdInTheNetworkMessageIsNotTheSameAsIdOfModifierInThisMessage) => r shouldBe remote - case SyntacticallyFailedModification(mod, _) => mod.id.sameElements(blocks.head.id) shouldBe true - } - } - "notify nvh actor about valid modifier" in { - val nvh = TestProbe() - - val intermediary = TestProbe() - - val parentActor = TestProbe() - - val networkProcessor: ActorRef = - parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) - - val (history, _) = NodeViewNMProcessorTests.formHistory - - val reader = HistoryReader(history) - - val correctBlock: Block = generateNextBlock(history) - - val correctBytes = HeaderProtoSerializer.toProto(correctBlock.header).toByteArray - - val remote = new InetSocketAddress("0.0.0.0", 9001) - - networkProcessor ! ModifierForValidation(reader, correctBlock.id, Header.modifierTypeId, correctBytes, remote) - - nvh.expectMsgPF() { - case ValidatedModifier(mod) => mod.id.sameElements(correctBlock.id) shouldBe true - } - } - } - -} +//package encry.nvg +// +//import java.net.InetSocketAddress +// +//import akka.actor.{ ActorRef, ActorSystem } +//import akka.testkit.{ TestKit, TestProbe } +//import encry.modifiers.InstanceFactory +//import encry.network.BlackList.BanReason.{ +// CorruptedSerializedBytes, +// ModifierIdInTheNetworkMessageIsNotTheSameAsIdOfModifierInThisMessage, +// PreSemanticInvalidModifier, +// SyntacticallyInvalidPersistentModifier +//} +//import encry.network.PeersKeeper.BanPeer +//import encry.nvg.ModifiersValidator.{ InvalidModifierBytes, ModifierForValidation, ValidatedModifier } +//import encry.nvg.NodeViewHolder.SyntacticallyFailedModification +//import encry.view.history.HistoryReader +//import org.encryfoundation.common.modifiers.history.{ Block, Header, HeaderProtoSerializer } +//import org.encryfoundation.common.utils.TaggedTypes.ModifierId +//import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } +//import scorex.utils.Random +// +//class ValidatorTests +// extends TestKit(ActorSystem("Tested-Akka-System")) +// with WordSpecLike +// with Matchers +// with BeforeAndAfterAll +// with InstanceFactory +// with OneInstancePerTest { +// +// override def afterAll(): Unit = system.terminate() +// +// "Modifiers validator" should { +// "notify intermediary actor about modifier with invalid raw bytes" in { +// val nvh = TestProbe() +// +// val intermediary = TestProbe() +// +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = +// parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) +// +// val (history, blocks) = NodeViewNMProcessorTests.formHistory +// +// val reader = HistoryReader(history) +// +// val corruptedBytes = Random.randomBytes(190) +// +// val remote = new InetSocketAddress("0.0.0.0", 9001) +// +// val randomId = ModifierId @@ Random.randomBytes() +// +// networkProcessor ! ModifierForValidation(reader, randomId, Header.modifierTypeId, corruptedBytes, remote) +// +// intermediary.expectMsgPF() { +// case BanPeer(r, CorruptedSerializedBytes) => r shouldBe remote +// case InvalidModifierBytes(id) => id.sameElements(blocks.head.id) shouldBe true +// } +// } +// "notify intermediary actor about pre semantic invalid modifier" in { +// val nvh = TestProbe() +// +// val intermediary = TestProbe() +// +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = +// parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) +// +// val (history, blocks) = NodeViewNMProcessorTests.formHistory +// +// val reader = HistoryReader(history) +// +// val corruptedBlock = blocks.head.copy( +// header = blocks.head.header.copy(height = -1000) +// ) +// +// val corruptedBytes = HeaderProtoSerializer.toProto(corruptedBlock.header).toByteArray +// +// val remote = new InetSocketAddress("0.0.0.0", 9001) +// +// networkProcessor ! ModifierForValidation(reader, +// ModifierId @@ Random.randomBytes(), +// Header.modifierTypeId, +// corruptedBytes, +// remote) +// +// intermediary.expectMsgPF() { +// case BanPeer(r, PreSemanticInvalidModifier(_)) => r shouldBe remote +// case SyntacticallyFailedModification(mod, _) => mod.id.sameElements(blocks.head.id) shouldBe true +// } +// } +// "notify intermediary actor about syntactically invalid modifier" in { +// val nvh = TestProbe() +// +// val intermediary = TestProbe() +// +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = +// parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) +// +// val (history, blocks) = NodeViewNMProcessorTests.formHistory +// +// val reader = HistoryReader(history) +// +// val corruptedBlock = blocks.head.copy( +// header = blocks.head.header.copy(parentId = ModifierId @@ blocks.head.header.id.drop(2)) +// ) +// +// val corruptedBytes = HeaderProtoSerializer.toProto(corruptedBlock.header).toByteArray +// +// val remote = new InetSocketAddress("0.0.0.0", 9001) +// +// networkProcessor ! ModifierForValidation(reader, +// ModifierId @@ Random.randomBytes(), +// Header.modifierTypeId, +// corruptedBytes, +// remote) +// +// intermediary.expectMsgPF() { +// case BanPeer(r, SyntacticallyInvalidPersistentModifier) => r shouldBe remote +// case SyntacticallyFailedModification(mod, _) => mod.id.sameElements(blocks.head.id) shouldBe true +// } +// } +// "notify intermediary about incorrect modifier id" in { +// val nvh = TestProbe() +// +// val intermediary = TestProbe() +// +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = +// parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) +// +// val (history, blocks) = NodeViewNMProcessorTests.formHistory +// +// val reader = HistoryReader(history) +// +// val corruptedBytes = HeaderProtoSerializer.toProto(blocks.head.header).toByteArray +// +// val remote = new InetSocketAddress("0.0.0.0", 9001) +// +// networkProcessor ! ModifierForValidation(reader, +// ModifierId @@ Random.randomBytes(), +// Header.modifierTypeId, +// corruptedBytes, +// remote) +// +// intermediary.expectMsgPF() { +// case BanPeer(r, ModifierIdInTheNetworkMessageIsNotTheSameAsIdOfModifierInThisMessage) => r shouldBe remote +// case SyntacticallyFailedModification(mod, _) => mod.id.sameElements(blocks.head.id) shouldBe true +// } +// } +// "notify nvh actor about valid modifier" in { +// val nvh = TestProbe() +// +// val intermediary = TestProbe() +// +// val parentActor = TestProbe() +// +// val networkProcessor: ActorRef = +// parentActor.childActorOf(ModifiersValidator.props(nvh.ref, intermediary.ref, settings)) +// +// val (history, _) = NodeViewNMProcessorTests.formHistory +// +// val reader = HistoryReader(history) +// +// val correctBlock: Block = generateNextBlock(history) +// +// val correctBytes = HeaderProtoSerializer.toProto(correctBlock.header).toByteArray +// +// val remote = new InetSocketAddress("0.0.0.0", 9001) +// +// networkProcessor ! ModifierForValidation(reader, correctBlock.id, Header.modifierTypeId, correctBytes, remote) +// +// nvh.expectMsgPF() { +// case ValidatedModifier(mod) => mod.id.sameElements(correctBlock.id) shouldBe true +// } +// } +// } +// +//} diff --git a/src/test/scala/encry/view/QWE.scala b/src/test/scala/encry/view/QWE.scala index 7baca8edeb..cfb4d1a640 100644 --- a/src/test/scala/encry/view/QWE.scala +++ b/src/test/scala/encry/view/QWE.scala @@ -1,98 +1,98 @@ -package encry.view - -import com.typesafe.scalalogging.StrictLogging -import encry.modifiers.InstanceFactory -import encry.settings.TestNetSettings -import encry.view.history.History -import org.encryfoundation.common.modifiers.history.Block -import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } - -class QWE - extends WordSpecLike - with Matchers - with InstanceFactory - with BeforeAndAfterAll - with OneInstancePerTest - with TestNetSettings - with StrictLogging { - - "qwer" should { - "qr" in { - val (history1_10, history2_10, _) = (0 until 10).foldLeft( - generateDummyHistory(testNetSettings), - generateDummyHistory(testNetSettings), - List.empty[Block] - ) { - case ((prevHistory1, prevHistory2, blocks: List[Block]), _) => - val block: Block = generateNextBlock(prevHistory1) - val a = prevHistory1 - .append(block.header) - .right - .get - ._1 - .append(block.payload) - .right - .get - ._1 - .reportModifierIsValid(block) - val b = prevHistory2 - .append(block.header) - .right - .get - ._1 - .append(block.payload) - .right - .get - ._1 - .reportModifierIsValid(block) - (a, b, (block +: blocks)) - } - logger.info(s"\n\n\n\nStart processing 1 fork blocks\n\n\n\n\n") - val (history3_15norm, blocksNorm15) = (0 until 5).foldLeft(history1_10, List.empty[Block]) { - case ((prevHistory, blocks: List[Block]), _) => - val block: Block = generateNextBlock(prevHistory) - prevHistory - .append(block.header) - .right - .get - ._1 - .append(block.payload) - .right - .get - ._1 - .reportModifierIsValid(block) -> (block +: blocks) - } - logger.info(s"\n\n\n\nStart processing 2 blocks\n\n\n\n\n") - val (h4_20, blocks4_fork) = (0 until 10).foldLeft(history2_10, List.empty[Block]) { - case ((prevHistory, blocks: List[Block]), _) => - val block: Block = generateNextBlock(prevHistory) - prevHistory - .append(block.header) - .right - .get - ._1 - .append(block.payload) - .right - .get - ._1 - .reportModifierIsValid(block) -> (block +: blocks) - } - - var tmpH = history3_15norm - logger.info(s"\n\n\n\nApplying fork to normal\n\n\n\n\n") - blocks4_fork.reverse.foreach { nextBlock => - val a = tmpH.append(nextBlock.header) - logger.info(s"after forkapp header: ${a}") - tmpH = a.right.get._1 - } - - blocks4_fork.reverse.foreach { nextBlock => - val a = tmpH.append(nextBlock.payload) - logger.info(s"after forkapp payload: ${a}") - tmpH = a.right.get._1 - logger.info(s"tmpH.getBestHeader -> ${tmpH.getBestHeader}") - logger.info(s"tmpH.getBestBlock -> ${tmpH.getBestBlock}") - } - } - } -} +//package encry.view +// +//import com.typesafe.scalalogging.StrictLogging +//import encry.modifiers.InstanceFactory +//import encry.settings.TestNetSettings +//import encry.view.history.History +//import org.encryfoundation.common.modifiers.history.Block +//import org.scalatest.{ BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike } +// +//class QWE +// extends WordSpecLike +// with Matchers +// with InstanceFactory +// with BeforeAndAfterAll +// with OneInstancePerTest +// with TestNetSettings +// with StrictLogging { +// +// "qwer" should { +// "qr" in { +// val (history1_10, history2_10, _) = (0 until 10).foldLeft( +// generateDummyHistory(testNetSettings), +// generateDummyHistory(testNetSettings), +// List.empty[Block] +// ) { +// case ((prevHistory1, prevHistory2, blocks: List[Block]), _) => +// val block: Block = generateNextBlock(prevHistory1) +// val a = prevHistory1 +// .append(block.header) +// .right +// .get +// ._1 +// .append(block.payload) +// .right +// .get +// ._1 +// .reportModifierIsValid(block) +// val b = prevHistory2 +// .append(block.header) +// .right +// .get +// ._1 +// .append(block.payload) +// .right +// .get +// ._1 +// .reportModifierIsValid(block) +// (a, b, (block +: blocks)) +// } +// logger.info(s"\n\n\n\nStart processing 1 fork blocks\n\n\n\n\n") +// val (history3_15norm, blocksNorm15) = (0 until 5).foldLeft(history1_10, List.empty[Block]) { +// case ((prevHistory, blocks: List[Block]), _) => +// val block: Block = generateNextBlock(prevHistory) +// prevHistory +// .append(block.header) +// .right +// .get +// ._1 +// .append(block.payload) +// .right +// .get +// ._1 +// .reportModifierIsValid(block) -> (block +: blocks) +// } +// logger.info(s"\n\n\n\nStart processing 2 blocks\n\n\n\n\n") +// val (h4_20, blocks4_fork) = (0 until 10).foldLeft(history2_10, List.empty[Block]) { +// case ((prevHistory, blocks: List[Block]), _) => +// val block: Block = generateNextBlock(prevHistory) +// prevHistory +// .append(block.header) +// .right +// .get +// ._1 +// .append(block.payload) +// .right +// .get +// ._1 +// .reportModifierIsValid(block) -> (block +: blocks) +// } +// +// var tmpH = history3_15norm +// logger.info(s"\n\n\n\nApplying fork to normal\n\n\n\n\n") +// blocks4_fork.reverse.foreach { nextBlock => +// val a = tmpH.append(nextBlock.header) +// logger.info(s"after forkapp header: ${a}") +// tmpH = a.right.get._1 +// } +// +// blocks4_fork.reverse.foreach { nextBlock => +// val a = tmpH.append(nextBlock.payload) +// logger.info(s"after forkapp payload: ${a}") +// tmpH = a.right.get._1 +// logger.info(s"tmpH.getBestHeader -> ${tmpH.getBestHeader}") +// logger.info(s"tmpH.getBestBlock -> ${tmpH.getBestBlock}") +// } +// } +// } +//} diff --git a/src/test/scala/encry/view/wallet/WalletSpec.scala b/src/test/scala/encry/view/wallet/WalletSpec.scala index 97a0ea31d0..f1cdd61361 100644 --- a/src/test/scala/encry/view/wallet/WalletSpec.scala +++ b/src/test/scala/encry/view/wallet/WalletSpec.scala @@ -110,66 +110,66 @@ class WalletSpec extends PropSpec with Matchers with InstanceFactory with EncryG wallet.getBalances.foldLeft(0L)(_ + _._2) shouldEqual txsQty * Props.boxValue } - property("Balance count (intrinsic coins + tokens) for multiple accounts") { - - val dataBox = DataBox(EncryProposition.heightLocked(Height @@ 10), 0L, Array.emptyByteArray) - - import encry.view.state.avlTree.utils.implicits.Instances._ - - val rootNode: LeafNode[StorageKey, StorageValue] = - LeafNode(StorageKey @@ Array(DataBox.`modifierTypeId`), StorageValue @@ DataBoxSerializer.toBytes(dataBox)) - val storageMock = mock[VersionalStorage] - val anotherDir: File = FileHelper.getRandomTempDir - val levelDb: DB = LevelDbFactory.factory.open(anotherDir, new Options) - val rootNodesStorage = RootNodesStorage[StorageKey, StorageValue](levelDb, 10, anotherDir) - val tree = AvlTree(rootNode, storageMock, rootNodesStorage) - val stateMock = mock[UtxoStateReader](RETURNS_DEEP_STUBS) - when(stateMock.tree).thenReturn(tree) - - val seed = "another accuse index island little scissors insect little absurd island keep valid" - val alsoSeed = "another accuse index island little island absurd little absurd scissors keep valid" - - val dir = FileHelper.getRandomTempDir - - val aM = AccountManager.init( - "another accuse index island little scissors insect little insect island keep valid", - "encry", - settings.copy(directory = dir.getAbsolutePath)) - - val txsQty: Int = 4 - - val blockHeader: Header = genHeader - - val wallet: EncryWallet = EncryWallet.readOrGenerate( - FileHelper.getRandomTempDir, - EncryWallet.getKeysDir(settings.copy(directory = dir.getAbsolutePath)), - settings.copy(directory = dir.getAbsolutePath) - ) - .addAccount(seed, settings.wallet.map(_.password).get, stateMock).toOption.get - - val keyManagerOne = wallet.accountManagers.head - - val keyManagerTwo = wallet.accountManagers(1) - - val extraAcc = keyManagerTwo.createAccount(Some(alsoSeed)) - - val validTxs1: Seq[Transaction] = genValidPaymentTxsToAddr(txsQty, keyManagerOne.mandatoryAccount.publicImage.address.address) - val validTxs2: Seq[Transaction] = genValidPaymentTxsToAddr(txsQty - 1, keyManagerTwo.mandatoryAccount.publicImage.address.address) - val validTxs3: Seq[Transaction] = genValidPaymentTxsToAddr(txsQty - 2, extraAcc.publicImage.address.address) - val validTxstoOther: Seq[Transaction] = genValidPaymentTxsToAddr(txsQty - 3, randomAddress) - - val blockPayload: Payload = Payload(ModifierId @@ Array.fill(32)(19: Byte), validTxs1 ++ validTxs2 ++ validTxs3 ++ validTxstoOther) - - val block: Block = Block(blockHeader, blockPayload) - - wallet.scanPersistent(block) - - val addr1 = Algos.encode(keyManagerOne.mandatoryAccount.publicKeyBytes) - val addr2 = Algos.encode(keyManagerTwo.mandatoryAccount.publicKeyBytes) - val addr3 = Algos.encode(extraAcc.publicKeyBytes) - - wallet.getBalances.filter(_._1._1 == addr1).map(_._2).sum shouldEqual txsQty * Props.boxValue - wallet.getBalances.filter(_._1._1 == addr2).map(_._2).sum shouldEqual (txsQty - 1) * Props.boxValue - wallet.getBalances.filter(_._1._1 == addr3).map(_._2).sum shouldEqual (txsQty - 2) * Props.boxValue - } +// property("Balance count (intrinsic coins + tokens) for multiple accounts") { +// +// val dataBox = DataBox(EncryProposition.heightLocked(Height @@ 10), 0L, Array.emptyByteArray) +// +// import encry.view.state.avlTree.utils.implicits.Instances._ +// +// val rootNode: LeafNode[StorageKey, StorageValue] = +// LeafNode(StorageKey @@ Array(DataBox.`modifierTypeId`), StorageValue @@ DataBoxSerializer.toBytes(dataBox)) +// val storageMock = mock[VersionalStorage] +// val anotherDir: File = FileHelper.getRandomTempDir +// val levelDb: DB = LevelDbFactory.factory.open(anotherDir, new Options) +// val rootNodesStorage = RootNodesStorage[StorageKey, StorageValue](levelDb, 10, anotherDir) +// val tree = AvlTree(rootNode, storageMock, rootNodesStorage) +// val stateMock = mock[UtxoStateReader](RETURNS_DEEP_STUBS) +// when(stateMock.tree).thenReturn(tree) +// +// val seed = "another accuse index island little scissors insect little absurd island keep valid" +// val alsoSeed = "another accuse index island little island absurd little absurd scissors keep valid" +// +// val dir = FileHelper.getRandomTempDir +// +// val aM = AccountManager.init( +// "another accuse index island little scissors insect little insect island keep valid", +// "encry", +// settings.copy(directory = dir.getAbsolutePath)) +// +// val txsQty: Int = 4 +// +// val blockHeader: Header = genHeader +// +// val wallet: EncryWallet = EncryWallet.readOrGenerate( +// FileHelper.getRandomTempDir, +// EncryWallet.getKeysDir(settings.copy(directory = dir.getAbsolutePath)), +// settings.copy(directory = dir.getAbsolutePath) +// ) +// .addAccount(seed, settings.wallet.map(_.password).get, stateMock).toOption.get +// +// val keyManagerOne = wallet.accountManagers.head +// +// val keyManagerTwo = wallet.accountManagers(1) +// +// val extraAcc = keyManagerTwo.createAccount(Some(alsoSeed)) +// +// val validTxs1: Seq[Transaction] = genValidPaymentTxsToAddr(txsQty, keyManagerOne.mandatoryAccount.publicImage.address.address) +// val validTxs2: Seq[Transaction] = genValidPaymentTxsToAddr(txsQty - 1, keyManagerTwo.mandatoryAccount.publicImage.address.address) +// val validTxs3: Seq[Transaction] = genValidPaymentTxsToAddr(txsQty - 2, extraAcc.publicImage.address.address) +// val validTxstoOther: Seq[Transaction] = genValidPaymentTxsToAddr(txsQty - 3, randomAddress) +// +// val blockPayload: Payload = Payload(ModifierId @@ Array.fill(32)(19: Byte), validTxs1 ++ validTxs2 ++ validTxs3 ++ validTxstoOther) +// +// val block: Block = Block(blockHeader, blockPayload) +// +// wallet.scanPersistent(block) +// +// val addr1 = Algos.encode(keyManagerOne.mandatoryAccount.publicKeyBytes) +// val addr2 = Algos.encode(keyManagerTwo.mandatoryAccount.publicKeyBytes) +// val addr3 = Algos.encode(extraAcc.publicKeyBytes) +// +// wallet.getBalances.filter(_._1._1 == addr1).map(_._2).sum shouldEqual txsQty * Props.boxValue +// wallet.getBalances.filter(_._1._1 == addr2).map(_._2).sum shouldEqual (txsQty - 1) * Props.boxValue +// wallet.getBalances.filter(_._1._1 == addr3).map(_._2).sum shouldEqual (txsQty - 2) * Props.boxValue +// } } \ No newline at end of file From 36cb65eec0e5de1597ab450500aab9c542daaa64 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 24 Mar 2020 13:20:34 +0300 Subject: [PATCH 143/177] history fixed --- .../encry/api/http/routes/InfoApiRoute.scala | 2 +- .../scala/encry/nvg/IntermediaryNVHView.scala | 1 + src/main/scala/encry/nvg/NVHHistory.scala | 57 ++++++++++++++----- src/main/scala/encry/nvg/NVHState.scala | 10 ++-- .../scala/encry/view/history/History.scala | 4 +- 5 files changed, 52 insertions(+), 22 deletions(-) diff --git a/src/main/scala/encry/api/http/routes/InfoApiRoute.scala b/src/main/scala/encry/api/http/routes/InfoApiRoute.scala index 7672b8a31a..e4cbc92acc 100644 --- a/src/main/scala/encry/api/http/routes/InfoApiRoute.scala +++ b/src/main/scala/encry/api/http/routes/InfoApiRoute.scala @@ -46,7 +46,7 @@ object InfoApiRoute { constants: Constants ): Json = { val stateVersion: Option[String] = readers.s.map(_.version).map(Algos.encode) - val stateRoot: Option[String] = readers.s.map(_.tree.rootHash).map(Algos.encode) + val stateRoot: Option[String] = None//readers.s.map(_.tree.rootHash).map(Algos.encode) val prevFullHeaderId: String = block.map(b => Algos.encode(b.header.parentId)).getOrElse("") InfoApi( nodeName, diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index e36dd2d1b3..cb8c98a010 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -32,6 +32,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, var historyReader: HistoryReader = HistoryReader.empty + println("init middle") val historyRef: ActorRef = context.actorOf(NVHHistory.props(ntp, settings)) var isModifierProcessingInProgress: Boolean = false diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index 32398aedb3..7a09ea2603 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -2,38 +2,48 @@ package encry.nvg import java.io.File -import akka.actor.{Actor, Props} +import akka.actor.{ Actor, Props } import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging +import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.RegisterHistory -import encry.nvg.IntermediaryNVHView.{InitGenesisHistory, ModifierToAppend} -import encry.nvg.NVHHistory.{ModifierAppliedToHistory, ProgressInfoForState} +import encry.nvg.IntermediaryNVHView.{ InitGenesisHistory, ModifierToAppend } +import encry.nvg.NVHHistory.{ ModifierAppliedToHistory, ProgressInfoForState } import encry.nvg.NVHState.StateAction -import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification} +import encry.nvg.NodeViewHolder.{ + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification +} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.NetworkTimeProvider import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.history.History.HistoryUpdateInfoAcc -import encry.view.history.{History, HistoryReader} +import encry.view.history.{ History, HistoryReader } import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } -class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends Actor with StrictLogging { +class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) + extends Actor + with StrictLogging + with AutoCloseable { - logger.info("start here!") - var history: History = initializeHistory + println("start here!") + var history: History = initializeHistory.getOrElse(genesis) var lastProgressInfo: ProgressInfo = ProgressInfo(none, Seq.empty, Seq.empty, none) context.parent ! RegisterHistory(HistoryReader(history)) + override def postStop(): Unit = println("stop!") + override def receive: Receive = { case ModifierToAppend(mod, isLocallyGenerated) if !history.isModifierDefined(mod.id) => val startProcessingTime: Long = System.currentTimeMillis() @@ -112,7 +122,8 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends A case InitGenesisHistory => logger.info("Init in InitGenesisHistory") - history = initializeHistory + history.close() + history = genesis } def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = none): Unit = @@ -130,19 +141,35 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends A ) } - def initializeHistory: History = + def initializeHistory: Option[History] = try { val history: History = History.readOrGenerate(settings, ntp) history.updateIdsForSyncInfo() logger.info(s"History best block height: ${history.getBestBlockHeight}") logger.info(s"History best header height: ${history.getBestHeaderHeight}") - history + Some(history) } catch { case error: Throwable => logger.info(s"During history initialization error ${error.getMessage} has happened.") - new File(settings.directory).listFiles.foreach(FileUtils.cleanDirectory) - initializeHistory + None + } + + def genesis: History = + try { + new File(s"${settings.directory}/history/").listFiles.foreach(FileUtils.cleanDirectory) + val history: History = History.readOrGenerate(settings, ntp) + history.updateIdsForSyncInfo() + logger.info(s"History best block height: ${history.getBestBlockHeight}") + logger.info(s"History best header height: ${history.getBestHeaderHeight}") + history + } catch { + case error: Throwable => + println("123") + EncryApp.forceStopApplication(1, + s"During genesis history initialization error ${error.getMessage} has happened.") } + + override def close(): Unit = history.close() } object NVHHistory { diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index 04ca2df2e1..89b3f0b081 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -84,11 +84,11 @@ object NVHState extends StrictLogging { case class TreeChunks(chunks: List[SnapshotChunk]) extends StateAction } - def restoreProps(settings: EncryAppSettings, - historyReader: HistoryReader, - influxRef: Option[ActorRef]): Props = { - - } +// def restoreProps(settings: EncryAppSettings, +// historyReader: HistoryReader, +// influxRef: Option[ActorRef]): Props = { +// +// } //genesis state def genesisProps(settings: EncryAppSettings, influxRef: Option[ActorRef]): Props = { diff --git a/src/main/scala/encry/view/history/History.scala b/src/main/scala/encry/view/history/History.scala index 57fafb1eec..ea35c55ee3 100644 --- a/src/main/scala/encry/view/history/History.scala +++ b/src/main/scala/encry/view/history/History.scala @@ -160,7 +160,9 @@ trait History extends HistoryModifiersValidator with AutoCloseable { } } - override def close(): Unit = historyStorage.close() + override def close(): Unit = { + historyStorage.close() + } def closeStorage(): Unit = historyStorage.close() } From 7f526dd453c551ecd422f2c0632f3c6f8a131231 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 24 Mar 2020 14:37:08 +0300 Subject: [PATCH 144/177] more logging --- .../encry/api/http/routes/InfoApiRoute.scala | 2 +- .../scala/encry/nvg/IntermediaryNVHView.scala | 23 +-- src/main/scala/encry/nvg/NVHState.scala | 162 +++++++----------- 3 files changed, 64 insertions(+), 123 deletions(-) diff --git a/src/main/scala/encry/api/http/routes/InfoApiRoute.scala b/src/main/scala/encry/api/http/routes/InfoApiRoute.scala index e4cbc92acc..25d60b713c 100644 --- a/src/main/scala/encry/api/http/routes/InfoApiRoute.scala +++ b/src/main/scala/encry/api/http/routes/InfoApiRoute.scala @@ -46,7 +46,7 @@ object InfoApiRoute { constants: Constants ): Json = { val stateVersion: Option[String] = readers.s.map(_.version).map(Algos.encode) - val stateRoot: Option[String] = None//readers.s.map(_.tree.rootHash).map(Algos.encode) + val stateRoot: Option[String] = readers.s.map(_.rootHash).map(Algos.encode) val prevFullHeaderId: String = block.map(b => Algos.encode(b.header.parentId)).getOrElse("") InfoApi( nodeName, diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index cb8c98a010..905ce95821 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -32,7 +32,6 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, var historyReader: HistoryReader = HistoryReader.empty - println("init middle") val historyRef: ActorRef = context.actorOf(NVHHistory.props(ntp, settings)) var isModifierProcessingInProgress: Boolean = false @@ -48,14 +47,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, historyReader = reader logger.info(s"NodeViewParent actor got init history. Going to init state actor.") context.become(awaitingViewActors(Some(sender()), state), discardOld = true) - context.actorOf( - NVHState - .restoreConsistentStateProps(settings, reader, influx) - .getOrElse { - historyRef ! InitGenesisHistory - NVHState.genesisProps(settings, influx) - } - ) + context.actorOf(NVHState.restoreProps(settings, reader, influx)) case RegisterHistory(_) => context.become(viewReceive(sender(), state.get, stateReader.get)) case RegisterState(reader) if history.isEmpty => @@ -95,20 +87,11 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, isLocallyGenerated = false ) if (!isModifierProcessingInProgress) getNextModifier() - case ModifierAppliedToHistory => isModifierProcessingInProgress = false; getNextModifier() + case ModifierAppliedToHistory => isModifierProcessingInProgress = false; getNextModifier() case msg: ProgressInfoForState if msg.pi.chainSwitchingNeeded && msg.pi.branchPoint.exists(point => !stateReader.version.sameElements(point))=> context.become(viewReceive( history, - context.actorOf( - NVHState.rollbackProps( - settings, - influx, - stateReader.safePointHeight, - msg.pi.branchPoint.get, - historyReader, - settings.constants - ) - ), + state, stateReader )) case msg: ProgressInfoForState => diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index 89b3f0b081..25bdf25f4f 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -16,11 +16,12 @@ import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, Vers import encry.storage.{RootNodesStorage, VersionalStorage} import encry.utils.CoreTaggedTypes.VersionTag import encry.view.NodeViewErrors.ModifierApplyError -import encry.view.history.HistoryReader +import encry.view.history.{History, HistoryReader} import encry.view.state.{UtxoState, UtxoStateReader} import encry.view.state.avlTree.AvlTree import encry.view.state.avlTree.utils.implicits.Instances._ import io.iohk.iodb.LSMStore +import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{Block, Header} import org.encryfoundation.common.utils.Algos @@ -30,9 +31,11 @@ import org.iq80.leveldb.Options import scala.util.{Failure, Success, Try} -class NVHState(influxRef: Option[ActorRef], var state: UtxoState, settings: EncryAppSettings) +class NVHState(influxRef: Option[ActorRef], var historyReader: HistoryReader, settings: EncryAppSettings) extends Actor with StrictLogging with AutoCloseable { + var state: UtxoState = restoreState(settings, historyReader, influxRef).getOrElse(genesis(settings, influxRef)) + override def preStart(): Unit = context.parent ! RegisterState(UtxoStateReader(state)) override def receive: Receive = { @@ -65,114 +68,38 @@ class NVHState(influxRef: Option[ActorRef], var state: UtxoState, settings: Encr state.close() } - override def close(): Unit = state.close() -} - -object NVHState extends StrictLogging { - - import encry.view.state.avlTree.utils.implicits.Instances._ - - sealed trait StateAction - object StateAction { - case class ModifierApplied(modifierId: PersistentModifier) extends StateAction - case class Rollback(branchPoint: ModifierId) extends StateAction - case class ApplyFailed(modifierId: PersistentModifier, errs: List[ModifierApplyError]) extends StateAction - case class ApplyModifier(modifier: PersistentModifier, - saveRootNodesFlag: Boolean, - isFullChainSynced: Boolean) extends StateAction - case object CreateTreeChunks extends StateAction - case class TreeChunks(chunks: List[SnapshotChunk]) extends StateAction - } - -// def restoreProps(settings: EncryAppSettings, -// historyReader: HistoryReader, -// influxRef: Option[ActorRef]): Props = { -// -// } - - //genesis state - def genesisProps(settings: EncryAppSettings, influxRef: Option[ActorRef]): Props = { + def genesis(settings: EncryAppSettings, influxRef: Option[ActorRef]): UtxoState = { logger.info("Init genesis!") val stateDir: File = UtxoState.getStateDir(settings) - stateDir.listFiles.foreach(_.delete()) stateDir.mkdir() val rootsDir: File = UtxoState.getRootsDir(settings) - rootsDir.listFiles.foreach(_.delete()) rootsDir.mkdir() - val state: UtxoState = UtxoState.genesis(stateDir, rootsDir, settings, influxRef) - Props(new NVHState(influxRef, state, settings)) - } - - //restoreConsistentState - def restoreConsistentStateProps(settings: EncryAppSettings, - historyReader: HistoryReader, - influxRef: Option[ActorRef]): Try[Props] = { - Try { - val stateDir: File = UtxoState.getStateDir(settings) - stateDir.mkdirs() - val rootsDir: File = UtxoState.getRootsDir(settings) - rootsDir.mkdir() - logger.info("init dirs") - val state: UtxoState = restoreConsistentState ( - UtxoState.create(stateDir, rootsDir, settings, influxRef), - historyReader, - influxRef, - settings - ) - Props(new NVHState(influxRef, state, settings)) - } match { - case a: Success[Props] => a - case err: Failure[Props] => - logger.info(s"Err: ${err.exception}") - err - } + UtxoState.genesis(stateDir, rootsDir, settings, influxRef) } - //rollback - def rollbackProps(settings: EncryAppSettings, - influxRef: Option[ActorRef], - safePointHeight: Int, - branchPoint: ModifierId, - historyReader: HistoryReader, - constants: Constants): Props = { - val branchPointHeight: Int = historyReader.getHeaderById(ModifierId !@@ branchPoint).get.height - val additionalBlocks: List[Block] = - (safePointHeight + 1 to branchPointHeight).foldLeft(List.empty[Block]) { - case (blocks: List[Block], height: Int) => - val headerAtHeight: Header = historyReader.getBestHeaderAtHeight(height).get - val blockAtHeight: Block = historyReader.getBlockByHeader(headerAtHeight).get - blocks :+ blockAtHeight + def restoreState(settings: EncryAppSettings, + historyReader: HistoryReader, + influxRef: Option[ActorRef]): Option[UtxoState] = + if (History.getHistoryIndexDir(settings).listFiles.nonEmpty) { + Try { + val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdirs() + val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() + restoreConsistentState( + UtxoState.create(stateDir, rootsDir, settings, influxRef), + historyReader, + influxRef, + settings + ) + } match { + case fail: Failure[UtxoState] => + logger.info(s"${fail.exception.getMessage} during state restore. Recover from Modifiers holder!") + new File(settings.directory).listFiles.foreach(dir => FileUtils.cleanDirectory(dir)) + fail.toOption + case res: Success[UtxoState] => res.toOption } - val dir: File = UtxoState.getStateDir(settings) - dir.mkdirs() - dir.listFiles.foreach(_.delete()) - val stateDir: File = UtxoState.getStateDir(settings) - stateDir.mkdirs() - val rootsDir: File = UtxoState.getRootsDir(settings) - rootsDir.mkdir() - val versionalStorage = settings.storage.state match { - case VersionalStorage.IODB => - logger.info("Init state with iodb storage") - IODBWrapper(new LSMStore(stateDir, keepVersions = settings.constants.DefaultKeepVersions)) - case VersionalStorage.LevelDB => - logger.info("Init state with levelDB storage") - val levelDBInit = LevelDbFactory.factory.open(stateDir, new Options) - VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settings.levelDB.copy(keySize = 33), keySize = 33)) - } - val rootStorage = { - val levelDBInit = LevelDbFactory.factory.open(rootsDir, new Options) - RootNodesStorage[StorageKey, StorageValue](levelDBInit, settings.constants.MaxRollbackDepth, rootsDir) - } - val state: UtxoState = UtxoState.rollbackTo( - VersionTag !@@ branchPoint, - additionalBlocks, - versionalStorage, - rootStorage, - constants, - influxRef - ).get - Props(new NVHState(influxRef, state, settings)) - } + } else none def restoreConsistentState(stateIn: UtxoState, history: HistoryReader, @@ -222,4 +149,35 @@ object NVHState extends StrictLogging { rootsDir.mkdir() UtxoState.create(stateDir, rootsDir, settings, influxRef) } + + override def close(): Unit = state.close() +} + +object NVHState extends StrictLogging { + + import encry.view.state.avlTree.utils.implicits.Instances._ + + sealed trait StateAction + object StateAction { + case class ModifierApplied(modifierId: PersistentModifier) extends StateAction + case class Rollback(branchPoint: ModifierId) extends StateAction + case class ApplyFailed(modifierId: PersistentModifier, errs: List[ModifierApplyError]) extends StateAction + case class ApplyModifier(modifier: PersistentModifier, + saveRootNodesFlag: Boolean, + isFullChainSynced: Boolean) extends StateAction + case object CreateTreeChunks extends StateAction + case class TreeChunks(chunks: List[SnapshotChunk]) extends StateAction + } + + def restoreProps(settings: EncryAppSettings, + historyReader: HistoryReader, + influxRef: Option[ActorRef]): Props = { + Props( + new NVHState( + influxRef, + historyReader, + settings + ) + ) + } } From 727ce4c6c5be6dd6374823cd0692d72927ce98d9 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 24 Mar 2020 15:06:52 +0300 Subject: [PATCH 145/177] fix state starting --- src/main/scala/encry/nvg/NVHState.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index 25bdf25f4f..9fb015b4eb 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -27,6 +27,7 @@ import org.encryfoundation.common.modifiers.history.{Block, Header} import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId} import org.encryfoundation.common.utils.constants.Constants +import org.encryfoundation.prismlang.core.Constants import org.iq80.leveldb.Options import scala.util.{Failure, Success, Try} @@ -80,7 +81,7 @@ class NVHState(influxRef: Option[ActorRef], var historyReader: HistoryReader, se def restoreState(settings: EncryAppSettings, historyReader: HistoryReader, influxRef: Option[ActorRef]): Option[UtxoState] = - if (History.getHistoryIndexDir(settings).listFiles.nonEmpty) { + if (historyReader.getBestHeaderHeight != settings.constants.PreGenesisHeight) { Try { val stateDir: File = UtxoState.getStateDir(settings) stateDir.mkdirs() From 5899b6fc088cb9371183b9abcb61e3f002b46959 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 24 Mar 2020 15:39:13 +0300 Subject: [PATCH 146/177] improve stateReader --- src/main/scala/encry/view/state/UtxoState.scala | 3 +++ src/main/scala/encry/view/state/UtxoStateReader.scala | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/src/main/scala/encry/view/state/UtxoState.scala b/src/main/scala/encry/view/state/UtxoState.scala index d3b7aa6734..0b1dcab45b 100644 --- a/src/main/scala/encry/view/state/UtxoState.scala +++ b/src/main/scala/encry/view/state/UtxoState.scala @@ -233,6 +233,9 @@ final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], override def avlStorage: VersionalStorage = tree.avlStorage override def rootHash: Array[Byte] = tree.rootHash + + override def getOperationsRootHash(toInsert: List[(StorageKey, StorageValue)], + toDelete: List[StorageKey]): Array[Byte] = tree.getOperationsRootHash(toInsert, toDelete) } object UtxoState extends StrictLogging { diff --git a/src/main/scala/encry/view/state/UtxoStateReader.scala b/src/main/scala/encry/view/state/UtxoStateReader.scala index 42c8c15655..989bcfdfd6 100644 --- a/src/main/scala/encry/view/state/UtxoStateReader.scala +++ b/src/main/scala/encry/view/state/UtxoStateReader.scala @@ -25,6 +25,9 @@ trait UtxoStateReader { def stateSafePointHeight: Height + def getOperationsRootHash(toInsert: List[(StorageKey, StorageValue)], + toDelete: List[StorageKey]): Array[Byte] + def boxById(boxId: ADKey): Option[EncryBaseBox] def boxesByIds(ids: Seq[ADKey]): Seq[EncryBaseBox] @@ -46,5 +49,7 @@ object UtxoStateReader { override def avlStorage: VersionalStorage = state.avlStorage override def rootHash: Array[Byte] = state.rootHash override def safePointHeight: Height = state.safePointHeight + override def getOperationsRootHash(toInsert: List[(StorageKey, StorageValue)], + toDelete: List[StorageKey]): Array[Byte] = state.getOperationsRootHash(toInsert, toDelete) } } \ No newline at end of file From 7fc476ed985eeac9b3a5867aead91c43bbf96859 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 24 Mar 2020 15:56:44 +0300 Subject: [PATCH 147/177] fix handling state reader at inter nvh view --- src/main/scala/encry/nvg/IntermediaryNVHView.scala | 2 +- src/main/scala/encry/view/state/UtxoState.scala | 2 +- src/main/scala/encry/view/state/UtxoStateReader.scala | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index 905ce95821..d740d320c0 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -50,7 +50,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, context.actorOf(NVHState.restoreProps(settings, reader, influx)) case RegisterHistory(_) => context.become(viewReceive(sender(), state.get, stateReader.get)) - case RegisterState(reader) if history.isEmpty => + case RegisterState(reader) => context.become(awaitingViewActors(history, Some(sender()), Some(reader)), discardOld = true) case RegisterHistory => context.become(viewReceive(history.get, sender(), stateReader.get)) diff --git a/src/main/scala/encry/view/state/UtxoState.scala b/src/main/scala/encry/view/state/UtxoState.scala index 0b1dcab45b..d00e7caf5f 100644 --- a/src/main/scala/encry/view/state/UtxoState.scala +++ b/src/main/scala/encry/view/state/UtxoState.scala @@ -235,7 +235,7 @@ final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], override def rootHash: Array[Byte] = tree.rootHash override def getOperationsRootHash(toInsert: List[(StorageKey, StorageValue)], - toDelete: List[StorageKey]): Array[Byte] = tree.getOperationsRootHash(toInsert, toDelete) + toDelete: List[StorageKey]): Try[Array[Byte]] = tree.getOperationsRootHash(toInsert, toDelete) } object UtxoState extends StrictLogging { diff --git a/src/main/scala/encry/view/state/UtxoStateReader.scala b/src/main/scala/encry/view/state/UtxoStateReader.scala index 989bcfdfd6..04566d1283 100644 --- a/src/main/scala/encry/view/state/UtxoStateReader.scala +++ b/src/main/scala/encry/view/state/UtxoStateReader.scala @@ -10,6 +10,8 @@ import org.encryfoundation.common.modifiers.state.box.EncryBaseBox import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ADKey, Height} +import scala.util.Try + trait UtxoStateReader { implicit val hf: Algos.HF = Algos.hash @@ -26,7 +28,7 @@ trait UtxoStateReader { def stateSafePointHeight: Height def getOperationsRootHash(toInsert: List[(StorageKey, StorageValue)], - toDelete: List[StorageKey]): Array[Byte] + toDelete: List[StorageKey]): Try[Array[Byte]] def boxById(boxId: ADKey): Option[EncryBaseBox] @@ -50,6 +52,6 @@ object UtxoStateReader { override def rootHash: Array[Byte] = state.rootHash override def safePointHeight: Height = state.safePointHeight override def getOperationsRootHash(toInsert: List[(StorageKey, StorageValue)], - toDelete: List[StorageKey]): Array[Byte] = state.getOperationsRootHash(toInsert, toDelete) + toDelete: List[StorageKey]): Try[Array[Byte]] = state.getOperationsRootHash(toInsert, toDelete) } } \ No newline at end of file From e68da83d14d51c15360d344e553f302d5e47e304 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 24 Mar 2020 16:14:39 +0300 Subject: [PATCH 148/177] fix context switching on inter nvh --- src/main/scala/encry/nvg/IntermediaryNVHView.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index d740d320c0..fe6897d96f 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -51,9 +51,10 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, case RegisterHistory(_) => context.become(viewReceive(sender(), state.get, stateReader.get)) case RegisterState(reader) => - context.become(awaitingViewActors(history, Some(sender()), Some(reader)), discardOld = true) + context.become(viewReceive(history.get, sender(), reader), discardOld = true) case RegisterHistory => context.become(viewReceive(history.get, sender(), stateReader.get)) + case msg => logger.info(s"Receive strange: ${msg} on inter nvh") } def viewReceive(history: ActorRef, state: ActorRef, stateReader: UtxoStateReader): Receive = { From d525f1b072ca35e3f37e9372dda7884019bcfaba Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 24 Mar 2020 16:47:08 +0300 Subject: [PATCH 149/177] fix modifier applying --- src/main/scala/encry/nvg/IntermediaryNVHView.scala | 1 + src/main/scala/encry/nvg/NVHHistory.scala | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index fe6897d96f..6eee7a2ec7 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -21,6 +21,7 @@ import encry.view.state.UtxoStateReader import io.iohk.iodb.ByteArrayWrapper import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{Block, Header} +import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId import scala.collection.IndexedSeq diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index 7a09ea2603..1925f2f755 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -107,7 +107,7 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) case _: Payload => true case _ => false }) context.parent ! ModifierAppendedToState(success = true) - + context.parent ! ModifierAppliedToHistory case StateAction.ApplyFailed(mod, e) => val (newHistory: History, progressInfo: ProgressInfo) = history.reportModifierIsInvalid(mod) context.parent ! SemanticallyFailedModification(mod, e) From 6b6816c699d5c89c733f4f39d8ae01df6e6a1489 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 24 Mar 2020 17:20:07 +0300 Subject: [PATCH 150/177] fix history reader propagation --- src/main/scala/encry/nvg/IntermediaryNVH.scala | 4 ++-- src/main/scala/encry/nvg/NodeViewNMProcessor.scala | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVH.scala b/src/main/scala/encry/nvg/IntermediaryNVH.scala index 846188f071..38c5e8fef1 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVH.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVH.scala @@ -97,8 +97,8 @@ class IntermediaryNVH( case msg @ DataFromPeer(_: ResponseManifestMessage, _) => snapshotProcessor.foreach(_ ! msg) case msg @ DataFromPeer(_: RequestChunkMessage, _) => snapshotProcessor.foreach(_ ! msg) case msg @ DataFromPeer(_: ResponseChunkMessage, _) => snapshotProcessor.foreach(_ ! msg) - case msg @ UpdateHistoryReader(newReader: HistoryReader) => - historyReader = newReader + case msg: HistoryReader => + historyReader = msg networkMessagesProcessor ! msg dataHolderRef ! msg case msg: LocallyGeneratedModifier => nodeViewHolder ! msg diff --git a/src/main/scala/encry/nvg/NodeViewNMProcessor.scala b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala index 824dee4acc..e00aa12601 100644 --- a/src/main/scala/encry/nvg/NodeViewNMProcessor.scala +++ b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala @@ -54,7 +54,7 @@ class NodeViewNMProcessor(settings: EncryAppSettings) extends Actor with StrictL ) def workingCycle(modifiersRequester: Option[Cancellable]): Receive = { - case UpdateHistoryReader(newReader: HistoryReader) => historyReader = newReader + case msg: HistoryReader => historyReader = msg //todo possible way to call CheckPayloadsToDownload case SemanticallySuccessfulModifier(block: Block) if historyReader.isFullChainSynced => From d667e60808aa5103e6af06a8c0ef715629af2047 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 24 Mar 2020 20:20:58 +0300 Subject: [PATCH 151/177] miner works incorrect --- src/main/scala/encry/Starter.scala | 2 +- src/main/scala/encry/local/miner/Miner.scala | 50 ++++++---- .../scala/encry/nvg/IntermediaryNVHView.scala | 99 +++++++++++++------ src/main/scala/encry/nvg/NVHHistory.scala | 90 ++++++++++------- .../encry/view/history/HistoryReader.scala | 10 +- .../encry/view/state/UtxoStateReader.scala | 10 ++ .../encry/view/wallet/WalletReader.scala | 14 +++ 7 files changed, 192 insertions(+), 83 deletions(-) create mode 100644 src/main/scala/encry/view/wallet/WalletReader.scala diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index bebdd2fc7c..135a9b3361 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -426,7 +426,7 @@ class Starter(settings: EncryAppSettings, ) val miner: ActorRef = context.system.actorOf(Miner.props(dataHolderForApi, memoryPool, nvhRouter, influxRef, newSettings), "miner") - if (newSettings.node.mining) miner ! StartMining + //if (newSettings.node.mining) miner ! StartMining if (newSettings.node.useCli) { context.system .actorOf(ConsoleListener.props(newSettings, dataHolderForApi, nodeId, timeProvider), "cliListener") diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index 7a59b31a61..5f78298532 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -3,7 +3,7 @@ package encry.local.miner import java.text.SimpleDateFormat import java.util.Date -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{Actor, ActorRef, Props, Stash} import akka.util.Timeout import akka.pattern._ import com.typesafe.scalalogging.StrictLogging @@ -23,11 +23,12 @@ import encry.stats.StatsSender._ import encry.utils.NetworkTime.Time import encry.view.state.avlTree.utils.implicits.Instances._ import encry.view.NodeViewHolder.CurrentView -import encry.view.history.History +import encry.view.history.{History, HistoryReader} import encry.mpg.MemoryPool._ -import encry.view.state.UtxoState +import encry.nvg.IntermediaryNVHView.NodeViewStarted +import encry.view.state.{UtxoState, UtxoStateReader} import encry.utils.implicits.UTXO._ -import encry.view.wallet.EncryWallet +import encry.view.wallet.{EncryWallet, WalletReader} import io.circe.syntax._ import io.circe.{Encoder, Json} import org.encryfoundation.common.crypto.PrivateKey25519 @@ -45,7 +46,7 @@ class Miner(dataHolder: ActorRef, mempool: ActorRef, nvh: ActorRef, influx: Option[ActorRef], - settings: EncryAppSettings) extends Actor with StrictLogging { + settings: EncryAppSettings) extends Actor with StrictLogging with Stash { implicit val timeout: Timeout = Timeout(5.seconds) @@ -66,6 +67,7 @@ class Miner(dataHolder: ActorRef, context.system.eventStream.subscribe(self, classOf[MinerMiningCommands]) context.system.eventStream.subscribe(self, classOf[SemanticallySuccessfulModifier]) context.system.eventStream.subscribe(self, classOf[BlockchainStatus]) + context.system.eventStream.subscribe(self, classOf[NodeViewStarted]) context.system.scheduler.schedule(5.seconds, 5.seconds) { logger.info(s"data holder: $dataHolder. Context: $context") dataHolder ! UpdatingMinerStatus(MinerStatus(context.children.nonEmpty && candidateOpt.nonEmpty, candidateOpt)) @@ -79,9 +81,15 @@ class Miner(dataHolder: ActorRef, def needNewCandidate(b: Block): Boolean = !candidateOpt.flatMap(_.parentOpt).map(_.id).exists(id => Algos.encode(id) == Algos.encode(b.header.id)) - override def receive: Receive = { - logger.info(s"settings.node.mining: ${settings.node.mining}. syncingDone: ${syncingDone}") - if (settings.node.mining && syncingDone) miningEnabled else miningDisabled + override def receive: Receive = awaitNodeView + + def awaitNodeView: Receive = { + case _: NodeViewStarted => + logger.info(s"settings.node.mining: ${settings.node.mining}. syncingDone: ${syncingDone}") + if (settings.node.mining && syncingDone) { + context.self ! StartMining + context.become(miningEnabled) + } else context.become(miningDisabled) } def mining: Receive = { @@ -89,6 +97,7 @@ class Miner(dataHolder: ActorRef, killAllWorkers() self ! StartMining case StartMining if syncingDone => + println(s"got start mining from ${sender()}") for (i <- 0 until numberOfWorkers) yield context.actorOf( Props(classOf[Worker], i, numberOfWorkers, self).withDispatcher("mining-dispatcher").withMailbox("mining-mailbox")) candidateOpt match { @@ -96,10 +105,11 @@ class Miner(dataHolder: ActorRef, logger.info(s"Starting mining at ${dateFormat.format(new Date(System.currentTimeMillis()))}") context.children.foreach(_ ! NextChallenge(candidateBlock)) case None => - logger.info("Candidate is empty! Producing new candidate!") + println(s"Candidate is empty! Producing new candidate! ${timeProvider.estimatedTime}") produceCandidate() } - case StartMining => logger.info("Can't start mining because of chain is not synced!") + case StartMining => + logger.info("Can't start mining because of chain is not synced!") case DisableMining if context.children.nonEmpty => println(s"Miner -> Disable mining context.children.nonEmpty") killAllWorkers() @@ -160,7 +170,9 @@ class Miner(dataHolder: ActorRef, } def unknownMessage: Receive = { - case m => logger.debug(s"Unexpected message $m") + case m => + println(m) + logger.debug(s"Unexpected message $m") } def chainEvents: Receive = { @@ -175,11 +187,11 @@ class Miner(dataHolder: ActorRef, self ! StartMining } - def createCandidate(view: CurrentView[History, UtxoState, EncryWallet], + def createCandidate(view: CurrentView[HistoryReader, UtxoStateReader, WalletReader], txsFromMempool: List[Transaction], - bestHeaderOpt: Option[Header]): CandidateBlock = { + bestHeaderOpt: Option[Header], + timestamp: Time): CandidateBlock = { val height: Height = Height @@ (bestHeaderOpt.map(_.height).getOrElse(TestNetConstants.PreGenesisHeight) + 1) - val timestamp: Time = timeProvider.estimatedTime val txsU: List[Transaction] = txsFromMempool.filter(view.state.validate(_, timestamp, height).isRight).distinct val filteredTxsWithoutDuplicateInputs = txsU.foldLeft(List.empty[String], IndexedSeq.empty[Transaction]) { case ((usedInputsIds, acc), tx) => @@ -223,7 +235,7 @@ class Miner(dataHolder: ActorRef, } def produceCandidate(): Unit = { - def lambda(txs: List[Transaction]) = (nodeView: CurrentView[History, UtxoState, EncryWallet]) => + def lambda(txs: List[Transaction], time: Time) = (nodeView: CurrentView[HistoryReader, UtxoStateReader, WalletReader]) => { val producingStartTime: Time = System.currentTimeMillis() startTime = producingStartTime @@ -240,15 +252,19 @@ class Miner(dataHolder: ActorRef, if (settings.influxDB.isDefined) context.actorSelection("user/statsSender") ! SleepTime(System.currentTimeMillis() - sleepTime) logger.info("Going to calculate last block:") + val envelope: CandidateEnvelope = CandidateEnvelope - .fromCandidate(createCandidate(nodeView, txs, bestHeaderOpt)) + .fromCandidate(createCandidate(nodeView, txs, bestHeaderOpt, time)) envelope } else CandidateEnvelope.empty candidate } + val time = timeProvider.estimatedTime + println(s"Estimate time on creation: ${time}") (mempool ? SendTransactionsToMiner).mapTo[TransactionsForMiner].foreach { txs => - nvh ! GetDataFromCurrentView[History, UtxoState, EncryWallet, CandidateEnvelope](lambda(txs.txs.toList)) + println(s"Send ged data lambda to nvh. time: ${timeProvider.estimatedTime}") + nvh ! GetDataFromCurrentView[HistoryReader, UtxoStateReader, WalletReader, CandidateEnvelope](lambda(txs.txs.toList, time)) } } } diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index 6eee7a2ec7..53c45cc7c6 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -1,38 +1,42 @@ package encry.nvg -import akka.actor.{Actor, ActorRef, Props} +import akka.actor.{Actor, ActorRef, Props, Stash} +import akka.pattern._ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo +import encry.local.miner.Miner.CandidateEnvelope import encry.network.Messages.MessageToNetwork.RequestFromLocal -import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{RegisterHistory, RegisterState} -import encry.nvg.IntermediaryNVHView.{InitGenesisHistory, ModifierToAppend} +import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{RegisterNodeView, RegisterState} +import encry.nvg.IntermediaryNVHView.{ModifierToAppend, NodeViewStarted} import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NVHHistory.{ModifierAppliedToHistory, ProgressInfoForState} +import encry.nvg.NVHHistory.{ModifierAppliedToHistory, NewWalletReader, ProgressInfoForState} import encry.nvg.NVHState.StateAction -import encry.nvg.NVHState.StateAction.ApplyModifier import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier -import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification} +import encry.nvg.NodeViewHolder.{GetDataFromCurrentView, SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification} import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage -import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider +import encry.view.NodeViewHolder.CurrentView import encry.view.history.HistoryReader import encry.view.state.UtxoStateReader +import encry.view.wallet.WalletReader import io.iohk.iodb.ByteArrayWrapper import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header} -import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import org.encryfoundation.common.modifiers.history.Block -import scala.collection.IndexedSeq -import scala.util.{Failure, Success} +import scala.concurrent.Future class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, influx: Option[ActorRef]) extends Actor - with StrictLogging { + with StrictLogging + with Stash { + + import context.dispatcher var historyReader: HistoryReader = HistoryReader.empty + var walletReader: WalletReader = WalletReader.empty + val historyRef: ActorRef = context.actorOf(NVHHistory.props(ntp, settings)) var isModifierProcessingInProgress: Boolean = false @@ -44,28 +48,58 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, def awaitingViewActors(history: Option[ActorRef] = None, state: Option[ActorRef] = None, stateReader: Option[UtxoStateReader] = None): Receive = { - case RegisterHistory(reader) if state.isEmpty => + case RegisterNodeView(reader, wallet) if state.isEmpty => + walletReader = wallet historyReader = reader logger.info(s"NodeViewParent actor got init history. Going to init state actor.") context.become(awaitingViewActors(Some(sender()), state), discardOld = true) context.actorOf(NVHState.restoreProps(settings, reader, influx)) - case RegisterHistory(_) => + case RegisterNodeView(reader, wallet) => + walletReader = wallet + historyReader = reader context.become(viewReceive(sender(), state.get, stateReader.get)) case RegisterState(reader) => context.become(viewReceive(history.get, sender(), reader), discardOld = true) - case RegisterHistory => + context.system.eventStream.publish(new NodeViewStarted {}) + case RegisterNodeView => context.become(viewReceive(history.get, sender(), stateReader.get)) - case msg => logger.info(s"Receive strange: ${msg} on inter nvh") + case msg => println(s"Receive strange: $msg on inter nvh from ${sender()}") } def viewReceive(history: ActorRef, state: ActorRef, stateReader: UtxoStateReader): Receive = { case RegisterState(reader) => context.become(viewReceive(history, sender(), reader)) - case LocallyGeneratedModifier(modifier: PersistentModifier) => + case GetDataFromCurrentView(f: (CurrentView[HistoryReader, UtxoStateReader, WalletReader] => CandidateEnvelope)) => + logger.info("Receive GetDataFromCurrentView on nvh") + f(CurrentView(historyReader, stateReader, walletReader)) match { + case res: Future[_] => + res.map(l => println(l.getClass)) + res.pipeTo(sender) + case res: CandidateEnvelope => + println(s"qwe ${res.c.get.timestamp}") + sender ! res + case res => + sender ! res + } +// f(CurrentView(historyReader, stateReader, walletReader)) match { +// case resultFuture: Future[_] => resultFuture.pipeTo(sender) +// case result => sender ! result +// } + + case LocallyGeneratedModifier(modifier: Block) => + logger.info(s"Self mined block: ${modifier}") + println(s"Self mined block timestamp: ${modifier.header.timestamp}") ModifiersCache.put( NodeViewHolder.toKey(modifier.id), - modifier, + modifier.header, + historyReader, + settings, + isLocallyGenerated = true + ) + ModifiersCache.put( + NodeViewHolder.toKey(modifier.payload.id), + modifier.payload, historyReader, settings, isLocallyGenerated = true @@ -90,17 +124,23 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, ) if (!isModifierProcessingInProgress) getNextModifier() case ModifierAppliedToHistory => isModifierProcessingInProgress = false; getNextModifier() - case msg: ProgressInfoForState if msg.pi.chainSwitchingNeeded && msg.pi.branchPoint.exists(point => !stateReader.version.sameElements(point))=> - context.become(viewReceive( - history, - state, - stateReader - )) + case msg: ProgressInfoForState + if msg.pi.chainSwitchingNeeded && msg.pi.branchPoint.exists( + point => !stateReader.version.sameElements(point) + ) => + context.become( + viewReceive( + history, + state, + stateReader + ) + ) case msg: ProgressInfoForState => toApply = msg.pi.toApply.map(mod => ByteArrayWrapper(mod.id)).toSet msg.pi.toApply.foreach(mod => state ! StateAction.ApplyModifier(mod, msg.saveRootNodeFlag, msg.isFullChainSynced)) - case msg: StateAction.ApplyFailed => historyRef ! msg - case msg: StateAction.ModifierApplied => + case msg: StateAction.ApplyFailed => historyRef ! msg + case NewWalletReader(reader) => walletReader = reader + case msg: StateAction.ModifierApplied => historyRef ! msg case msg: SyntacticallyFailedModification => context.parent ! msg case msg: StatsSenderMessage => context.parent ! msg @@ -129,12 +169,13 @@ object IntermediaryNVHView { sealed trait IntermediaryNVHViewActions object IntermediaryNVHViewActions { - case class RegisterHistory(historyReader: HistoryReader) extends IntermediaryNVHViewActions - case class RegisterState(stateReader: UtxoStateReader) extends IntermediaryNVHViewActions + case class RegisterNodeView(historyReader: HistoryReader, walletReader: WalletReader) extends IntermediaryNVHViewActions + case class RegisterState(stateReader: UtxoStateReader) extends IntermediaryNVHViewActions } final case class ModifierToAppend(modifier: PersistentModifier, isLocallyGenerated: Boolean) case object InitGenesisHistory + trait NodeViewStarted def props(settings: EncryAppSettings, ntp: NetworkTimeProvider, influxRef: Option[ActorRef]): Props = Props(new IntermediaryNVHView(settings, ntp, influxRef)) diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index 1925f2f755..0c5a9a85f3 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -9,9 +9,9 @@ import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo import encry.network.Messages.MessageToNetwork.RequestFromLocal -import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.RegisterHistory +import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.RegisterNodeView import encry.nvg.IntermediaryNVHView.{ InitGenesisHistory, ModifierToAppend } -import encry.nvg.NVHHistory.{ ModifierAppliedToHistory, ProgressInfoForState } +import encry.nvg.NVHHistory.{ ModifierAppliedToHistory, NewWalletReader, ProgressInfoForState } import encry.nvg.NVHState.StateAction import encry.nvg.NodeViewHolder.{ SemanticallyFailedModification, @@ -20,10 +20,12 @@ import encry.nvg.NodeViewHolder.{ } import encry.settings.EncryAppSettings import encry.stats.StatsSender._ +import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.history.History.HistoryUpdateInfoAcc import encry.view.history.{ History, HistoryReader } +import encry.view.wallet.{ EncryWallet, WalletReader } import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } @@ -35,21 +37,22 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) with StrictLogging with AutoCloseable { - println("start here!") - var history: History = initializeHistory.getOrElse(genesis) + final case class HistoryView(history: History, wallet: EncryWallet) + + var historyView: HistoryView = initializeHistory.getOrElse(genesis) var lastProgressInfo: ProgressInfo = ProgressInfo(none, Seq.empty, Seq.empty, none) - context.parent ! RegisterHistory(HistoryReader(history)) + context.parent ! RegisterNodeView(HistoryReader(historyView.history), WalletReader(historyView.wallet)) override def postStop(): Unit = println("stop!") override def receive: Receive = { - case ModifierToAppend(mod, isLocallyGenerated) if !history.isModifierDefined(mod.id) => + case ModifierToAppend(mod, isLocallyGenerated) if !historyView.history.isModifierDefined(mod.id) => val startProcessingTime: Long = System.currentTimeMillis() logger.info(s"Start modifier ${mod.encodedId} of type ${mod.modifierTypeId} processing by history.") context.parent ! StartApplyingModifier(mod.id, mod.modifierTypeId, startProcessingTime) - history.append(mod) match { + historyView.history.append(mod) match { case Left(error: Throwable) => logger.info( s"Error ${error.getMessage} has occurred during processing modifier by history component. " + @@ -62,8 +65,8 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) s"Modifier ${mod.encodedId} of type ${mod.modifierTypeId} processed successfully by history. " + s"Time of processing is: ${(System.currentTimeMillis() - startProcessingTime) / 1000}s." ) - history.insertUpdateInfo(newUpdateInformation) - if (mod.modifierTypeId == Header.modifierTypeId) history.updateIdsForSyncInfo() + historyView.history.insertUpdateInfo(newUpdateInformation) + if (mod.modifierTypeId == Header.modifierTypeId) historyView.history.updateIdsForSyncInfo() context.parent ! EndOfApplyingModifier(mod.id) context.parent ! ModifierAppendedToHistory(mod match { case _: Header => true @@ -73,9 +76,9 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) logger.info(s"Progress info contains an non empty toApply. Going to notify state about new toApply.") context.parent ! ProgressInfoForState( progressInfo, - (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2, - history.isFullChainSynced, - HistoryReader(history) + (historyView.history.getBestHeaderHeight - historyView.history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2, + historyView.history.isFullChainSynced, + HistoryReader(historyView.history) ) lastProgressInfo = progressInfo if (!isLocallyGenerated) progressInfo.toApply.foreach { @@ -85,7 +88,7 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) } else { logger.info(s"Progress info contains an empty toApply. Going to form request download.") if (!isLocallyGenerated) requestDownloads(progressInfo, mod.id.some) - context.parent ! HeightStatistics(history.getBestHeaderHeight, -1) //todo incorrect state height + context.parent ! HeightStatistics(historyView.history.getBestHeaderHeight, -1) //todo incorrect state height context.parent ! SemanticallySuccessfulModifier(mod) } } @@ -95,41 +98,51 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) logger.info(s"Got modifier ${mod.encodedId} on history actor which already contains in history.") case StateAction.ModifierApplied(mod: PersistentModifier) => - history = history.reportModifierIsValid(mod) - context.parent ! HistoryReader(history) - context.parent ! BlockAndHeaderInfo(history.getBestHeader, history.getBestBlock) - context.parent ! SemanticallySuccessfulModifier(mod) - if (history.getBestHeaderId.exists(besId => history.getBestBlockId.exists(_.sameElements(besId)))) - history.isFullChainSynced = true - context.parent ! HeightStatistics(history.getBestHeaderHeight, -1) //todo incorrect state height + val newHistory = historyView.history.reportModifierIsValid(mod) + historyView = historyView.copy(history = newHistory) + context.parent ! HistoryReader(historyView.history) + context.parent ! BlockAndHeaderInfo(historyView.history.getBestHeader, historyView.history.getBestBlock) + if (historyView.history.getBestHeaderId.exists( + besId => historyView.history.getBestBlockId.exists(_.sameElements(besId)) + )) + historyView.history.isFullChainSynced = true + context.parent ! HeightStatistics(historyView.history.getBestHeaderHeight, -1) //todo incorrect state height if (mod match { case _: Block => true case _: Payload => true case _ => false }) context.parent ! ModifierAppendedToState(success = true) + if (lastProgressInfo.chainSwitchingNeeded) + historyView.wallet.rollback(VersionTag !@@ lastProgressInfo.branchPoint.get).get + historyView.wallet.scanPersistent(mod) + context.parent ! NewWalletReader(WalletReader(historyView.wallet)) context.parent ! ModifierAppliedToHistory + context.parent ! SemanticallySuccessfulModifier(mod) + case StateAction.ApplyFailed(mod, e) => - val (newHistory: History, progressInfo: ProgressInfo) = history.reportModifierIsInvalid(mod) + val (newHistory: History, progressInfo: ProgressInfo) = historyView.history.reportModifierIsInvalid(mod) context.parent ! SemanticallyFailedModification(mod, e) context.parent ! ProgressInfoForState( progressInfo, - (history.getBestHeaderHeight - history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2, - history.isFullChainSynced, - HistoryReader(history) + (historyView.history.getBestHeaderHeight - historyView.history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2, + historyView.history.isFullChainSynced, + HistoryReader(historyView.history) ) lastProgressInfo = progressInfo - history = newHistory + historyView = historyView.copy(history = newHistory) case InitGenesisHistory => logger.info("Init in InitGenesisHistory") - history.close() - history = genesis + historyView.history.close() + historyView.wallet.close() + historyView = genesis + } def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = none): Unit = pi.toDownload.foreach { case (tid: ModifierTypeId, id: ModifierId) => - if (tid != Payload.modifierTypeId || (history.isFullChainSynced && tid == Payload.modifierTypeId)) { + if (tid != Payload.modifierTypeId || (historyView.history.isFullChainSynced && tid == Payload.modifierTypeId)) { logger.info( s"History holder created download request for modifier ${Algos.encode(id)} of type $tid. " + s"Previous modifier is ${previousModifier.map(Algos.encode)}." @@ -141,35 +154,43 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) ) } - def initializeHistory: Option[History] = + def initializeHistory: Option[HistoryView] = try { val history: History = History.readOrGenerate(settings, ntp) history.updateIdsForSyncInfo() + val wallet: EncryWallet = + EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) logger.info(s"History best block height: ${history.getBestBlockHeight}") logger.info(s"History best header height: ${history.getBestHeaderHeight}") - Some(history) + Some(HistoryView(history, wallet)) } catch { case error: Throwable => logger.info(s"During history initialization error ${error.getMessage} has happened.") None } - def genesis: History = + def genesis: HistoryView = try { new File(s"${settings.directory}/history/").listFiles.foreach(FileUtils.cleanDirectory) + new File(s"${settings.directory}/wallet/").listFiles.foreach(FileUtils.cleanDirectory) + new File(s"${settings.directory}/keys/").listFiles.foreach(FileUtils.cleanDirectory) val history: History = History.readOrGenerate(settings, ntp) history.updateIdsForSyncInfo() + val wallet: EncryWallet = + EncryWallet.readOrGenerate(EncryWallet.getWalletDir(settings), EncryWallet.getKeysDir(settings), settings) logger.info(s"History best block height: ${history.getBestBlockHeight}") logger.info(s"History best header height: ${history.getBestHeaderHeight}") - history + HistoryView(history, wallet) } catch { case error: Throwable => - println("123") EncryApp.forceStopApplication(1, s"During genesis history initialization error ${error.getMessage} has happened.") } - override def close(): Unit = history.close() + override def close(): Unit = { + historyView.history.close() + historyView.wallet.close() + } } object NVHHistory { @@ -179,5 +200,6 @@ object NVHHistory { reader: HistoryReader) case object ModifierAppliedToHistory final case object InsertNewUpdates + final case class NewWalletReader(reader: WalletReader) def props(ntp: NetworkTimeProvider, settings: EncryAppSettings): Props = Props(new NVHHistory(settings, ntp)) } diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 7ae699175f..88082ee910 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -6,7 +6,8 @@ import encry.view.history.ValidationError.HistoryApiError import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.{ Block, Header } import org.encryfoundation.common.network.SyncInfo -import org.encryfoundation.common.utils.TaggedTypes.ModifierId +import org.encryfoundation.common.utils.TaggedTypes.{ Difficulty, ModifierId } + import scala.collection.immutable.HashSet trait HistoryReader { @@ -54,6 +55,8 @@ trait HistoryReader { def getBestBlock: Option[Block] def testApplicable(modifier: PersistentModifier): Either[ValidationError, PersistentModifier] + + def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] } object HistoryReader { @@ -83,7 +86,8 @@ object HistoryReader { (None, HeaderChain.empty) def testApplicable(modifier: PersistentModifier): Either[ValidationError, PersistentModifier] = Left(HistoryApiError("")) - def getBestHeaderId: Option[ModifierId] = None + def getBestHeaderId: Option[ModifierId] = None + def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] = Left(HistoryApiError("")) } def apply(history: History): HistoryReader = new HistoryReader { @@ -113,5 +117,7 @@ object HistoryReader { def testApplicable(modifier: PersistentModifier): Either[ValidationError, PersistentModifier] = history.testApplicable(modifier) def getBestHeaderId: Option[ModifierId] = history.getBestHeaderId + def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] = + history.requiredDifficultyAfter(parent) } } diff --git a/src/main/scala/encry/view/state/UtxoStateReader.scala b/src/main/scala/encry/view/state/UtxoStateReader.scala index 04566d1283..2c62d9adf2 100644 --- a/src/main/scala/encry/view/state/UtxoStateReader.scala +++ b/src/main/scala/encry/view/state/UtxoStateReader.scala @@ -5,10 +5,13 @@ import encry.storage.VersionalStorage.{StorageKey, StorageValue} import encry.utils.CoreTaggedTypes.VersionTag import encry.view.state.avlTree.{AvlTree, Node} import encry.view.state.avlTree.utils.implicits.Instances._ +import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.modifiers.state.StateModifierSerializer +import org.encryfoundation.common.modifiers.state.box.Box.Amount import org.encryfoundation.common.modifiers.state.box.EncryBaseBox import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ADKey, Height} +import org.encryfoundation.common.validation.ValidationResult import scala.util.Try @@ -37,6 +40,10 @@ trait UtxoStateReader { def typedBoxById[B <: EncryBaseBox](boxId: ADKey): Option[EncryBaseBox] def safePointHeight: Height + + val tree: AvlTree[StorageKey, StorageValue] + + def validate(tx: Transaction, blockTimeStamp: Long, blockHeight: Height, allowedOutputDelta: Amount = 0L): Either[ValidationResult, Transaction] } object UtxoStateReader { @@ -53,5 +60,8 @@ object UtxoStateReader { override def safePointHeight: Height = state.safePointHeight override def getOperationsRootHash(toInsert: List[(StorageKey, StorageValue)], toDelete: List[StorageKey]): Try[Array[Byte]] = state.getOperationsRootHash(toInsert, toDelete) + def validate(tx: Transaction, blockTimeStamp: Long, blockHeight: Height, allowedOutputDelta: Amount = 0L): Either[ValidationResult, Transaction] = + state.validate(tx, blockTimeStamp, blockHeight, allowedOutputDelta) + val tree: AvlTree[StorageKey, StorageValue] = state.tree } } \ No newline at end of file diff --git a/src/main/scala/encry/view/wallet/WalletReader.scala b/src/main/scala/encry/view/wallet/WalletReader.scala new file mode 100644 index 0000000000..e41f4ed980 --- /dev/null +++ b/src/main/scala/encry/view/wallet/WalletReader.scala @@ -0,0 +1,14 @@ +package encry.view.wallet + +trait WalletReader { + val accountManagers: Seq[AccountManager] +} + +object WalletReader { + def apply(wallet: EncryWallet): WalletReader = new WalletReader { + val accountManagers: Seq[AccountManager] = wallet.accountManagers + } + def empty: WalletReader = new WalletReader { + override val accountManagers: Seq[AccountManager] = Seq.empty + } +} From b06dcb17189d1bc63a8b2252b28f5f315e3f4460 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 24 Mar 2020 20:33:06 +0300 Subject: [PATCH 152/177] more additions --- src/main/scala/encry/Starter.scala | 2 +- src/main/scala/encry/local/miner/Miner.scala | 215 +++++++++++-------- 2 files changed, 127 insertions(+), 90 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 135a9b3361..2faca868ea 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -425,7 +425,7 @@ class Starter(settings: EncryAppSettings, IntermediaryNVH.props(newSettings, networkRouter, timeProvider, influxRef, memoryPool, dataHolderForApi) ) val miner: ActorRef = - context.system.actorOf(Miner.props(dataHolderForApi, memoryPool, nvhRouter, influxRef, newSettings), "miner") + context.system.actorOf(Miner.props(dataHolderForApi, memoryPool, nvhRouter, influxRef, newSettings, timeProvider), "miner") //if (newSettings.node.mining) miner ! StartMining if (newSettings.node.useCli) { context.system diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index 5f78298532..d45586ad74 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -3,40 +3,39 @@ package encry.local.miner import java.text.SimpleDateFormat import java.util.Date -import akka.actor.{Actor, ActorRef, Props, Stash} -import akka.util.Timeout +import akka.actor.{ Actor, ActorRef, Props, Stash } import akka.pattern._ +import akka.util.Timeout import com.typesafe.scalalogging.StrictLogging import encry.EncryApp -import encry.EncryApp._ -import encry.api.http.DataHolderForApi.{UpdatingMinerStatus, UpdatingTransactionsNumberForApi} -import encry.consensus.{CandidateBlock, EncrySupplyController, EquihashPowScheme} +import encry.api.http.DataHolderForApi.UpdatingMinerStatus +import encry.consensus.{ CandidateBlock, EncrySupplyController, EquihashPowScheme } import encry.local.miner.Miner._ import encry.local.miner.Worker.NextChallenge import encry.modifiers.mempool.TransactionFactory -import encry.network.DeliveryManager -import encry.network.DeliveryManager.{BlockchainStatus, FullBlockChainIsSynced} +import encry.mpg.MemoryPool._ +import encry.network.DeliveryManager.{ BlockchainStatus, FullBlockChainIsSynced } +import encry.nvg.IntermediaryNVHView.NodeViewStarted import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier -import encry.nvg.NodeViewHolder.{GetDataFromCurrentView, SemanticallySuccessfulModifier} +import encry.nvg.NodeViewHolder.{ GetDataFromCurrentView, SemanticallySuccessfulModifier } import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.NetworkTime.Time -import encry.view.state.avlTree.utils.implicits.Instances._ -import encry.view.NodeViewHolder.CurrentView -import encry.view.history.{History, HistoryReader} -import encry.mpg.MemoryPool._ -import encry.nvg.IntermediaryNVHView.NodeViewStarted -import encry.view.state.{UtxoState, UtxoStateReader} +import encry.utils.NetworkTimeProvider import encry.utils.implicits.UTXO._ -import encry.view.wallet.{EncryWallet, WalletReader} +import encry.view.NodeViewHolder.CurrentView +import encry.view.history.HistoryReader +import encry.view.state.avlTree.utils.implicits.Instances._ +import encry.view.state.{ UtxoState, UtxoStateReader } +import encry.view.wallet.WalletReader import io.circe.syntax._ -import io.circe.{Encoder, Json} +import io.circe.{ Encoder, Json } import org.encryfoundation.common.crypto.PrivateKey25519 -import org.encryfoundation.common.modifiers.history.{Block, Header} +import org.encryfoundation.common.modifiers.history.{ Block, Header } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction import org.encryfoundation.common.modifiers.state.box.Box.Amount import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, Height, ModifierId} +import org.encryfoundation.common.utils.TaggedTypes.{ Difficulty, Height, ModifierId } import org.encryfoundation.common.utils.constants.TestNetConstants import scala.collection._ @@ -46,22 +45,31 @@ class Miner(dataHolder: ActorRef, mempool: ActorRef, nvh: ActorRef, influx: Option[ActorRef], - settings: EncryAppSettings) extends Actor with StrictLogging with Stash { + settings: EncryAppSettings, + ntp: NetworkTimeProvider) + extends Actor + with StrictLogging + with Stash { implicit val timeout: Timeout = Timeout(5.seconds) + import context.dispatcher + type TransactionIdAsKey = scala.collection.mutable.WrappedArray.ofByte def toKey(id: ModifierId): TransactionIdAsKey = new mutable.WrappedArray.ofByte(id) - val dateFormat: SimpleDateFormat = new SimpleDateFormat("HH:mm:ss") - var startTime: Long = System.currentTimeMillis() - var sleepTime: Long = System.currentTimeMillis() + val dateFormat: SimpleDateFormat = new SimpleDateFormat("HH:mm:ss") + var startTime: Long = System.currentTimeMillis() + var sleepTime: Long = System.currentTimeMillis() var candidateOpt: Option[CandidateBlock] = None - var syncingDone: Boolean = settings.node.offlineGeneration - val numberOfWorkers: Int = settings.node.numberOfMiningWorkers - val powScheme: EquihashPowScheme = EquihashPowScheme(TestNetConstants.n, TestNetConstants.k, - TestNetConstants.Version, TestNetConstants.PreGenesisHeight, TestNetConstants.MaxTarget) + var syncingDone: Boolean = settings.node.offlineGeneration + val numberOfWorkers: Int = settings.node.numberOfMiningWorkers + val powScheme: EquihashPowScheme = EquihashPowScheme(TestNetConstants.n, + TestNetConstants.k, + TestNetConstants.Version, + TestNetConstants.PreGenesisHeight, + TestNetConstants.MaxTarget) override def preStart(): Unit = { context.system.eventStream.subscribe(self, classOf[MinerMiningCommands]) @@ -98,15 +106,22 @@ class Miner(dataHolder: ActorRef, self ! StartMining case StartMining if syncingDone => println(s"got start mining from ${sender()}") - for (i <- 0 until numberOfWorkers) yield context.actorOf( - Props(classOf[Worker], i, numberOfWorkers, self).withDispatcher("mining-dispatcher").withMailbox("mining-mailbox")) + for (i <- 0 until numberOfWorkers) + yield + context.actorOf( + Props(classOf[Worker], i, numberOfWorkers, self) + .withDispatcher("mining-dispatcher") + .withMailbox("mining-mailbox") + ) candidateOpt match { case Some(candidateBlock) => logger.info(s"Starting mining at ${dateFormat.format(new Date(System.currentTimeMillis()))}") context.children.foreach(_ ! NextChallenge(candidateBlock)) case None => - println(s"Candidate is empty! Producing new candidate! ${timeProvider.estimatedTime}") - produceCandidate() + println(s"Candidate is empty! Producing new candidate! ${ntp.estimatedTime}") + val time = ntp.estimatedTime + println(s"<<<<<<<< $time") + produceCandidate(time) } case StartMining => logger.info("Can't start mining because of chain is not synced!") @@ -121,8 +136,10 @@ class Miner(dataHolder: ActorRef, candidateOpt = None context.become(miningDisabled) case MinedBlock(block, workerIdx) if candidateOpt.exists(_.timestamp == block.header.timestamp) => - logger.info(s"Going to propagate new block (${block.header.height}, ${block.header.encodedId}, ${block.payload.txs.size}" + - s" from worker $workerIdx with nonce: ${block.header.nonce}.") + logger.info( + s"Going to propagate new block (${block.header.height}, ${block.header.encodedId}, ${block.payload.txs.size}" + + s" from worker $workerIdx with nonce: ${block.header.nonce}." + ) logger.debug(s"Set previousSelfMinedBlockId: ${Algos.encode(block.id)}") killAllWorkers() nvh ! LocallyGeneratedModifier(block) @@ -155,14 +172,18 @@ class Miner(dataHolder: ActorRef, def receiveSemanticallySuccessfulModifier: Receive = { case SemanticallySuccessfulModifier(mod: Block) if needNewCandidate(mod) => - logger.info(s"Got new block. Starting to produce candidate at height: ${mod.header.height + 1} " + - s"at ${dateFormat.format(new Date(System.currentTimeMillis()))}") - produceCandidate() + logger.info( + s"Got new block. Starting to produce candidate at height: ${mod.header.height + 1} " + + s"at ${dateFormat.format(new Date(System.currentTimeMillis()))}" + ) + val time = ntp.estimatedTime + println(s">>>><<<<<<>>><<><><. $time") + produceCandidate(time) case SemanticallySuccessfulModifier(_) => logger.info("Got new block. But needNewCandidate - false") } def receiverCandidateBlock: Receive = { - case c: CandidateBlock => procCandidateBlock(c) + case c: CandidateBlock => procCandidateBlock(c) case cEnv: CandidateEnvelope if cEnv.c.nonEmpty => procCandidateBlock(cEnv.c.get) case _: CandidateEnvelope => logger.debug("Received empty CandidateEnvelope, going to suspend mining for a while") @@ -191,35 +212,44 @@ class Miner(dataHolder: ActorRef, txsFromMempool: List[Transaction], bestHeaderOpt: Option[Header], timestamp: Time): CandidateBlock = { - val height: Height = Height @@ (bestHeaderOpt.map(_.height).getOrElse(TestNetConstants.PreGenesisHeight) + 1) + val height: Height = Height @@ (bestHeaderOpt.map(_.height).getOrElse(TestNetConstants.PreGenesisHeight) + 1) val txsU: List[Transaction] = txsFromMempool.filter(view.state.validate(_, timestamp, height).isRight).distinct - val filteredTxsWithoutDuplicateInputs = txsU.foldLeft(List.empty[String], IndexedSeq.empty[Transaction]) { - case ((usedInputsIds, acc), tx) => - if (tx.inputs.forall(input => !usedInputsIds.contains(Algos.encode(input.boxId)))) { - (usedInputsIds ++ tx.inputs.map(input => Algos.encode(input.boxId))) -> (acc :+ tx) - } else usedInputsIds -> acc - }._2 - val feesTotal: Amount = filteredTxsWithoutDuplicateInputs.map(_.fee).sum - val supplyTotal: Amount = EncrySupplyController.supplyAt(height, settings.constants) + val filteredTxsWithoutDuplicateInputs = txsU + .foldLeft(List.empty[String], IndexedSeq.empty[Transaction]) { + case ((usedInputsIds, acc), tx) => + if (tx.inputs.forall(input => !usedInputsIds.contains(Algos.encode(input.boxId)))) { + (usedInputsIds ++ tx.inputs.map(input => Algos.encode(input.boxId))) -> (acc :+ tx) + } else usedInputsIds -> acc + } + ._2 + val feesTotal: Amount = filteredTxsWithoutDuplicateInputs.map(_.fee).sum + val supplyTotal: Amount = EncrySupplyController.supplyAt(height, settings.constants) val minerSecret: PrivateKey25519 = view.vault.accountManagers.head.mandatoryAccount val coinbase: Transaction = TransactionFactory .coinbaseTransactionScratch(minerSecret.publicImage, timestamp, supplyTotal, feesTotal, height) val txs: Seq[Transaction] = filteredTxsWithoutDuplicateInputs.sortBy(_.timestamp) :+ coinbase - val difficulty: Difficulty = bestHeaderOpt.map(parent => view.history.requiredDifficultyAfter(parent) match { - case Right(value) => value - case Left(value) => EncryApp.forceStopApplication(999, value.toString) - }) + val difficulty: Difficulty = bestHeaderOpt + .map( + parent => + view.history.requiredDifficultyAfter(parent) match { + case Right(value) => value + case Left(value) => EncryApp.forceStopApplication(999, value.toString) + } + ) .getOrElse(TestNetConstants.InitialDifficulty) val combinedStateChange: UtxoState.StateChange = combineAll(txs.map(UtxoState.tx2StateChange).toList) logger.info(s"Root node hash before producing candidate: ${view.state.tree.rootNode.hash}") - val newStateRoot = view.state.tree.getOperationsRootHash( - combinedStateChange.outputsToDb.toList, combinedStateChange.inputsToDb.toList - ).get + val newStateRoot = view.state.tree + .getOperationsRootHash( + combinedStateChange.outputsToDb.toList, + combinedStateChange.inputsToDb.toList + ) + .get logger.info(s"State root node hash should be: ${Algos.encode(newStateRoot)} after applying block") @@ -228,43 +258,48 @@ class Miner(dataHolder: ActorRef, val candidate: CandidateBlock = CandidateBlock(bestHeaderOpt, TestNetConstants.Version, txs, timestamp, difficulty, newStateRoot) - logger.info(s"Sending candidate block with ${candidate.transactions.length - 1} transactions " + - s"and 1 coinbase for height $height.") + logger.info( + s"Sending candidate block with ${candidate.transactions.length - 1} transactions " + + s"and 1 coinbase for height $height." + ) candidate } - def produceCandidate(): Unit = { - def lambda(txs: List[Transaction], time: Time) = (nodeView: CurrentView[HistoryReader, UtxoStateReader, WalletReader]) => - { - val producingStartTime: Time = System.currentTimeMillis() - startTime = producingStartTime - val bestHeaderOpt: Option[Header] = nodeView.history.getBestBlock.map(_.header) - bestHeaderOpt match { - case Some(h) => logger.info(s"Best header at height ${h.height}") - case None => logger.info(s"No best header opt") + def produceCandidate(time: Time): Unit = { + def lambda(txs: List[Transaction], time: Time) = + (nodeView: CurrentView[HistoryReader, UtxoStateReader, WalletReader]) => { + val producingStartTime: Time = System.currentTimeMillis() + startTime = producingStartTime + val bestHeaderOpt: Option[Header] = nodeView.history.getBestBlock.map(_.header) + bestHeaderOpt match { + case Some(h) => logger.info(s"Best header at height ${h.height}") + case None => logger.info(s"No best header opt") + } + val candidate: CandidateEnvelope = + if ((bestHeaderOpt.isDefined && + (syncingDone || nodeView.history.isFullChainSynced)) || settings.node.offlineGeneration) { + logger.info( + s"Starting candidate generation at " + + s"${dateFormat.format(new Date(System.currentTimeMillis()))}" + ) + if (settings.influxDB.isDefined) + context.actorSelection("user/statsSender") ! SleepTime(System.currentTimeMillis() - sleepTime) + logger.info("Going to calculate last block:") + + val envelope: CandidateEnvelope = + CandidateEnvelope + .fromCandidate(createCandidate(nodeView, txs, bestHeaderOpt, time)) + envelope + } else CandidateEnvelope.empty + candidate } - val candidate: CandidateEnvelope = - if ((bestHeaderOpt.isDefined && - (syncingDone || nodeView.history.isFullChainSynced)) || settings.node.offlineGeneration) { - logger.info(s"Starting candidate generation at " + - s"${dateFormat.format(new Date(System.currentTimeMillis()))}") - if (settings.influxDB.isDefined) - context.actorSelection("user/statsSender") ! SleepTime(System.currentTimeMillis() - sleepTime) - logger.info("Going to calculate last block:") - - val envelope: CandidateEnvelope = - CandidateEnvelope - .fromCandidate(createCandidate(nodeView, txs, bestHeaderOpt, time)) - envelope - } else CandidateEnvelope.empty - candidate - } - val time = timeProvider.estimatedTime println(s"Estimate time on creation: ${time}") (mempool ? SendTransactionsToMiner).mapTo[TransactionsForMiner].foreach { txs => - println(s"Send ged data lambda to nvh. time: ${timeProvider.estimatedTime}") - nvh ! GetDataFromCurrentView[HistoryReader, UtxoStateReader, WalletReader, CandidateEnvelope](lambda(txs.txs.toList, time)) + println(s"Send ged data lambda to nvh. time: ${ntp.estimatedTime}") + nvh ! GetDataFromCurrentView[HistoryReader, UtxoStateReader, WalletReader, CandidateEnvelope]( + lambda(txs.txs.toList, time) + ) } } } @@ -297,15 +332,17 @@ object Miner { ).asJson } - implicit val jsonEncoder: Encoder[MinerStatus] = (r: MinerStatus) => Map( - "isMining" -> r.isMining.asJson, - "candidateBlock" -> r.candidateBlock.map(_.asJson).getOrElse("None".asJson) - ).asJson + implicit val jsonEncoder: Encoder[MinerStatus] = (r: MinerStatus) => + Map( + "isMining" -> r.isMining.asJson, + "candidateBlock" -> r.candidateBlock.map(_.asJson).getOrElse("None".asJson) + ).asJson def props(dataHolder: ActorRef, mempool: ActorRef, nvh: ActorRef, influx: Option[ActorRef], - settings: EncryAppSettings): Props = - Props(new Miner(dataHolder, mempool, nvh, influx, settings)) -} \ No newline at end of file + settings: EncryAppSettings, + ntp: NetworkTimeProvider): Props = + Props(new Miner(dataHolder, mempool, nvh, influx, settings, ntp)) +} From db590cdc5185fe182ecb4cdbb67b17cb62a353ab Mon Sep 17 00:00:00 2001 From: aleksandr Date: Wed, 25 Mar 2020 12:03:28 +0300 Subject: [PATCH 153/177] fix calculating correct root node hash at miner --- src/main/scala/encry/nvg/IntermediaryNVHView.scala | 1 + src/main/scala/encry/nvg/NVHState.scala | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index 53c45cc7c6..a44dcd78da 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -69,6 +69,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, def viewReceive(history: ActorRef, state: ActorRef, stateReader: UtxoStateReader): Receive = { case RegisterState(reader) => context.become(viewReceive(history, sender(), reader)) + case reader: UtxoStateReader => context.become(viewReceive(history, state, reader)) case GetDataFromCurrentView(f: (CurrentView[HistoryReader, UtxoStateReader, WalletReader] => CandidateEnvelope)) => logger.info("Receive GetDataFromCurrentView on nvh") diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index 9fb015b4eb..317427afb3 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -52,6 +52,7 @@ class NVHState(influxRef: Option[ActorRef], var historyReader: HistoryReader, se state = stateAfterApply logger.info(s"Successfully apply modifier: ${Algos.encode(modifier.id)} of type ${modifier.modifierTypeId}") context.parent ! ModifierApplied(modifier) + context.parent ! UtxoStateReader(state) case Left(e: List[ModifierApplyError]) => logger.info(s"Application to state failed cause $e") context.parent ! ApplyFailed(modifier, e) From f90d443501e529e123cb39a5b86e68a66c30294c Mon Sep 17 00:00:00 2001 From: aleksandr Date: Wed, 25 Mar 2020 12:18:26 +0300 Subject: [PATCH 154/177] code cleanup --- .../scala/encry/api/http/DataHolderForApi.scala | 6 +++--- src/main/scala/encry/local/miner/Miner.scala | 17 ++++------------- .../scala/encry/nvg/IntermediaryNVHView.scala | 3 --- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src/main/scala/encry/api/http/DataHolderForApi.scala b/src/main/scala/encry/api/http/DataHolderForApi.scala index 5970961611..c25b7bbec5 100644 --- a/src/main/scala/encry/api/http/DataHolderForApi.scala +++ b/src/main/scala/encry/api/http/DataHolderForApi.scala @@ -55,7 +55,7 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) val launchTimeFuture: Future[NetworkTime.Time] = ntp.time() def awaitNVHRef: Receive = { - case UpdateHistoryReader(history) => + case history: HistoryReader => unstashAll() logger.info("Got updated history at nvh") context.become(workingCycle(nvhRef = sender(), history = Some(history))) @@ -123,7 +123,7 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) connectedPeersCollection) ) - case UpdateHistoryReader(reader: HistoryReader) => + case reader: HistoryReader => context.become( workingCycle(nvhRef, blackList, @@ -137,7 +137,7 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) connectedPeersCollection) ) - case ChangedState(reader: UtxoStateReader) => + case reader: UtxoStateReader => context.become( workingCycle(nvhRef, blackList, diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index d45586ad74..54ddaa8dda 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -105,7 +105,6 @@ class Miner(dataHolder: ActorRef, killAllWorkers() self ! StartMining case StartMining if syncingDone => - println(s"got start mining from ${sender()}") for (i <- 0 until numberOfWorkers) yield context.actorOf( @@ -118,10 +117,8 @@ class Miner(dataHolder: ActorRef, logger.info(s"Starting mining at ${dateFormat.format(new Date(System.currentTimeMillis()))}") context.children.foreach(_ ! NextChallenge(candidateBlock)) case None => - println(s"Candidate is empty! Producing new candidate! ${ntp.estimatedTime}") - val time = ntp.estimatedTime - println(s"<<<<<<<< $time") - produceCandidate(time) + logger.info(s"Candidate is empty! Producing new candidate!") + produceCandidate(ntp.estimatedTime) } case StartMining => logger.info("Can't start mining because of chain is not synced!") @@ -176,9 +173,7 @@ class Miner(dataHolder: ActorRef, s"Got new block. Starting to produce candidate at height: ${mod.header.height + 1} " + s"at ${dateFormat.format(new Date(System.currentTimeMillis()))}" ) - val time = ntp.estimatedTime - println(s">>>><<<<<<>>><<><><. $time") - produceCandidate(time) + produceCandidate(ntp.estimatedTime) case SemanticallySuccessfulModifier(_) => logger.info("Got new block. But needNewCandidate - false") } @@ -191,9 +186,7 @@ class Miner(dataHolder: ActorRef, } def unknownMessage: Receive = { - case m => - println(m) - logger.debug(s"Unexpected message $m") + case m => logger.debug(s"Unexpected message $m") } def chainEvents: Receive = { @@ -294,9 +287,7 @@ class Miner(dataHolder: ActorRef, } else CandidateEnvelope.empty candidate } - println(s"Estimate time on creation: ${time}") (mempool ? SendTransactionsToMiner).mapTo[TransactionsForMiner].foreach { txs => - println(s"Send ged data lambda to nvh. time: ${ntp.estimatedTime}") nvh ! GetDataFromCurrentView[HistoryReader, UtxoStateReader, WalletReader, CandidateEnvelope]( lambda(txs.txs.toList, time) ) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index a44dcd78da..069bf36180 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -75,10 +75,8 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, logger.info("Receive GetDataFromCurrentView on nvh") f(CurrentView(historyReader, stateReader, walletReader)) match { case res: Future[_] => - res.map(l => println(l.getClass)) res.pipeTo(sender) case res: CandidateEnvelope => - println(s"qwe ${res.c.get.timestamp}") sender ! res case res => sender ! res @@ -90,7 +88,6 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, case LocallyGeneratedModifier(modifier: Block) => logger.info(s"Self mined block: ${modifier}") - println(s"Self mined block timestamp: ${modifier.header.timestamp}") ModifiersCache.put( NodeViewHolder.toKey(modifier.id), modifier.header, From 463fe3b02e57a441a6df9a647a7e495b05916701 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Wed, 25 Mar 2020 14:11:31 +0300 Subject: [PATCH 155/177] fix mining start after sync proc --- src/main/scala/encry/nvg/NVHHistory.scala | 33 ++++++++-------- src/main/scala/encry/nvg/NodeViewHolder.scala | 38 ++----------------- 2 files changed, 20 insertions(+), 51 deletions(-) diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index 0c5a9a85f3..b74b9980d2 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -2,35 +2,33 @@ package encry.nvg import java.io.File -import akka.actor.{ Actor, Props } +import akka.actor.{Actor, Props} import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo +import encry.local.miner.Miner.EnableMining +import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.RegisterNodeView -import encry.nvg.IntermediaryNVHView.{ InitGenesisHistory, ModifierToAppend } -import encry.nvg.NVHHistory.{ ModifierAppliedToHistory, NewWalletReader, ProgressInfoForState } +import encry.nvg.IntermediaryNVHView.{InitGenesisHistory, ModifierToAppend} +import encry.nvg.NVHHistory.{ModifierAppliedToHistory, NewWalletReader, ProgressInfoForState} import encry.nvg.NVHState.StateAction -import encry.nvg.NodeViewHolder.{ - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification -} +import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.history.History.HistoryUpdateInfoAcc -import encry.view.history.{ History, HistoryReader } -import encry.view.wallet.{ EncryWallet, WalletReader } +import encry.view.history.{History, HistoryReader} +import encry.view.wallet.{EncryWallet, WalletReader} import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } +import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends Actor @@ -68,10 +66,10 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) historyView.history.insertUpdateInfo(newUpdateInformation) if (mod.modifierTypeId == Header.modifierTypeId) historyView.history.updateIdsForSyncInfo() context.parent ! EndOfApplyingModifier(mod.id) - context.parent ! ModifierAppendedToHistory(mod match { - case _: Header => true - case _: Payload => false - }, success = true) + context.parent ! ModifierAppendedToHistory( mod match { + case _: Header => true + case _: Payload => false + }, success = true ) if (progressInfo.toApply.nonEmpty) { logger.info(s"Progress info contains an non empty toApply. Going to notify state about new toApply.") context.parent ! ProgressInfoForState( @@ -118,7 +116,8 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) context.parent ! NewWalletReader(WalletReader(historyView.wallet)) context.parent ! ModifierAppliedToHistory context.parent ! SemanticallySuccessfulModifier(mod) - + if (historyView.history.isFullChainSynced) context.system.eventStream.publish(FullBlockChainIsSynced) + if (settings.node.mining && historyView.history.isFullChainSynced) context.system.eventStream.publish(EnableMining) case StateAction.ApplyFailed(mod, e) => val (newHistory: History, progressInfo: ProgressInfo) = historyView.history.reportModifierIsInvalid(mod) context.parent ! SemanticallyFailedModification(mod, e) diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index 030445f836..ef0ba859b6 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -1,48 +1,18 @@ package encry.nvg -import java.io.File - -import akka.actor.{ Actor, ActorRef, Props } -import akka.pattern._ -import cats.syntax.option._ -import com.typesafe.scalalogging.StrictLogging -import encry.EncryApp -import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.consensus.HistoryConsensus.ProgressInfo -import encry.local.miner.Miner.{ DisableMining, EnableMining, StartMining } -import encry.network.DeliveryManager.FullBlockChainIsSynced -import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.network.NodeViewSynchronizer.ReceivableMessages._ -import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NodeViewHolder.ReceivableMessages.{ CreateAccountManagerFromSeed, LocallyGeneratedModifier } -import encry.nvg.NodeViewHolder._ -import encry.nvg.fast.sync.SnapshotProcessor.SnapshotManifest.ManifestId -import encry.nvg.fast.sync.SnapshotProcessor._ -import encry.settings.EncryAppSettings -import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag -import encry.utils.NetworkTimeProvider import encry.view.NodeViewErrors.ModifierApplyError -import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.NodeViewHolder.CurrentView -import encry.view.history.storage.HistoryStorage -import encry.view.history.{ History, HistoryHeadersProcessor, HistoryPayloadsProcessor, HistoryReader } -import encry.mpg.MemoryPool._ +import encry.view.history.{History, HistoryReader} import encry.view.state.UtxoState -import encry.view.state.avlTree.AvlTree import encry.view.wallet.EncryWallet -import io.iohk.iodb.ByteArrayWrapper -import org.apache.commons.io.FileUtils -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.modifiers.mempool.transaction.Transaction -import org.encryfoundation.common.modifiers.{ PersistentModifier, PersistentNodeViewModifier } -import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ ADDigest, ModifierId, ModifierTypeId } +import org.encryfoundation.common.modifiers.{PersistentModifier, PersistentNodeViewModifier} +import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} -import scala.collection.{ mutable, IndexedSeq, Seq } -import scala.concurrent.Future -import scala.concurrent.duration._ -import scala.util.{ Failure, Success, Try } +import scala.collection.{IndexedSeq, mutable} // //class NodeViewHolder( // settings: EncryAppSettings, From ddd8e66fae2434093d970bbe9112f0278d324119 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Thu, 26 Mar 2020 14:45:29 +0300 Subject: [PATCH 156/177] rolled back full chain --- .../scala/encry/network/MessageBuilder.scala | 5 ++-- .../scala/encry/nvg/ModifiersValidator.scala | 27 +++++++++++++++++++ src/main/scala/encry/nvg/NVHHistory.scala | 1 + .../scala/encry/nvg/NodeViewNMProcessor.scala | 2 +- .../scala/encry/view/history/HistoryApi.scala | 2 +- 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 8dc72f1897..fa2456a6d6 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -6,7 +6,7 @@ import akka.actor.{Actor, ActorRef, Props} import akka.pattern._ import akka.util.Timeout import com.typesafe.scalalogging.StrictLogging -import encry.consensus.HistoryConsensus.{Equal, Older, Younger} +import encry.consensus.HistoryConsensus.{Equal, Older, Unknown, Younger} import encry.network.ConnectedPeersCollection.PeerInfo import encry.network.DM.{IsRequested, RequestSent, RequestStatus} import encry.network.MessageBuilder.{GetPeerInfo, GetPeers, MsgSent} @@ -41,7 +41,7 @@ case class MessageBuilder(peersKeeper: ActorRef, } case RequestFromLocal(None, modTypeId, modsIds) => Try { - (peersKeeper ? (MessageBuilder.PeerWithOlderHistory || MessageBuilder.PeerWithEqualHistory)).mapTo[ConnectedPeer].map { peer => + (peersKeeper ? (MessageBuilder.PeerWithUnknownHistory || MessageBuilder.PeerWithOlderHistory || MessageBuilder.PeerWithEqualHistory)).mapTo[ConnectedPeer].map { peer => logger.info(s"Going to req mods from ${peer.socketAddress} of type ${modTypeId}") (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => logger.info(s"Requested or received: ${status.requested.length}. Not request or not received: ${status.notRequested.length}") @@ -113,6 +113,7 @@ object MessageBuilder { val PeerWithEqualHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Equal) val PeerWithOlderHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Older) val PeerWithYoungerHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Younger) + val PeerWithUnknownHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Unknown) def props(peersKeeper: ActorRef, deliveryManager: ActorRef): Props = Props(new MessageBuilder(peersKeeper, deliveryManager)) diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index 94ef506adf..214fbe6557 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -41,6 +41,33 @@ class ModifiersValidator( logger.info(s"Modifier ${Algos.encode(id)} is incorrect cause: ${error.getMessage}.") intermediaryNVH ! BanPeer(remote, CorruptedSerializedBytes) intermediaryNVH ! InvalidModifierBytes(id) + case Right(modifier: Header) if modifier.height < 59501 => + val preSemanticValidation: Either[PreSemanticValidationException, Unit] = + ModifiersValidator.isPreSemanticValid(modifier, reader, settings) + val syntacticValidation: Boolean = + ModifiersValidator.isSyntacticallyValid(modifier, settings.constants.ModifierIdSize) + if (preSemanticValidation.isRight && syntacticValidation) { + if (modifier.id.sameElements(id)) { + logger.debug(s"Modifier ${modifier.encodedId} is valid.") + intermediaryNVH ! ValidatedModifierFromNetwork(modifierTypeId) + nodeViewHolderRef ! ValidatedModifier(modifier) + } else { + logger.info(s"Modifier ${modifier.encodedId} should have ${Algos.encode(id)} id!") + intermediaryNVH ! BanPeer(remote, ModifierIdInTheNetworkMessageIsNotTheSameAsIdOfModifierInThisMessage) + intermediaryNVH ! SyntacticallyFailedModification(modifier, List.empty) + } + } else if (!syntacticValidation) { + logger.info(s"Modifier ${modifier.encodedId} is syntactically invalid.") + intermediaryNVH ! BanPeer(remote, SyntacticallyInvalidPersistentModifier) + intermediaryNVH ! SyntacticallyFailedModification(modifier, List.empty) + } else + preSemanticValidation.leftMap { + case IllegalHeight(error) => + logger.info(s"Modifier ${modifier.encodedId} is invalid cause: $error.") + intermediaryNVH ! BanPeer(remote, PreSemanticInvalidModifier(error)) + intermediaryNVH ! SyntacticallyFailedModification(modifier, List.empty) + } + case Right(modifier: Header) => case Right(modifier) => val preSemanticValidation: Either[PreSemanticValidationException, Unit] = ModifiersValidator.isPreSemanticValid(modifier, reader, settings) diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index b74b9980d2..0f2b2cdeb7 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -118,6 +118,7 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) context.parent ! SemanticallySuccessfulModifier(mod) if (historyView.history.isFullChainSynced) context.system.eventStream.publish(FullBlockChainIsSynced) if (settings.node.mining && historyView.history.isFullChainSynced) context.system.eventStream.publish(EnableMining) + case StateAction.ApplyFailed(mod, e) => val (newHistory: History, progressInfo: ProgressInfo) = historyView.history.reportModifierIsInvalid(mod) context.parent ! SemanticallyFailedModification(mod, e) diff --git a/src/main/scala/encry/nvg/NodeViewNMProcessor.scala b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala index e00aa12601..09b8956ac0 100644 --- a/src/main/scala/encry/nvg/NodeViewNMProcessor.scala +++ b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala @@ -127,7 +127,7 @@ class NodeViewNMProcessor(settings: EncryAppSettings) extends Actor with StrictL case CheckPayloadsToDownload => val newIds: Seq[ModifierId] = historyReader.payloadsIdsToDownload(settings.network.networkChunkSize) - logger.debug(s"newIds: ${newIds.map(Algos.encode).mkString(",")}") + logger.info(s"newIds: ${newIds.map(Algos.encode).mkString(",")}") if (newIds.nonEmpty) context.parent ! RequestFromLocal(none, Payload.modifierTypeId, newIds.toList) val nextCheckModsScheduler: Cancellable = context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)(self ! CheckPayloadsToDownload) diff --git a/src/main/scala/encry/view/history/HistoryApi.scala b/src/main/scala/encry/view/history/HistoryApi.scala index 91f5090cb8..656c1e2050 100644 --- a/src/main/scala/encry/view/history/HistoryApi.scala +++ b/src/main/scala/encry/view/history/HistoryApi.scala @@ -159,7 +159,7 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore // Already synced and header is not too far back. Download required modifiers if (header.height >= blockDownloadProcessor.minimalBlockHeight) (Payload.modifierTypeId -> header.payloadId).some // Headers chain is synced after this header. Start downloading full blocks - else if (!isHeadersChainSynced && isNewHeader(header)) { + else if (!isHeadersChainSynced && (header.height == 59500)) { isHeadersChainSyncedVar = true blockDownloadProcessor.updateBestBlock(header) none From 9700ab5dd8fdabe5f3a7f71a88e59b92b25357c1 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 30 Mar 2020 11:38:38 +0300 Subject: [PATCH 157/177] ec-fix --- src/main/scala/encry/network/MessageBuilder.scala | 6 +++--- src/main/scala/encry/nvg/NVHHistory.scala | 2 +- src/main/scala/encry/nvg/NodeViewNMProcessor.scala | 7 +++---- src/main/scala/encry/view/history/HistoryApi.scala | 7 ++++--- src/main/scala/encry/view/history/HistoryReader.scala | 4 +++- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 8dc72f1897..1e83e8745a 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -6,7 +6,7 @@ import akka.actor.{Actor, ActorRef, Props} import akka.pattern._ import akka.util.Timeout import com.typesafe.scalalogging.StrictLogging -import encry.consensus.HistoryConsensus.{Equal, Older, Younger} +import encry.consensus.HistoryConsensus.{Equal, Older, Unknown, Younger} import encry.network.ConnectedPeersCollection.PeerInfo import encry.network.DM.{IsRequested, RequestSent, RequestStatus} import encry.network.MessageBuilder.{GetPeerInfo, GetPeers, MsgSent} @@ -41,7 +41,7 @@ case class MessageBuilder(peersKeeper: ActorRef, } case RequestFromLocal(None, modTypeId, modsIds) => Try { - (peersKeeper ? (MessageBuilder.PeerWithOlderHistory || MessageBuilder.PeerWithEqualHistory)).mapTo[ConnectedPeer].map { peer => + (peersKeeper ? (MessageBuilder.PeerWithOlderHistory || MessageBuilder.PeerWithEqualHistory || MessageBuilder.PeerWithYoungerHistory || MessageBuilder.PeerWithUnknownHistory)).mapTo[ConnectedPeer].map { peer => logger.info(s"Going to req mods from ${peer.socketAddress} of type ${modTypeId}") (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => logger.info(s"Requested or received: ${status.requested.length}. Not request or not received: ${status.notRequested.length}") @@ -113,7 +113,7 @@ object MessageBuilder { val PeerWithEqualHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Equal) val PeerWithOlderHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Older) val PeerWithYoungerHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Younger) - + val PeerWithUnknownHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Unknown) def props(peersKeeper: ActorRef, deliveryManager: ActorRef): Props = Props(new MessageBuilder(peersKeeper, deliveryManager)) } \ No newline at end of file diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index b74b9980d2..f2a676f45c 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -141,7 +141,7 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = none): Unit = pi.toDownload.foreach { case (tid: ModifierTypeId, id: ModifierId) => - if (tid != Payload.modifierTypeId || (historyView.history.isFullChainSynced && tid == Payload.modifierTypeId)) { + if (tid != Payload.modifierTypeId) { logger.info( s"History holder created download request for modifier ${Algos.encode(id)} of type $tid. " + s"Previous modifier is ${previousModifier.map(Algos.encode)}." diff --git a/src/main/scala/encry/nvg/NodeViewNMProcessor.scala b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala index e00aa12601..d4a050029d 100644 --- a/src/main/scala/encry/nvg/NodeViewNMProcessor.scala +++ b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala @@ -83,9 +83,8 @@ class NodeViewNMProcessor(settings: EncryAppSettings) extends Actor with StrictL if (invData._1 == Payload.modifierTypeId && !historyReader.isFullChainSynced) logger.info(s"Got inv message from $remote with ${invData._2.size} ids but full chain is not synced.") else { - val isHeader: Boolean = invData._1 == Header.modifierTypeId - val isPayloadAvailable: Boolean = - historyReader.isHeadersChainSyncedVar && invData._1 == Payload.modifierTypeId + val isHeader: Boolean = invData._1 == Header.modifierTypeId && !historyReader.isHeadersChainSynced + val isPayloadAvailable: Boolean = false val isRequestAvailable: Boolean = isHeader || isPayloadAvailable if (isRequestAvailable) { val ids: Seq[ModifierId] = invData._2.filterNot { mid: ModifierId => @@ -127,7 +126,7 @@ class NodeViewNMProcessor(settings: EncryAppSettings) extends Actor with StrictL case CheckPayloadsToDownload => val newIds: Seq[ModifierId] = historyReader.payloadsIdsToDownload(settings.network.networkChunkSize) - logger.debug(s"newIds: ${newIds.map(Algos.encode).mkString(",")}") + logger.info(s"newIds: ${newIds.map(Algos.encode).mkString(",")}") if (newIds.nonEmpty) context.parent ! RequestFromLocal(none, Payload.modifierTypeId, newIds.toList) val nextCheckModsScheduler: Cancellable = context.system.scheduler.scheduleOnce(settings.network.modifierDeliverTimeCheck)(self ! CheckPayloadsToDownload) diff --git a/src/main/scala/encry/view/history/HistoryApi.scala b/src/main/scala/encry/view/history/HistoryApi.scala index 91f5090cb8..e2f9349a1b 100644 --- a/src/main/scala/encry/view/history/HistoryApi.scala +++ b/src/main/scala/encry/view/history/HistoryApi.scala @@ -160,6 +160,7 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore if (header.height >= blockDownloadProcessor.minimalBlockHeight) (Payload.modifierTypeId -> header.payloadId).some // Headers chain is synced after this header. Start downloading full blocks else if (!isHeadersChainSynced && isNewHeader(header)) { + logger.info(s"Update header best header at height: ${header.height}") isHeadersChainSyncedVar = true blockDownloadProcessor.updateBestBlock(header) none @@ -296,9 +297,9 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore case None => (None, headerChainBack(toHeader.height + 1, toHeader, _ => false)) } - def isNewHeader(header: Header): Boolean = - timeProvider.estimatedTime - header.timestamp < - settings.constants.DesiredBlockInterval.toMillis * settings.constants.NewHeaderTimeMultiplier + def isNewHeader(header: Header): Boolean = header.height == 55000 +// timeProvider.estimatedTime - header.timestamp < +// settings.constants.DesiredBlockInterval.toMillis * settings.constants.NewHeaderTimeMultiplier def isHeadersChainSynced: Boolean = isHeadersChainSyncedVar diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 88082ee910..efbfb56cad 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -34,7 +34,7 @@ trait HistoryReader { var isFullChainSynced: Boolean - var isHeadersChainSyncedVar: Boolean = false + def isHeadersChainSynced: Boolean def isModifierDefined(id: ModifierId): Boolean @@ -88,6 +88,7 @@ object HistoryReader { Left(HistoryApiError("")) def getBestHeaderId: Option[ModifierId] = None def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] = Left(HistoryApiError("")) + def isHeadersChainSynced: Boolean = false } def apply(history: History): HistoryReader = new HistoryReader { @@ -119,5 +120,6 @@ object HistoryReader { def getBestHeaderId: Option[ModifierId] = history.getBestHeaderId def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] = history.requiredDifficultyAfter(parent) + def isHeadersChainSynced: Boolean = history.isHeadersChainSynced } } From 9d091886daf87626db64c4f10ce4f107d4448adb Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Mon, 30 Mar 2020 18:17:31 +0300 Subject: [PATCH 158/177] history http api fixed, data holder fixed --- .../scala/encry/api/http/DataHolderForApi.scala | 12 ++++++------ .../encry/api/http/routes/HistoryApiRoute.scala | 16 +++++----------- .../scala/encry/view/history/HistoryReader.scala | 4 ++++ .../scala/encry/view/wallet/WalletReader.scala | 12 ++++++++++++ 4 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/main/scala/encry/api/http/DataHolderForApi.scala b/src/main/scala/encry/api/http/DataHolderForApi.scala index c25b7bbec5..9ddf9b4cca 100644 --- a/src/main/scala/encry/api/http/DataHolderForApi.scala +++ b/src/main/scala/encry/api/http/DataHolderForApi.scala @@ -27,7 +27,7 @@ import encry.utils.{NetworkTime, NetworkTimeProvider} import encry.view.NodeViewHolder.ReceivableMessages.GetDataFromCurrentView import encry.view.history.{History, HistoryReader} import encry.view.state.{UtxoState, UtxoStateReader} -import encry.view.wallet.EncryWallet +import encry.view.wallet.{EncryWallet, WalletReader} import org.encryfoundation.common.crypto.PrivateKey25519 import org.encryfoundation.common.modifiers.history.{Block, Header} import org.encryfoundation.common.modifiers.state.box.Box.Amount @@ -184,24 +184,24 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) case RemovePeerFromBanList(peer) => context.system.eventStream.publish(RemovePeerFromBlackList(peer)) case GetViewPrintAddress => - (nvhRef ? GetDataFromCurrentView[History, UtxoState, EncryWallet, String] { view => + (nvhRef ? GetDataFromCurrentView[HistoryReader, UtxoStateReader, WalletReader, String] { view => view.vault.publicKeys.foldLeft("") { (str, k) => str + s"Pay2PubKeyAddress : ${k.address.address} , Pay2ContractHashAddress : ${k.address.p2ch.address}" + "\n" } }).pipeTo(sender) case GetViewCreateKey => - (nvhRef ? GetDataFromCurrentView[History, UtxoState, EncryWallet, PrivateKey25519] { view => + (nvhRef ? GetDataFromCurrentView[HistoryReader, UtxoStateReader, WalletReader, PrivateKey25519] { view => view.vault.accountManagers.head.createAccount(None) }).pipeTo(sender) case GetViewPrintPubKeys => - (nvhRef ? GetDataFromCurrentView[History, UtxoState, EncryWallet, List[String]] { view => + (nvhRef ? GetDataFromCurrentView[HistoryReader, UtxoStateReader, WalletReader, List[String]] { view => view.vault.publicKeys.foldLeft(List.empty[String])((str, k) => str :+ Algos.encode(k.pubKeyBytes)) }).pipeTo(sender) case GetViewGetBalance => - (nvhRef ? GetDataFromCurrentView[History, UtxoState, EncryWallet, Map[String, List[(String, Amount)]]] { view => + (nvhRef ? GetDataFromCurrentView[HistoryReader, UtxoStateReader, WalletReader, Map[String, List[(String, Amount)]]] { view => val balance: Map[String, List[(String, Amount)]] = view.vault.getBalances.map { case ((key, token), amount) => Map(key -> List((token, amount))) }.foldLeft(Map.empty[String, List[(String, Amount)]]) { case (el1, el2) => el1 |+| el2 } @@ -209,7 +209,7 @@ class DataHolderForApi(settings: EncryAppSettings, ntp: NetworkTimeProvider) }).pipeTo(sender) case GetViewPrintPrivKeys => - (nvhRef ? GetDataFromCurrentView[History, UtxoState, EncryWallet, String] { view => + (nvhRef ? GetDataFromCurrentView[HistoryReader, UtxoStateReader, WalletReader, String] { view => view.vault.accountManagers.head.accounts.foldLeft("")((str, k) => str + Algos.encode(k.privKeyBytes) + "\n") }).pipeTo(sender) diff --git a/src/main/scala/encry/api/http/routes/HistoryApiRoute.scala b/src/main/scala/encry/api/http/routes/HistoryApiRoute.scala index 76cbe9b558..c9821dcaed 100644 --- a/src/main/scala/encry/api/http/routes/HistoryApiRoute.scala +++ b/src/main/scala/encry/api/http/routes/HistoryApiRoute.scala @@ -1,20 +1,14 @@ package encry.api.http.routes -import akka.actor.{ ActorRef, ActorRefFactory } +import akka.actor.{ActorRef, ActorRefFactory} import akka.http.scaladsl.server.Route import akka.pattern.ask -import encry.api.http.DataHolderForApi.{ - GetDataFromHistory, - GetFullBlockByIdCommand, - GetLastHeaderIdAtHeightHelper, - GetLastHeadersHelper, - GetMinerStatus -} +import encry.api.http.DataHolderForApi.{GetDataFromHistory, GetFullBlockByIdCommand, GetLastHeaderIdAtHeightHelper, GetLastHeadersHelper, GetMinerStatus} import encry.local.miner.Miner.MinerStatus import encry.settings.RESTApiSettings -import encry.view.history.History +import encry.view.history.{History, HistoryReader} import io.circe.syntax._ -import org.encryfoundation.common.modifiers.history.{ Block, Header } +import org.encryfoundation.common.modifiers.history.{Block, Header} import org.encryfoundation.common.utils.Algos case class HistoryApiRoute(dataHolder: ActorRef, settings: RESTApiSettings, nodeId: Array[Byte])( @@ -32,7 +26,7 @@ case class HistoryApiRoute(dataHolder: ActorRef, settings: RESTApiSettings, node } def getHeaderIdsR: Route = (pathEndOrSingleSlash & get & paging) { (offset, limit) => - (dataHolder ? GetDataFromHistory).mapTo[History] + (dataHolder ? GetDataFromHistory).mapTo[HistoryReader] .map { _.getHeaderIds(offset, limit).map(Algos.encode).asJson }.okJson() diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index efbfb56cad..8673a90f5b 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -57,6 +57,8 @@ trait HistoryReader { def testApplicable(modifier: PersistentModifier): Either[ValidationError, PersistentModifier] def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] + + def getHeaderIds(count: Int, offset: Int = 0): Seq[ModifierId] } object HistoryReader { @@ -89,6 +91,7 @@ object HistoryReader { def getBestHeaderId: Option[ModifierId] = None def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] = Left(HistoryApiError("")) def isHeadersChainSynced: Boolean = false + def getHeaderIds(count: Int, offset: Int = 0): Seq[ModifierId] = Seq.empty } def apply(history: History): HistoryReader = new HistoryReader { @@ -121,5 +124,6 @@ object HistoryReader { def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] = history.requiredDifficultyAfter(parent) def isHeadersChainSynced: Boolean = history.isHeadersChainSynced + def getHeaderIds(count: Int, offset: Int = 0): Seq[ModifierId] = history.getHeaderIds(count, offset) } } diff --git a/src/main/scala/encry/view/wallet/WalletReader.scala b/src/main/scala/encry/view/wallet/WalletReader.scala index e41f4ed980..5fae6dde24 100644 --- a/src/main/scala/encry/view/wallet/WalletReader.scala +++ b/src/main/scala/encry/view/wallet/WalletReader.scala @@ -1,14 +1,26 @@ package encry.view.wallet +import org.encryfoundation.common.crypto.PublicKey25519 + trait WalletReader { val accountManagers: Seq[AccountManager] + def publicKeys: Set[PublicKey25519] + def getBalances: Seq[((String, String), Long)] } object WalletReader { def apply(wallet: EncryWallet): WalletReader = new WalletReader { val accountManagers: Seq[AccountManager] = wallet.accountManagers + + override def publicKeys: Set[PublicKey25519] = wallet.publicKeys + + def getBalances: Seq[((String, String), Long)] = wallet.getBalances } def empty: WalletReader = new WalletReader { override val accountManagers: Seq[AccountManager] = Seq.empty + + override def publicKeys: Set[PublicKey25519] = Set.empty + + def getBalances: Seq[((String, String), Long)] = Seq.empty } } From 0982166831ba642713a18194b9673d79159a8008 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Mon, 30 Mar 2020 18:40:27 +0300 Subject: [PATCH 159/177] ec-fix --- src/main/scala/encry/nvg/NVHHistory.scala | 2 +- src/main/scala/encry/nvg/NodeViewNMProcessor.scala | 4 ++-- src/main/scala/encry/view/history/HistoryApi.scala | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index 4d78e9c59d..0f2b2cdeb7 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -142,7 +142,7 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) def requestDownloads(pi: ProgressInfo, previousModifier: Option[ModifierId] = none): Unit = pi.toDownload.foreach { case (tid: ModifierTypeId, id: ModifierId) => - if (tid != Payload.modifierTypeId) { + if (tid != Payload.modifierTypeId || (historyView.history.isFullChainSynced && tid == Payload.modifierTypeId)) { logger.info( s"History holder created download request for modifier ${Algos.encode(id)} of type $tid. " + s"Previous modifier is ${previousModifier.map(Algos.encode)}." diff --git a/src/main/scala/encry/nvg/NodeViewNMProcessor.scala b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala index d4a050029d..d3fa0a3ea0 100644 --- a/src/main/scala/encry/nvg/NodeViewNMProcessor.scala +++ b/src/main/scala/encry/nvg/NodeViewNMProcessor.scala @@ -83,8 +83,8 @@ class NodeViewNMProcessor(settings: EncryAppSettings) extends Actor with StrictL if (invData._1 == Payload.modifierTypeId && !historyReader.isFullChainSynced) logger.info(s"Got inv message from $remote with ${invData._2.size} ids but full chain is not synced.") else { - val isHeader: Boolean = invData._1 == Header.modifierTypeId && !historyReader.isHeadersChainSynced - val isPayloadAvailable: Boolean = false + val isHeader: Boolean = invData._1 == Header.modifierTypeId + val isPayloadAvailable: Boolean = historyReader.isHeadersChainSynced && invData._1 == Payload.modifierTypeId val isRequestAvailable: Boolean = isHeader || isPayloadAvailable if (isRequestAvailable) { val ids: Seq[ModifierId] = invData._2.filterNot { mid: ModifierId => diff --git a/src/main/scala/encry/view/history/HistoryApi.scala b/src/main/scala/encry/view/history/HistoryApi.scala index e2f9349a1b..3582b743e9 100644 --- a/src/main/scala/encry/view/history/HistoryApi.scala +++ b/src/main/scala/encry/view/history/HistoryApi.scala @@ -297,9 +297,9 @@ trait HistoryApi extends HistoryDBApi { //scalastyle:ignore case None => (None, headerChainBack(toHeader.height + 1, toHeader, _ => false)) } - def isNewHeader(header: Header): Boolean = header.height == 55000 -// timeProvider.estimatedTime - header.timestamp < -// settings.constants.DesiredBlockInterval.toMillis * settings.constants.NewHeaderTimeMultiplier + def isNewHeader(header: Header): Boolean = + timeProvider.estimatedTime - header.timestamp < + settings.constants.DesiredBlockInterval.toMillis * settings.constants.NewHeaderTimeMultiplier def isHeadersChainSynced: Boolean = isHeadersChainSyncedVar From 1a2f69a9180700bd0577bd3ff6638a2d90f17fa5 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 31 Mar 2020 10:29:27 +0300 Subject: [PATCH 160/177] more logging --- src/main/scala/encry/local/miner/Miner.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/encry/local/miner/Miner.scala b/src/main/scala/encry/local/miner/Miner.scala index 54ddaa8dda..82c2abe3c7 100644 --- a/src/main/scala/encry/local/miner/Miner.scala +++ b/src/main/scala/encry/local/miner/Miner.scala @@ -235,7 +235,7 @@ class Miner(dataHolder: ActorRef, val combinedStateChange: UtxoState.StateChange = combineAll(txs.map(UtxoState.tx2StateChange).toList) - logger.info(s"Root node hash before producing candidate: ${view.state.tree.rootNode.hash}") + logger.info(s"Root node hash before producing candidate: ${Algos.encode(view.state.tree.rootNode.hash)}") val newStateRoot = view.state.tree .getOperationsRootHash( From e61f6b7ac3f628f96fff3c48a9b96b39b14a46b1 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 31 Mar 2020 10:35:25 +0300 Subject: [PATCH 161/177] change logic of asking modifier from unknown node --- src/main/scala/encry/network/MessageBuilder.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/network/MessageBuilder.scala b/src/main/scala/encry/network/MessageBuilder.scala index 1e83e8745a..7e5138cda6 100644 --- a/src/main/scala/encry/network/MessageBuilder.scala +++ b/src/main/scala/encry/network/MessageBuilder.scala @@ -6,7 +6,7 @@ import akka.actor.{Actor, ActorRef, Props} import akka.pattern._ import akka.util.Timeout import com.typesafe.scalalogging.StrictLogging -import encry.consensus.HistoryConsensus.{Equal, Older, Unknown, Younger} +import encry.consensus.HistoryConsensus.{Equal, Fork, Older, Unknown, Younger} import encry.network.ConnectedPeersCollection.PeerInfo import encry.network.DM.{IsRequested, RequestSent, RequestStatus} import encry.network.MessageBuilder.{GetPeerInfo, GetPeers, MsgSent} @@ -41,7 +41,7 @@ case class MessageBuilder(peersKeeper: ActorRef, } case RequestFromLocal(None, modTypeId, modsIds) => Try { - (peersKeeper ? (MessageBuilder.PeerWithOlderHistory || MessageBuilder.PeerWithEqualHistory || MessageBuilder.PeerWithYoungerHistory || MessageBuilder.PeerWithUnknownHistory)).mapTo[ConnectedPeer].map { peer => + (peersKeeper ? (MessageBuilder.PeerWithOlderHistory || MessageBuilder.PeerWithEqualHistory || MessageBuilder.PeerWithForkHistory)).mapTo[ConnectedPeer].map { peer => logger.info(s"Going to req mods from ${peer.socketAddress} of type ${modTypeId}") (deliveryManager ? IsRequested(modsIds)).mapTo[RequestStatus].foreach { status => logger.info(s"Requested or received: ${status.requested.length}. Not request or not received: ${status.notRequested.length}") @@ -114,6 +114,7 @@ object MessageBuilder { val PeerWithOlderHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Older) val PeerWithYoungerHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Younger) val PeerWithUnknownHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Unknown) + val PeerWithForkHistory = GetPeerByPredicate((info: PeerInfo) => info.historyComparisonResult == Fork) def props(peersKeeper: ActorRef, deliveryManager: ActorRef): Props = Props(new MessageBuilder(peersKeeper, deliveryManager)) } \ No newline at end of file From d1614d08e0fa37aa9f9457af38d3cd0acce8b3e3 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 31 Mar 2020 10:40:21 +0300 Subject: [PATCH 162/177] change pipeline of update utxo state at inter nvh view --- src/main/scala/encry/nvg/IntermediaryNVHView.scala | 4 +++- src/main/scala/encry/nvg/NVHState.scala | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index 069bf36180..ff182d28aa 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -69,7 +69,9 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, def viewReceive(history: ActorRef, state: ActorRef, stateReader: UtxoStateReader): Receive = { case RegisterState(reader) => context.become(viewReceive(history, sender(), reader)) - case reader: UtxoStateReader => context.become(viewReceive(history, state, reader)) + case reader: UtxoStateReader => + logger.info("Update reader at inter nvh view") + context.become(viewReceive(history, state, reader)) case GetDataFromCurrentView(f: (CurrentView[HistoryReader, UtxoStateReader, WalletReader] => CandidateEnvelope)) => logger.info("Receive GetDataFromCurrentView on nvh") diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index 317427afb3..93c59448d9 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -51,8 +51,8 @@ class NVHState(influxRef: Option[ActorRef], var historyReader: HistoryReader, se } state = stateAfterApply logger.info(s"Successfully apply modifier: ${Algos.encode(modifier.id)} of type ${modifier.modifierTypeId}") - context.parent ! ModifierApplied(modifier) context.parent ! UtxoStateReader(state) + context.parent ! ModifierApplied(modifier) case Left(e: List[ModifierApplyError]) => logger.info(s"Application to state failed cause $e") context.parent ! ApplyFailed(modifier, e) From 6cafc82d38a72958777dfa76aaccf901fe71857d Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 31 Mar 2020 16:04:56 +0300 Subject: [PATCH 163/177] more logging --- src/main/scala/encry/nvg/ModifiersValidator.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/encry/nvg/ModifiersValidator.scala b/src/main/scala/encry/nvg/ModifiersValidator.scala index 214fbe6557..18ed3fae8e 100644 --- a/src/main/scala/encry/nvg/ModifiersValidator.scala +++ b/src/main/scala/encry/nvg/ModifiersValidator.scala @@ -48,7 +48,7 @@ class ModifiersValidator( ModifiersValidator.isSyntacticallyValid(modifier, settings.constants.ModifierIdSize) if (preSemanticValidation.isRight && syntacticValidation) { if (modifier.id.sameElements(id)) { - logger.debug(s"Modifier ${modifier.encodedId} is valid.") + logger.info(s"Modifier ${modifier.encodedId} is valid.") intermediaryNVH ! ValidatedModifierFromNetwork(modifierTypeId) nodeViewHolderRef ! ValidatedModifier(modifier) } else { @@ -75,7 +75,7 @@ class ModifiersValidator( ModifiersValidator.isSyntacticallyValid(modifier, settings.constants.ModifierIdSize) if (preSemanticValidation.isRight && syntacticValidation) { if (modifier.id.sameElements(id)) { - logger.debug(s"Modifier ${modifier.encodedId} is valid.") + logger.info(s"Modifier ${modifier.encodedId} is valid.") intermediaryNVH ! ValidatedModifierFromNetwork(modifierTypeId) nodeViewHolderRef ! ValidatedModifier(modifier) } else { From 698584a87ca394e7ad76ae2888bc1ff403a29229 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Tue, 31 Mar 2020 16:30:34 +0300 Subject: [PATCH 164/177] once more logging --- src/main/scala/encry/nvg/IntermediaryNVHView.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index ff182d28aa..0eef7b5fe4 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -23,6 +23,7 @@ import encry.view.wallet.WalletReader import io.iohk.iodb.ByteArrayWrapper import org.encryfoundation.common.modifiers.PersistentModifier import org.encryfoundation.common.modifiers.history.Block +import org.encryfoundation.common.utils.Algos import scala.concurrent.Future @@ -107,6 +108,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, if (!isModifierProcessingInProgress) getNextModifier() case ValidatedModifier(modifier: PersistentModifier) => + logger.info(s"Receive modifier (${Algos.encode(modifier.id)}) at inter nvh view") val isInHistory: Boolean = historyReader.isModifierDefined(modifier.id) val isInCache: Boolean = ModifiersCache.contains(NodeViewHolder.toKey(modifier.id)) if (isInHistory || isInCache) @@ -122,6 +124,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, settings, isLocallyGenerated = false ) + logger.info(s"isModifierProcessingInProgress: ${isModifierProcessingInProgress}") if (!isModifierProcessingInProgress) getNextModifier() case ModifierAppliedToHistory => isModifierProcessingInProgress = false; getNextModifier() case msg: ProgressInfoForState From 3e4bea84441a089ffb4a143ad4cfdd7a85ed7499 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Tue, 31 Mar 2020 17:18:56 +0300 Subject: [PATCH 165/177] fixed pipeline for non best block processing by history --- src/main/scala/encry/nvg/IntermediaryNVHView.scala | 5 +---- src/main/scala/encry/nvg/NVHHistory.scala | 3 ++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index 0eef7b5fe4..b7c1881da6 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -84,10 +84,6 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, case res => sender ! res } -// f(CurrentView(historyReader, stateReader, walletReader)) match { -// case resultFuture: Future[_] => resultFuture.pipeTo(sender) -// case result => sender ! result -// } case LocallyGeneratedModifier(modifier: Block) => logger.info(s"Self mined block: ${modifier}") @@ -131,6 +127,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, if msg.pi.chainSwitchingNeeded && msg.pi.branchPoint.exists( point => !stateReader.version.sameElements(point) ) => + //todo drop state here context.become( viewReceive( history, diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index 0f2b2cdeb7..c7660b286f 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -85,6 +85,7 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) } } else { logger.info(s"Progress info contains an empty toApply. Going to form request download.") + context.parent ! ModifierAppliedToHistory if (!isLocallyGenerated) requestDownloads(progressInfo, mod.id.some) context.parent ! HeightStatistics(historyView.history.getBestHeaderHeight, -1) //todo incorrect state height context.parent ! SemanticallySuccessfulModifier(mod) @@ -121,6 +122,7 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) case StateAction.ApplyFailed(mod, e) => val (newHistory: History, progressInfo: ProgressInfo) = historyView.history.reportModifierIsInvalid(mod) + historyView = historyView.copy(history = newHistory) context.parent ! SemanticallyFailedModification(mod, e) context.parent ! ProgressInfoForState( progressInfo, @@ -129,7 +131,6 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) HistoryReader(historyView.history) ) lastProgressInfo = progressInfo - historyView = historyView.copy(history = newHistory) case InitGenesisHistory => logger.info("Init in InitGenesisHistory") From f898e03ccbfbd59affa22bff2557613b2c8c9672 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Wed, 1 Apr 2020 11:07:49 +0300 Subject: [PATCH 166/177] more logging --- src/main/scala/encry/view/state/UtxoState.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/encry/view/state/UtxoState.scala b/src/main/scala/encry/view/state/UtxoState.scala index d00e7caf5f..5b5f16545b 100644 --- a/src/main/scala/encry/view/state/UtxoState.scala +++ b/src/main/scala/encry/view/state/UtxoState.scala @@ -79,7 +79,7 @@ final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], logger.info(s"Current root node hash: ${Algos.encode(tree.rootNode.hash)}") val lastTxId = block.payload.txs.last.id val totalFees: Amount = block.payload.txs.init.map(_.fee).sum - val validstartTime = System.nanoTime() + val validstartTime = System.currentTimeMillis() val res: Either[ValidationResult, List[Transaction]] = block.payload.txs.map(tx => { if (tx.id sameElements lastTxId) validate(tx, block.header.timestamp, Height @@ block.header.height, totalFees + EncrySupplyController.supplyAt(Height @@ block.header.height, constants)) @@ -87,13 +87,13 @@ final case class UtxoState(tree: AvlTree[StorageKey, StorageValue], }).toList .traverse(Validated.fromEither) .toEither - val validationTime = System.nanoTime() - validstartTime + val validationTime = System.currentTimeMillis() - validstartTime //todo: influx ref doesn't init during restart influxRef.foreach(_ ! UtxoStat( block.payload.txs.length, validationTime )) - logger.info(s"Validation time: ${validationTime/1000000} ms. Txs: ${block.payload.txs.length}") + logger.info(s"Validation time: ${validationTime/1000} s. Txs: ${block.payload.txs.length}") res.fold( err => { logger.info(s"Failed to state cause ${err.message}") From f57b9a79cccdba1ec5f6ddab71e403b155a1b03b Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 1 Apr 2020 17:14:12 +0300 Subject: [PATCH 167/177] fixed semantically failed mod case logging added --- .../scala/encry/nvg/IntermediaryNVHView.scala | 31 +++++++++++---- src/main/scala/encry/nvg/ModifiersCache.scala | 20 +++++++--- src/main/scala/encry/nvg/NVHHistory.scala | 38 ++++++++++++------- 3 files changed, 62 insertions(+), 27 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index b7c1881da6..e69f20f08d 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -22,7 +22,7 @@ import encry.view.state.UtxoStateReader import encry.view.wallet.WalletReader import io.iohk.iodb.ByteArrayWrapper import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.Block +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.utils.Algos import scala.concurrent.Future @@ -44,6 +44,8 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, var toApply = Set.empty[ByteArrayWrapper] + var idInAwait: String = "" + override def receive: Receive = awaitingViewActors() def awaitingViewActors(history: Option[ActorRef] = None, @@ -104,7 +106,13 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, if (!isModifierProcessingInProgress) getNextModifier() case ValidatedModifier(modifier: PersistentModifier) => - logger.info(s"Receive modifier (${Algos.encode(modifier.id)}) at inter nvh view") + logger.info(s"Receive modifier (${modifier.encodedId}) at inter nvh view. Additional info about this modifier: ${ + modifier match { + case h: Header => ("header ",h.encodedId, h.height, Algos.encode(h.parentId)) + case p: Payload => ("payload ", p.encodedId, p.headerId) + case b: Block => ("block", b.encodedId, b.header.height, b.payload.encodedId) + } + }") val isInHistory: Boolean = historyReader.isModifierDefined(modifier.id) val isInCache: Boolean = ModifiersCache.contains(NodeViewHolder.toKey(modifier.id)) if (isInHistory || isInCache) @@ -112,7 +120,8 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, s"Modifier ${modifier.encodedId} can't be placed into the cache cause: " + s"contains in cache: $isInCache, contains in history: $isInHistory." ) - else + else { + logger.info(s"Going to put mod ${modifier.encodedId} to the cache.") ModifiersCache.put( NodeViewHolder.toKey(modifier.id), modifier, @@ -120,9 +129,13 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, settings, isLocallyGenerated = false ) - logger.info(s"isModifierProcessingInProgress: ${isModifierProcessingInProgress}") + } + logger.info(s"isModifierProcessingInProgress: $isModifierProcessingInProgress") if (!isModifierProcessingInProgress) getNextModifier() - case ModifierAppliedToHistory => isModifierProcessingInProgress = false; getNextModifier() + case ModifierAppliedToHistory => + logger.info(s"Got ModifierAppliedToHistory. In await: $idInAwait") + isModifierProcessingInProgress = false + getNextModifier() case msg: ProgressInfoForState if msg.pi.chainSwitchingNeeded && msg.pi.branchPoint.exists( point => !stateReader.version.sameElements(point) @@ -153,15 +166,19 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, def awaitingHistoryBranchPoint(history: ActorRef): Receive = ??? - def getNextModifier(): Unit = + def getNextModifier(): Unit = { + logger.info(s"Going to getNextModifier!") ModifiersCache .popCandidate(historyReader, settings) .foreach { case (mod: PersistentModifier, isLocallyGenerated) => isModifierProcessingInProgress = true - logger.info(s"Got new modifiers in compute application function: ${mod.encodedId}.") + logger.info(s"Got new modifiers in getNextModifier function: ${mod.encodedId}.") historyRef ! ModifierToAppend(mod, isLocallyGenerated) + idInAwait = mod.encodedId + logger.info(s"Start awaiting $idInAwait for appending. In await. ") } + } } diff --git a/src/main/scala/encry/nvg/ModifiersCache.scala b/src/main/scala/encry/nvg/ModifiersCache.scala index 26747abc52..e8de4c080d 100644 --- a/src/main/scala/encry/nvg/ModifiersCache.scala +++ b/src/main/scala/encry/nvg/ModifiersCache.scala @@ -3,11 +3,12 @@ package encry.nvg import com.typesafe.scalalogging.StrictLogging import encry.settings.EncryAppSettings import encry.view.history.HistoryReader -import encry.view.history.ValidationError.{ FatalValidationError, NonFatalValidationError } +import encry.view.history.ValidationError.{FatalValidationError, NonFatalValidationError} import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.Header +import org.encryfoundation.common.modifiers.history.{Header, Payload} import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.ModifierId + import scala.collection.concurrent.TrieMap import scala.collection.immutable.SortedMap import scala.collection.mutable @@ -104,8 +105,10 @@ object ModifiersCache extends StrictLogging { case (k, v) => v._1 match { case _: Header if history.getBestHeaderId.exists(_ sameElements v._1.parentId) => true - case _ => + case p: Payload => val isApplicableMod: Boolean = isApplicable(k) + logger.info(s"exhaustiveSearch. Payload. ${p.encodedId}. ${Algos.encode(p.headerId)}. " + + s"isApplicableMod: $isApplicableMod.") isApplicableMod } }).collect { case Some(v) => v._1 } @@ -140,9 +143,13 @@ object ModifiersCache extends StrictLogging { } } if (bestHeadersIds.nonEmpty) bestHeadersIds - else + else { + logger.info(s"Cache. Else condition. history.getBestBlockHeight: ${history.getBestBlockHeight}" + + s" history.headerIdsAtHeight(history.getBestBlockHeight + 1): ${history.headerIdsAtHeight(history.getBestBlockHeight + 1)} " + + s" ") history.headerIdsAtHeight(history.getBestBlockHeight + 1).headOption match { case Some(id) => + logger.info(s"Cache. Some. Id: ${Algos.encode(id)}.") history.getHeaderById(id) match { case Some(header: Header) if isApplicable(new mutable.WrappedArray.ofByte(header.payloadId)) => List(new mutable.WrappedArray.ofByte(header.payloadId)) @@ -150,11 +157,12 @@ object ModifiersCache extends StrictLogging { case _ => List.empty[Key] } case None if isChainSynced => - logger.debug(s"No payloads for current history") + logger.info(s"No payloads for current history") exhaustiveSearch case None => - logger.debug(s"No payloads for current history") + logger.info(s"No payloads for current history") List.empty[Key] } + } } } diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index c7660b286f..ce4c153bea 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -2,7 +2,7 @@ package encry.nvg import java.io.File -import akka.actor.{Actor, Props} +import akka.actor.{ Actor, Props } import cats.syntax.option._ import com.typesafe.scalalogging.StrictLogging import encry.EncryApp @@ -12,23 +12,27 @@ import encry.local.miner.Miner.EnableMining import encry.network.DeliveryManager.FullBlockChainIsSynced import encry.network.Messages.MessageToNetwork.RequestFromLocal import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.RegisterNodeView -import encry.nvg.IntermediaryNVHView.{InitGenesisHistory, ModifierToAppend} -import encry.nvg.NVHHistory.{ModifierAppliedToHistory, NewWalletReader, ProgressInfoForState} +import encry.nvg.IntermediaryNVHView.{ InitGenesisHistory, ModifierToAppend } +import encry.nvg.NVHHistory.{ ModifierAppliedToHistory, NewWalletReader, ProgressInfoForState } import encry.nvg.NVHState.StateAction -import encry.nvg.NodeViewHolder.{SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification} +import encry.nvg.NodeViewHolder.{ + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification +} import encry.settings.EncryAppSettings import encry.stats.StatsSender._ import encry.utils.CoreTaggedTypes.VersionTag import encry.utils.NetworkTimeProvider import encry.view.NodeViewErrors.ModifierApplyError.HistoryApplyError import encry.view.history.History.HistoryUpdateInfoAcc -import encry.view.history.{History, HistoryReader} -import encry.view.wallet.{EncryWallet, WalletReader} +import encry.view.history.{ History, HistoryReader } +import encry.view.wallet.{ EncryWallet, WalletReader } import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.utils.Algos -import org.encryfoundation.common.utils.TaggedTypes.{ModifierId, ModifierTypeId} +import org.encryfoundation.common.utils.TaggedTypes.{ ModifierId, ModifierTypeId } class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) extends Actor @@ -43,6 +47,8 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) context.parent ! RegisterNodeView(HistoryReader(historyView.history), WalletReader(historyView.wallet)) + var modsInToApply: List[String] = List.empty[String] + override def postStop(): Unit = println("stop!") override def receive: Receive = { @@ -66,12 +72,13 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) historyView.history.insertUpdateInfo(newUpdateInformation) if (mod.modifierTypeId == Header.modifierTypeId) historyView.history.updateIdsForSyncInfo() context.parent ! EndOfApplyingModifier(mod.id) - context.parent ! ModifierAppendedToHistory( mod match { - case _: Header => true - case _: Payload => false - }, success = true ) + context.parent ! ModifierAppendedToHistory(mod match { + case _: Header => true + case _: Payload => false + }, success = true) if (progressInfo.toApply.nonEmpty) { logger.info(s"Progress info contains an non empty toApply. Going to notify state about new toApply.") + modsInToApply = progressInfo.toApply.map(_.encodedId).toList context.parent ! ProgressInfoForState( progressInfo, (historyView.history.getBestHeaderHeight - historyView.history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2, @@ -115,15 +122,18 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) historyView.wallet.rollback(VersionTag !@@ lastProgressInfo.branchPoint.get).get historyView.wallet.scanPersistent(mod) context.parent ! NewWalletReader(WalletReader(historyView.wallet)) - context.parent ! ModifierAppliedToHistory context.parent ! SemanticallySuccessfulModifier(mod) if (historyView.history.isFullChainSynced) context.system.eventStream.publish(FullBlockChainIsSynced) - if (settings.node.mining && historyView.history.isFullChainSynced) context.system.eventStream.publish(EnableMining) + if (settings.node.mining && historyView.history.isFullChainSynced) + context.system.eventStream.publish(EnableMining) + modsInToApply = modsInToApply.filterNot(_ == mod.encodedId) + if (modsInToApply.isEmpty) context.parent ! ModifierAppliedToHistory case StateAction.ApplyFailed(mod, e) => val (newHistory: History, progressInfo: ProgressInfo) = historyView.history.reportModifierIsInvalid(mod) historyView = historyView.copy(history = newHistory) context.parent ! SemanticallyFailedModification(mod, e) + modsInToApply = progressInfo.toApply.map(_.encodedId).toList context.parent ! ProgressInfoForState( progressInfo, (historyView.history.getBestHeaderHeight - historyView.history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2, From 1acfe9fd969e841b63028d1088b251fd456d95ab Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 1 Apr 2020 17:32:23 +0300 Subject: [PATCH 168/177] added statistics --- .../scala/encry/nvg/IntermediaryNVHView.scala | 49 +++++++++++++------ src/main/scala/encry/nvg/NVHHistory.scala | 1 + src/main/scala/encry/nvg/NodeViewHolder.scala | 15 +----- src/main/scala/encry/stats/StatsSender.scala | 10 ++-- .../encry/view/history/HistoryReader.scala | 8 +++ 5 files changed, 50 insertions(+), 33 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index e69f20f08d..acf29bf2b9 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -1,18 +1,23 @@ package encry.nvg -import akka.actor.{Actor, ActorRef, Props, Stash} +import akka.actor.{ Actor, ActorRef, Props, Stash } import akka.pattern._ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.local.miner.Miner.CandidateEnvelope import encry.network.Messages.MessageToNetwork.RequestFromLocal -import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{RegisterNodeView, RegisterState} -import encry.nvg.IntermediaryNVHView.{ModifierToAppend, NodeViewStarted} +import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{ RegisterNodeView, RegisterState } +import encry.nvg.IntermediaryNVHView.{ ModifierToAppend, NodeViewStarted } import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NVHHistory.{ModifierAppliedToHistory, NewWalletReader, ProgressInfoForState} +import encry.nvg.NVHHistory.{ ModifierAppliedToHistory, NewWalletReader, ProgressInfoForState } import encry.nvg.NVHState.StateAction import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier -import encry.nvg.NodeViewHolder.{GetDataFromCurrentView, SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification} +import encry.nvg.NodeViewHolder.{ + GetDataFromCurrentView, + SemanticallyFailedModification, + SemanticallySuccessfulModifier, + SyntacticallyFailedModification +} import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider @@ -22,8 +27,9 @@ import encry.view.state.UtxoStateReader import encry.view.wallet.WalletReader import io.iohk.iodb.ByteArrayWrapper import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} +import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } import org.encryfoundation.common.utils.Algos +import scala.concurrent.duration._ import scala.concurrent.Future @@ -46,6 +52,20 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, var idInAwait: String = "" + context.system.scheduler.schedule(1.seconds, 10.seconds) { + logger.info( + s"\n\n History best header id is: ${historyReader.getBestHeaderId.map(Algos.encode)}.\n " + + s"History best header height is: ${historyReader.getBestHeaderHeight}.\n " + + s"History best block id is: ${historyReader.getBestBlockId.map(Algos.encode)}.\n " + + s"History best block height is: ${historyReader.getBestBlockHeight}.\n " + + s"History best block header is: ${historyReader.getHeaderOfBestBlock.map(_.encodedId)}.\n " + + s"Cache size is: ${ModifiersCache.size}.\n " + ) + logger.info( + s"Cache elements are: ${ModifiersCache.cache.keys.toList.map(key => Algos.encode(key.toArray)).mkString(",")}." + ) + } + override def receive: Receive = awaitingViewActors() def awaitingViewActors(history: Option[ActorRef] = None, @@ -106,13 +126,13 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, if (!isModifierProcessingInProgress) getNextModifier() case ValidatedModifier(modifier: PersistentModifier) => - logger.info(s"Receive modifier (${modifier.encodedId}) at inter nvh view. Additional info about this modifier: ${ - modifier match { - case h: Header => ("header ",h.encodedId, h.height, Algos.encode(h.parentId)) + logger.info( + s"Receive modifier (${modifier.encodedId}) at inter nvh view. Additional info about this modifier: ${modifier match { + case h: Header => ("header ", h.encodedId, h.height, Algos.encode(h.parentId)) case p: Payload => ("payload ", p.encodedId, p.headerId) - case b: Block => ("block", b.encodedId, b.header.height, b.payload.encodedId) - } - }") + case b: Block => ("block", b.encodedId, b.header.height, b.payload.encodedId) + }}" + ) val isInHistory: Boolean = historyReader.isModifierDefined(modifier.id) val isInCache: Boolean = ModifiersCache.contains(NodeViewHolder.toKey(modifier.id)) if (isInHistory || isInCache) @@ -186,8 +206,9 @@ object IntermediaryNVHView { sealed trait IntermediaryNVHViewActions object IntermediaryNVHViewActions { - case class RegisterNodeView(historyReader: HistoryReader, walletReader: WalletReader) extends IntermediaryNVHViewActions - case class RegisterState(stateReader: UtxoStateReader) extends IntermediaryNVHViewActions + case class RegisterNodeView(historyReader: HistoryReader, walletReader: WalletReader) + extends IntermediaryNVHViewActions + case class RegisterState(stateReader: UtxoStateReader) extends IntermediaryNVHViewActions } final case class ModifierToAppend(modifier: PersistentModifier, isLocallyGenerated: Boolean) diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index ce4c153bea..731ab0a3b1 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -106,6 +106,7 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) case StateAction.ModifierApplied(mod: PersistentModifier) => val newHistory = historyView.history.reportModifierIsValid(mod) historyView = historyView.copy(history = newHistory) + historyView.history.getBestHeader.foreach(context.parent ! BestHeaderInChain(_)) context.parent ! HistoryReader(historyView.history) context.parent ! BlockAndHeaderInfo(historyView.history.getBestHeader, historyView.history.getBestBlock) if (historyView.history.getBestHeaderId.exists( diff --git a/src/main/scala/encry/nvg/NodeViewHolder.scala b/src/main/scala/encry/nvg/NodeViewHolder.scala index ef0ba859b6..7f182b9dc5 100644 --- a/src/main/scala/encry/nvg/NodeViewHolder.scala +++ b/src/main/scala/encry/nvg/NodeViewHolder.scala @@ -31,20 +31,7 @@ import scala.collection.{IndexedSeq, mutable} // context.parent ! UpdateHistoryReader(HistoryReader(nodeView.history)) // context.parent ! BlockAndHeaderInfo(nodeView.history.getBestHeader, nodeView.history.getBestBlock) // -// context.system.scheduler.schedule(1.seconds, 10.seconds) { -// logger.info( -// s"\n\n History best header id is: ${nodeView.history.getBestHeaderId.map(Algos.encode)}.\n " + -// s"History best header height is: ${nodeView.history.getBestHeaderHeight}.\n " + -// s"History best block id is: ${nodeView.history.getBestBlockId.map(Algos.encode)}.\n " + -// s"History best block height is: ${nodeView.history.getBestBlockHeight}.\n " + -// s"History best block header is: ${nodeView.history.getHeaderOfBestBlock.map(_.encodedId)}.\n " + -// s"State height is: ${nodeView.state.height}.\n " + -// s"Cache size is: ${ModifiersCache.size}.\n " -// ) -// logger.info( -// s"Cache elements are: ${ModifiersCache.cache.keys.toList.map(key => Algos.encode(key.toArray)).mkString(",")}." -// ) -// } + // // override def receive: Receive = { // case ValidatedModifier(modifier: PersistentModifier) => diff --git a/src/main/scala/encry/stats/StatsSender.scala b/src/main/scala/encry/stats/StatsSender.scala index 3295f2c3c1..eebf4ac81c 100644 --- a/src/main/scala/encry/stats/StatsSender.scala +++ b/src/main/scala/encry/stats/StatsSender.scala @@ -41,14 +41,14 @@ class StatsSender(influxDBSettings: InfluxDBSettings, networkSettings: NetworkSe influxDB.write( influxDBSettings.udpPort, util.Arrays.asList( - s"difficulty,nodeName=$nodeName diff=${fb.difficulty.toString},height=${fb.height}", //++ - s"""height,nodeName=$nodeName header="${fb.encodedId}",height=${fb.height}""", //++ + s"difficulty,nodeName=$nodeName diff=${fb.difficulty.toString},height=${fb.height}", + s"""height,nodeName=$nodeName header="${fb.encodedId}",height=${fb.height}""", s"stateWeight,nodeName=$nodeName,height=${fb.height} " + - s"value=${new File("encry/data/state/").listFiles.foldLeft(0L)(_ + _.length())}", //++ + s"value=${new File("encry/data/state/").listFiles.foldLeft(0L)(_ + _.length())}", s"historyWeight,nodeName=$nodeName,height=${fb.height} " + - s"value=${new File("encry/data/history/").listFiles.foldLeft(0L)(_ + _.length())}", //++ + s"value=${new File("encry/data/history/").listFiles.foldLeft(0L)(_ + _.length())}", s"supply,nodeName=$nodeName,height=${fb.height} " + - s"value=${EncrySupplyController.supplyAt(fb.height.asInstanceOf[Height], constants)}" //++ + s"value=${EncrySupplyController.supplyAt(fb.height.asInstanceOf[Height], constants)}" ) ) diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 8673a90f5b..1e7236a14d 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -59,6 +59,10 @@ trait HistoryReader { def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] def getHeaderIds(count: Int, offset: Int = 0): Seq[ModifierId] + + def getBestBlockId: Option[ModifierId] + + def getHeaderOfBestBlock: Option[Header] } object HistoryReader { @@ -92,6 +96,8 @@ object HistoryReader { def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] = Left(HistoryApiError("")) def isHeadersChainSynced: Boolean = false def getHeaderIds(count: Int, offset: Int = 0): Seq[ModifierId] = Seq.empty + def getBestBlockId: Option[ModifierId] = None + def getHeaderOfBestBlock: Option[Header] = None } def apply(history: History): HistoryReader = new HistoryReader { @@ -125,5 +131,7 @@ object HistoryReader { history.requiredDifficultyAfter(parent) def isHeadersChainSynced: Boolean = history.isHeadersChainSynced def getHeaderIds(count: Int, offset: Int = 0): Seq[ModifierId] = history.getHeaderIds(count, offset) + def getBestBlockId: Option[ModifierId] = history.getBestBlockId + def getHeaderOfBestBlock: Option[Header] = history.getHeaderOfBestBlock } } From f77935da7e8e64ce029d65708872e592905e9e46 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 1 Apr 2020 18:49:01 +0300 Subject: [PATCH 169/177] more logging --- src/main/scala/encry/nvg/NVHHistory.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index 731ab0a3b1..56b0cdd001 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -111,8 +111,13 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) context.parent ! BlockAndHeaderInfo(historyView.history.getBestHeader, historyView.history.getBestBlock) if (historyView.history.getBestHeaderId.exists( besId => historyView.history.getBestBlockId.exists(_.sameElements(besId)) - )) + )) { + logger.info(s"\n\n\nGot message StateAction.ModifierApplied with mod ${mod.encodedId} of type ${mod.modifierTypeId}. " + + s"Set up historyView.history.isFullChainSynced = true. Condition is: " + + s" (historyView.history.getBestHeaderId: ${(historyView.history.getBestHeaderId.map(Algos.encode))}. " + + s" (historyView.history.getBestBlockId.exists(_.sameElements(besId)): ${historyView.history.getBestBlockId.map(Algos.encode)}.\n\n\n") historyView.history.isFullChainSynced = true + } context.parent ! HeightStatistics(historyView.history.getBestHeaderHeight, -1) //todo incorrect state height if (mod match { case _: Block => true From be6f7be04a167f2d2bb63f53a7d5764611448c02 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 1 Apr 2020 18:49:53 +0300 Subject: [PATCH 170/177] added ModifiersCache.setChainSynced() to history --- src/main/scala/encry/nvg/NVHHistory.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index 56b0cdd001..c79fa40ba5 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -117,6 +117,7 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) s" (historyView.history.getBestHeaderId: ${(historyView.history.getBestHeaderId.map(Algos.encode))}. " + s" (historyView.history.getBestBlockId.exists(_.sameElements(besId)): ${historyView.history.getBestBlockId.map(Algos.encode)}.\n\n\n") historyView.history.isFullChainSynced = true + ModifiersCache.setChainSynced() } context.parent ! HeightStatistics(historyView.history.getBestHeaderHeight, -1) //todo incorrect state height if (mod match { From 3510d95a328d837e913240cf686f7de98b5bc02a Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 1 Apr 2020 19:07:36 +0300 Subject: [PATCH 171/177] more logging --- src/main/scala/encry/nvg/ModifiersCache.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/scala/encry/nvg/ModifiersCache.scala b/src/main/scala/encry/nvg/ModifiersCache.scala index e8de4c080d..ec7d2c5604 100644 --- a/src/main/scala/encry/nvg/ModifiersCache.scala +++ b/src/main/scala/encry/nvg/ModifiersCache.scala @@ -110,6 +110,10 @@ object ModifiersCache extends StrictLogging { logger.info(s"exhaustiveSearch. Payload. ${p.encodedId}. ${Algos.encode(p.headerId)}. " + s"isApplicableMod: $isApplicableMod.") isApplicableMod + case sm: _ => + val isApplicableMod: Boolean = isApplicable(k) + logger.info(s"exhaustiveSearch. Something. ${sm.encodedId}. isApplicableMod: $isApplicableMod.") + isApplicableMod } }).collect { case Some(v) => v._1 } From 3a2525c54cd5248471e354b36040ecec68b5ded7 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 1 Apr 2020 19:08:51 +0300 Subject: [PATCH 172/177] more logging --- src/main/scala/encry/nvg/ModifiersCache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/encry/nvg/ModifiersCache.scala b/src/main/scala/encry/nvg/ModifiersCache.scala index ec7d2c5604..a400f840e7 100644 --- a/src/main/scala/encry/nvg/ModifiersCache.scala +++ b/src/main/scala/encry/nvg/ModifiersCache.scala @@ -110,7 +110,7 @@ object ModifiersCache extends StrictLogging { logger.info(s"exhaustiveSearch. Payload. ${p.encodedId}. ${Algos.encode(p.headerId)}. " + s"isApplicableMod: $isApplicableMod.") isApplicableMod - case sm: _ => + case sm => val isApplicableMod: Boolean = isApplicable(k) logger.info(s"exhaustiveSearch. Something. ${sm.encodedId}. isApplicableMod: $isApplicableMod.") isApplicableMod From dbbd96451e857ae61ff74ecfc472528fe8b493b4 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 1 Apr 2020 19:14:41 +0300 Subject: [PATCH 173/177] logging removed --- src/main/scala/encry/nvg/ModifiersCache.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/scala/encry/nvg/ModifiersCache.scala b/src/main/scala/encry/nvg/ModifiersCache.scala index a400f840e7..b0e9bfcc57 100644 --- a/src/main/scala/encry/nvg/ModifiersCache.scala +++ b/src/main/scala/encry/nvg/ModifiersCache.scala @@ -110,9 +110,8 @@ object ModifiersCache extends StrictLogging { logger.info(s"exhaustiveSearch. Payload. ${p.encodedId}. ${Algos.encode(p.headerId)}. " + s"isApplicableMod: $isApplicableMod.") isApplicableMod - case sm => + case _ => val isApplicableMod: Boolean = isApplicable(k) - logger.info(s"exhaustiveSearch. Something. ${sm.encodedId}. isApplicableMod: $isApplicableMod.") isApplicableMod } }).collect { case Some(v) => v._1 } From 50b56fe5fde9e7e8b1a37a392642a1b130b0ee65 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 1 Apr 2020 20:00:23 +0300 Subject: [PATCH 174/177] more additions --- src/main/scala/encry/Starter.scala | 2 +- .../scala/encry/nvg/IntermediaryNVHView.scala | 3 +- src/main/scala/encry/nvg/ModifiersCache.scala | 4 +- .../encry/view/history/HistoryReader.scala | 78 ++++++++++--------- 4 files changed, 47 insertions(+), 40 deletions(-) diff --git a/src/main/scala/encry/Starter.scala b/src/main/scala/encry/Starter.scala index 2faca868ea..dfe89363cf 100644 --- a/src/main/scala/encry/Starter.scala +++ b/src/main/scala/encry/Starter.scala @@ -426,7 +426,7 @@ class Starter(settings: EncryAppSettings, ) val miner: ActorRef = context.system.actorOf(Miner.props(dataHolderForApi, memoryPool, nvhRouter, influxRef, newSettings, timeProvider), "miner") - //if (newSettings.node.mining) miner ! StartMining + if (newSettings.node.mining) miner ! StartMining if (newSettings.node.useCli) { context.system .actorOf(ConsoleListener.props(newSettings, dataHolderForApi, nodeId, timeProvider), "cliListener") diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index acf29bf2b9..3c7728ba8e 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -196,7 +196,8 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, logger.info(s"Got new modifiers in getNextModifier function: ${mod.encodedId}.") historyRef ! ModifierToAppend(mod, isLocallyGenerated) idInAwait = mod.encodedId - logger.info(s"Start awaiting $idInAwait for appending. In await. ") + logger.info(s"Start awaiting $idInAwait for appending. In await. " + + s"reader is full synced ${historyReader.isFullChainSynced}") } } diff --git a/src/main/scala/encry/nvg/ModifiersCache.scala b/src/main/scala/encry/nvg/ModifiersCache.scala index b0e9bfcc57..3244b2b2d8 100644 --- a/src/main/scala/encry/nvg/ModifiersCache.scala +++ b/src/main/scala/encry/nvg/ModifiersCache.scala @@ -156,7 +156,9 @@ object ModifiersCache extends StrictLogging { history.getHeaderById(id) match { case Some(header: Header) if isApplicable(new mutable.WrappedArray.ofByte(header.payloadId)) => List(new mutable.WrappedArray.ofByte(header.payloadId)) - case _ if history.isFullChainSynced => exhaustiveSearch + case _ if history.isFullChainSynced => + logger.info(s"Cache. history.getHeaderById(id.) _ => exhaustiveSearch") + exhaustiveSearch case _ => List.empty[Key] } case None if isChainSynced => diff --git a/src/main/scala/encry/view/history/HistoryReader.scala b/src/main/scala/encry/view/history/HistoryReader.scala index 1e7236a14d..11e7d63b8e 100644 --- a/src/main/scala/encry/view/history/HistoryReader.scala +++ b/src/main/scala/encry/view/history/HistoryReader.scala @@ -1,12 +1,13 @@ package encry.view.history -import encry.consensus.HistoryConsensus.{ HistoryComparisonResult, Older } +import com.typesafe.scalalogging.StrictLogging +import encry.consensus.HistoryConsensus.{HistoryComparisonResult, Older} import encry.modifiers.history.HeaderChain import encry.view.history.ValidationError.HistoryApiError import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header } +import org.encryfoundation.common.modifiers.history.{Block, Header} import org.encryfoundation.common.network.SyncInfo -import org.encryfoundation.common.utils.TaggedTypes.{ Difficulty, ModifierId } +import org.encryfoundation.common.utils.TaggedTypes.{Difficulty, ModifierId} import scala.collection.immutable.HashSet @@ -65,7 +66,7 @@ trait HistoryReader { def getHeaderOfBestBlock: Option[Header] } -object HistoryReader { +object HistoryReader extends StrictLogging { def empty: HistoryReader = new HistoryReader { def isModifierDefined(id: ModifierId): Boolean = false def getBestHeaderHeight: Int = -1 @@ -100,38 +101,41 @@ object HistoryReader { def getHeaderOfBestBlock: Option[Header] = None } - def apply(history: History): HistoryReader = new HistoryReader { - def isModifierDefined(id: ModifierId): Boolean = history.isModifierDefined(id) - def getBestHeaderHeight: Int = history.getBestHeaderHeight - def getBestBlockHeight: Int = history.getBestBlockHeight - def getBestHeaderAtHeight(h: Int): Option[Header] = history.getBestHeaderAtHeight(h) - def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = history.continuationIds(info, size) - def compare(si: SyncInfo): HistoryComparisonResult = history.compare(si) - var isFullChainSynced: Boolean = history.isFullChainSynced - def modifierBytesById(id: ModifierId): Option[Array[Byte]] = history.modifierBytesById(id) - def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = history.payloadsIdsToDownload(howMany, HashSet.empty) - def syncInfo: SyncInfo = history.syncInfo - def isFastSyncInProcess: Boolean = history.fastSyncInProgress.fastSyncVal - def getHeaderById(id: ModifierId): Option[Header] = history.getHeaderById(id) - def headerIdsAtHeight(height: Int): List[ModifierId] = history.headerIdsAtHeight(height).toList - def getBlockByHeaderId(id: ModifierId): Option[Block] = history.getBlockByHeaderId(id) - def getBlockByHeader(header: Header): Option[Block] = history.getBlockByHeader(header) - def lastHeaders(count: Int): HeaderChain = history.lastHeaders(count) - def getBestHeader: Option[Header] = history.getBestHeader - def getBestBlock: Option[Block] = history.getBestBlock - def getChainToHeader( - fromHeaderOpt: Option[Header], - toHeader: Header - ): (Option[ModifierId], HeaderChain) = - history.getChainToHeader(fromHeaderOpt, toHeader) - def testApplicable(modifier: PersistentModifier): Either[ValidationError, PersistentModifier] = - history.testApplicable(modifier) - def getBestHeaderId: Option[ModifierId] = history.getBestHeaderId - def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] = - history.requiredDifficultyAfter(parent) - def isHeadersChainSynced: Boolean = history.isHeadersChainSynced - def getHeaderIds(count: Int, offset: Int = 0): Seq[ModifierId] = history.getHeaderIds(count, offset) - def getBestBlockId: Option[ModifierId] = history.getBestBlockId - def getHeaderOfBestBlock: Option[Header] = history.getHeaderOfBestBlock + def apply(history: History): HistoryReader = { + logger.info(s"Going to create HistoryReader with isFullChainSynced: ${history.isFullChainSynced}") + new HistoryReader { + def isModifierDefined(id: ModifierId): Boolean = history.isModifierDefined(id) + def getBestHeaderHeight: Int = history.getBestHeaderHeight + def getBestBlockHeight: Int = history.getBestBlockHeight + def getBestHeaderAtHeight(h: Int): Option[Header] = history.getBestHeaderAtHeight(h) + def continuationIds(info: SyncInfo, size: Int): Seq[ModifierId] = history.continuationIds(info, size) + def compare(si: SyncInfo): HistoryComparisonResult = history.compare(si) + var isFullChainSynced: Boolean = history.isFullChainSynced + def modifierBytesById(id: ModifierId): Option[Array[Byte]] = history.modifierBytesById(id) + def payloadsIdsToDownload(howMany: Int): Seq[ModifierId] = history.payloadsIdsToDownload(howMany, HashSet.empty) + def syncInfo: SyncInfo = history.syncInfo + def isFastSyncInProcess: Boolean = history.fastSyncInProgress.fastSyncVal + def getHeaderById(id: ModifierId): Option[Header] = history.getHeaderById(id) + def headerIdsAtHeight(height: Int): List[ModifierId] = history.headerIdsAtHeight(height).toList + def getBlockByHeaderId(id: ModifierId): Option[Block] = history.getBlockByHeaderId(id) + def getBlockByHeader(header: Header): Option[Block] = history.getBlockByHeader(header) + def lastHeaders(count: Int): HeaderChain = history.lastHeaders(count) + def getBestHeader: Option[Header] = history.getBestHeader + def getBestBlock: Option[Block] = history.getBestBlock + def getChainToHeader( + fromHeaderOpt: Option[Header], + toHeader: Header + ): (Option[ModifierId], HeaderChain) = + history.getChainToHeader(fromHeaderOpt, toHeader) + def testApplicable(modifier: PersistentModifier): Either[ValidationError, PersistentModifier] = + history.testApplicable(modifier) + def getBestHeaderId: Option[ModifierId] = history.getBestHeaderId + def requiredDifficultyAfter(parent: Header): Either[HistoryApiError, Difficulty] = + history.requiredDifficultyAfter(parent) + def isHeadersChainSynced: Boolean = history.isHeadersChainSynced + def getHeaderIds(count: Int, offset: Int = 0): Seq[ModifierId] = history.getHeaderIds(count, offset) + def getBestBlockId: Option[ModifierId] = history.getBestBlockId + def getHeaderOfBestBlock: Option[Header] = history.getHeaderOfBestBlock + } } } From 6daf37d48bf71ee4519c04b5d7adb6d12e2e8b76 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 1 Apr 2020 21:06:19 +0300 Subject: [PATCH 175/177] logging --- .../scala/encry/nvg/IntermediaryNVHView.scala | 18 +++++++++++++----- src/main/scala/encry/nvg/NVHHistory.scala | 9 ++++++++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index 3c7728ba8e..de68115655 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -170,10 +170,16 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, ) case msg: ProgressInfoForState => toApply = msg.pi.toApply.map(mod => ByteArrayWrapper(mod.id)).toSet - msg.pi.toApply.foreach(mod => state ! StateAction.ApplyModifier(mod, msg.saveRootNodeFlag, msg.isFullChainSynced)) - case msg: StateAction.ApplyFailed => historyRef ! msg - case NewWalletReader(reader) => walletReader = reader + msg.pi.toApply.foreach { mod => + logger.info(s"mid to state: ${mod.encodedId}.") + state ! StateAction.ApplyModifier(mod, msg.saveRootNodeFlag, msg.isFullChainSynced) + } + case msg: StateAction.ApplyFailed => + logger.info(s"Mid to history. inWait $idInAwait, StateAction.ApplyFailed mod: ${msg.modifierId.encodedId}.") + historyRef ! msg + case NewWalletReader(reader) => walletReader = reader case msg: StateAction.ModifierApplied => + logger.info(s"Mid to history. inWait $idInAwait, StateAction.ModifierApplied mod: ${msg.modifierId.encodedId}.") historyRef ! msg case msg: SyntacticallyFailedModification => context.parent ! msg case msg: StatsSenderMessage => context.parent ! msg @@ -196,8 +202,10 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, logger.info(s"Got new modifiers in getNextModifier function: ${mod.encodedId}.") historyRef ! ModifierToAppend(mod, isLocallyGenerated) idInAwait = mod.encodedId - logger.info(s"Start awaiting $idInAwait for appending. In await. " + - s"reader is full synced ${historyReader.isFullChainSynced}") + logger.info( + s"Start awaiting $idInAwait for appending. In await. " + + s"reader is full synced ${historyReader.isFullChainSynced}" + ) } } diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index c79fa40ba5..39eb30a315 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -104,6 +104,8 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) logger.info(s"Got modifier ${mod.encodedId} on history actor which already contains in history.") case StateAction.ModifierApplied(mod: PersistentModifier) => + logger.info(s"History actor got StateAction.ModifierApplied. id: ${mod.encodedId}. " + + s"In await: ${modsInToApply.mkString(",")}.") val newHistory = historyView.history.reportModifierIsValid(mod) historyView = historyView.copy(history = newHistory) historyView.history.getBestHeader.foreach(context.parent ! BestHeaderInChain(_)) @@ -134,13 +136,18 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) if (settings.node.mining && historyView.history.isFullChainSynced) context.system.eventStream.publish(EnableMining) modsInToApply = modsInToApply.filterNot(_ == mod.encodedId) - if (modsInToApply.isEmpty) context.parent ! ModifierAppliedToHistory + logger.info(s"modsInToApply after cleaning is: ${modsInToApply.mkString(",")}.") + if (modsInToApply.isEmpty) { + context.parent ! ModifierAppliedToHistory + } case StateAction.ApplyFailed(mod, e) => + logger.info(s"Got StateAction.ApplyFailed. Mod: ${mod.encodedId}.") val (newHistory: History, progressInfo: ProgressInfo) = historyView.history.reportModifierIsInvalid(mod) historyView = historyView.copy(history = newHistory) context.parent ! SemanticallyFailedModification(mod, e) modsInToApply = progressInfo.toApply.map(_.encodedId).toList + logger.info(s"After rollback history new modsInToApply: ${modsInToApply.mkString(",")}.") context.parent ! ProgressInfoForState( progressInfo, (historyView.history.getBestHeaderHeight - historyView.history.getBestBlockHeight - 1) < settings.constants.MaxRollbackDepth * 2, From 5da589a7e16eed7e26b3e4f2505d75503925cd98 Mon Sep 17 00:00:00 2001 From: Timofey Gusev Date: Wed, 1 Apr 2020 21:19:23 +0300 Subject: [PATCH 176/177] more logging --- src/main/scala/encry/nvg/IntermediaryNVHView.scala | 2 ++ src/main/scala/encry/nvg/NVHHistory.scala | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index de68115655..ebf57bc444 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -160,6 +160,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, if msg.pi.chainSwitchingNeeded && msg.pi.branchPoint.exists( point => !stateReader.version.sameElements(point) ) => + logger.info(s"State should be dropped here! Ids: ${msg.pi.toApply.map(_.encodedId).mkString(",")}.") //todo drop state here context.become( viewReceive( @@ -169,6 +170,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, ) ) case msg: ProgressInfoForState => + logger.info(s"Got progress info from history with ids: ${msg.pi.toApply.map(_.encodedId).mkString(",")}.") toApply = msg.pi.toApply.map(mod => ByteArrayWrapper(mod.id)).toSet msg.pi.toApply.foreach { mod => logger.info(s"mid to state: ${mod.encodedId}.") diff --git a/src/main/scala/encry/nvg/NVHHistory.scala b/src/main/scala/encry/nvg/NVHHistory.scala index 39eb30a315..3877bc9d01 100644 --- a/src/main/scala/encry/nvg/NVHHistory.scala +++ b/src/main/scala/encry/nvg/NVHHistory.scala @@ -77,7 +77,8 @@ class NVHHistory(settings: EncryAppSettings, ntp: NetworkTimeProvider) case _: Payload => false }, success = true) if (progressInfo.toApply.nonEmpty) { - logger.info(s"Progress info contains an non empty toApply. Going to notify state about new toApply.") + logger.info(s"Progress info contains a non empty toApply. Going to notify state about new toApply. " + + s" Mods in toApply: ${progressInfo.toApply.map(_.encodedId).mkString(",")}.") modsInToApply = progressInfo.toApply.map(_.encodedId).toList context.parent ! ProgressInfoForState( progressInfo, From a645d5c08d399b05a6208fd99db4122cc3d8f627 Mon Sep 17 00:00:00 2001 From: aleksandr Date: Thu, 2 Apr 2020 14:57:59 +0300 Subject: [PATCH 177/177] add rollback pipeline --- .../scala/encry/nvg/IntermediaryNVHView.scala | 38 +++++----- src/main/scala/encry/nvg/NVHState.scala | 75 ++++++++++++++----- 2 files changed, 77 insertions(+), 36 deletions(-) diff --git a/src/main/scala/encry/nvg/IntermediaryNVHView.scala b/src/main/scala/encry/nvg/IntermediaryNVHView.scala index ebf57bc444..8e049dd3fa 100644 --- a/src/main/scala/encry/nvg/IntermediaryNVHView.scala +++ b/src/main/scala/encry/nvg/IntermediaryNVHView.scala @@ -1,23 +1,19 @@ package encry.nvg -import akka.actor.{ Actor, ActorRef, Props, Stash } +import akka.actor.{Actor, ActorRef, Props, Stash} import akka.pattern._ import com.typesafe.scalalogging.StrictLogging import encry.api.http.DataHolderForApi.BlockAndHeaderInfo import encry.local.miner.Miner.CandidateEnvelope import encry.network.Messages.MessageToNetwork.RequestFromLocal -import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{ RegisterNodeView, RegisterState } -import encry.nvg.IntermediaryNVHView.{ ModifierToAppend, NodeViewStarted } +import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.{RegisterNodeView, RegisterState} +import encry.nvg.IntermediaryNVHView.{ModifierToAppend, NodeViewStarted} import encry.nvg.ModifiersValidator.ValidatedModifier -import encry.nvg.NVHHistory.{ ModifierAppliedToHistory, NewWalletReader, ProgressInfoForState } +import encry.nvg.NVHHistory.{ModifierAppliedToHistory, NewWalletReader, ProgressInfoForState} import encry.nvg.NVHState.StateAction +import encry.nvg.NVHState.StateAction.{Restore, RollbackTo, StateStarted} import encry.nvg.NodeViewHolder.ReceivableMessages.LocallyGeneratedModifier -import encry.nvg.NodeViewHolder.{ - GetDataFromCurrentView, - SemanticallyFailedModification, - SemanticallySuccessfulModifier, - SyntacticallyFailedModification -} +import encry.nvg.NodeViewHolder.{GetDataFromCurrentView, SemanticallyFailedModification, SemanticallySuccessfulModifier, SyntacticallyFailedModification} import encry.settings.EncryAppSettings import encry.stats.StatsSender.StatsSenderMessage import encry.utils.NetworkTimeProvider @@ -27,10 +23,10 @@ import encry.view.state.UtxoStateReader import encry.view.wallet.WalletReader import io.iohk.iodb.ByteArrayWrapper import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{ Block, Header, Payload } +import org.encryfoundation.common.modifiers.history.{Block, Header, Payload} import org.encryfoundation.common.utils.Algos -import scala.concurrent.duration._ +import scala.concurrent.duration._ import scala.concurrent.Future class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, influx: Option[ActorRef]) @@ -77,6 +73,7 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, logger.info(s"NodeViewParent actor got init history. Going to init state actor.") context.become(awaitingViewActors(Some(sender()), state), discardOld = true) context.actorOf(NVHState.restoreProps(settings, reader, influx)) + case StateStarted => sender() ! Restore case RegisterNodeView(reader, wallet) => walletReader = wallet historyReader = reader @@ -162,13 +159,8 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, ) => logger.info(s"State should be dropped here! Ids: ${msg.pi.toApply.map(_.encodedId).mkString(",")}.") //todo drop state here - context.become( - viewReceive( - history, - state, - stateReader - ) - ) + context.stop(state) + context.become(stateRollbackAwaiting(msg, history, stateReader.safePointHeight)) case msg: ProgressInfoForState => logger.info(s"Got progress info from history with ids: ${msg.pi.toApply.map(_.encodedId).mkString(",")}.") toApply = msg.pi.toApply.map(mod => ByteArrayWrapper(mod.id)).toSet @@ -192,6 +184,14 @@ class IntermediaryNVHView(settings: EncryAppSettings, ntp: NetworkTimeProvider, case msg: HistoryReader => historyReader = msg; context.parent ! msg } + def stateRollbackAwaiting(pi: ProgressInfoForState, historyRef: ActorRef, safePoint: Int): Receive = { + case StateStarted => sender() ! RollbackTo(pi.pi.branchPoint.get, safePoint) + case RegisterState(reader) => + context.become(viewReceive(historyRef, sender(), reader)) + val newProgressInfo = pi.pi.copy(toRemove = Seq.empty) + self ! pi.copy(pi = newProgressInfo) + } + def awaitingHistoryBranchPoint(history: ActorRef): Receive = ??? def getNextModifier(): Unit = { diff --git a/src/main/scala/encry/nvg/NVHState.scala b/src/main/scala/encry/nvg/NVHState.scala index 93c59448d9..c9f4aa9ff7 100644 --- a/src/main/scala/encry/nvg/NVHState.scala +++ b/src/main/scala/encry/nvg/NVHState.scala @@ -6,7 +6,7 @@ import akka.actor.{Actor, ActorRef, Props} import cats.syntax.option.none import com.typesafe.scalalogging.StrictLogging import encry.nvg.IntermediaryNVHView.IntermediaryNVHViewActions.RegisterState -import encry.nvg.NVHState.StateAction.{ApplyFailed, ApplyModifier, CreateTreeChunks, ModifierApplied} +import encry.nvg.NVHState.StateAction.{ApplyFailed, ApplyModifier, CreateTreeChunks, ModifierApplied, Restore, Rollback, RollbackTo, StateStarted} import encry.nvg.fast.sync.SnapshotProcessor.SnapshotChunk import encry.settings.EncryAppSettings import encry.stats.StatsSender.TransactionsInBlock @@ -16,40 +16,50 @@ import encry.storage.levelDb.versionalLevelDB.{LevelDbFactory, VLDBWrapper, Vers import encry.storage.{RootNodesStorage, VersionalStorage} import encry.utils.CoreTaggedTypes.VersionTag import encry.view.NodeViewErrors.ModifierApplyError -import encry.view.history.{History, HistoryReader} -import encry.view.state.{UtxoState, UtxoStateReader} +import encry.view.history.HistoryReader +import encry.view.state.UtxoState.logger import encry.view.state.avlTree.AvlTree import encry.view.state.avlTree.utils.implicits.Instances._ +import encry.view.state.{UtxoState, UtxoStateReader} import io.iohk.iodb.LSMStore import org.apache.commons.io.FileUtils import org.encryfoundation.common.modifiers.PersistentModifier -import org.encryfoundation.common.modifiers.history.{Block, Header} +import org.encryfoundation.common.modifiers.history.Block import org.encryfoundation.common.utils.Algos import org.encryfoundation.common.utils.TaggedTypes.{ADDigest, ModifierId} -import org.encryfoundation.common.utils.constants.Constants -import org.encryfoundation.prismlang.core.Constants import org.iq80.leveldb.Options import scala.util.{Failure, Success, Try} class NVHState(influxRef: Option[ActorRef], var historyReader: HistoryReader, settings: EncryAppSettings) - extends Actor with StrictLogging with AutoCloseable { + extends Actor with StrictLogging { - var state: UtxoState = restoreState(settings, historyReader, influxRef).getOrElse(genesis(settings, influxRef)) + override def preStart(): Unit = context.parent ! StateStarted - override def preStart(): Unit = context.parent ! RegisterState(UtxoStateReader(state)) + override def receive: Receive = awaitingInitialCommand + + def awaitingInitialCommand: Receive = { + case Restore => + val newState = restoreState(settings, historyReader, influxRef).getOrElse(genesis(settings, influxRef)) + context.parent ! RegisterState(UtxoStateReader(newState)) + context.become(modifierApplying(newState)) + case RollbackTo(branchPoint, safePointHeight) => + val newState = rollback(branchPoint, safePointHeight) + context.parent ! RegisterState(UtxoStateReader(newState)) + context.become(modifierApplying(newState)) + } - override def receive: Receive = { + def modifierApplying(state: UtxoState): Receive = { case ApplyModifier(modifier: PersistentModifier, - saveRootNodesFlag: Boolean, - isFullChainSynced: Boolean) => + saveRootNodesFlag: Boolean, + isFullChainSynced: Boolean) => state.applyModifier(modifier, saveRootNodesFlag) match { case Right(stateAfterApply) => modifier match { case b: Block if isFullChainSynced => context.parent ! TransactionsInBlock(b.payload.txs.size) case _ => } - state = stateAfterApply + context.become(modifierApplying(stateAfterApply)) logger.info(s"Successfully apply modifier: ${Algos.encode(modifier.id)} of type ${modifier.modifierTypeId}") context.parent ! UtxoStateReader(state) context.parent ! ModifierApplied(modifier) @@ -67,7 +77,6 @@ class NVHState(influxRef: Option[ActorRef], var historyReader: HistoryReader, se override def postStop(): Unit = { logger.info("Close state!") - state.close() } def genesis(settings: EncryAppSettings, influxRef: Option[ActorRef]): UtxoState = { @@ -152,15 +161,45 @@ class NVHState(influxRef: Option[ActorRef], var historyReader: HistoryReader, se UtxoState.create(stateDir, rootsDir, settings, influxRef) } - override def close(): Unit = state.close() + def rollback(branchPoint: ModifierId, safePointHeight: Int): UtxoState = { + val dir: File = UtxoState.getStateDir(settings) + dir.mkdirs() + dir.listFiles.foreach(_.delete()) + val stateDir: File = UtxoState.getStateDir(settings) + stateDir.mkdirs() + val rootsDir: File = UtxoState.getRootsDir(settings) + rootsDir.mkdir() + val branchPointHeight = historyReader.getHeaderById(ModifierId !@@ branchPoint).get.height + val additionalBlocks = (safePointHeight + 1 to branchPointHeight).foldLeft(List.empty[Block]){ + case (blocks, height) => + val headerAtHeight = historyReader.getBestHeaderAtHeight(height).get + val blockAtHeight = historyReader.getBlockByHeader(headerAtHeight).get + blocks :+ blockAtHeight + } + val storage = settings.storage.state match { + case VersionalStorage.IODB => + logger.info("Init state with iodb storage") + IODBWrapper(new LSMStore(stateDir, keepVersions = settings.constants.DefaultKeepVersions, keySize = 33)) + case VersionalStorage.LevelDB => + logger.info("Init state with levelDB storage") + val levelDBInit = LevelDbFactory.factory.open(stateDir, new Options) + VLDBWrapper(VersionalLevelDBCompanion(levelDBInit, settings.levelDB.copy(keySize = 33), keySize = 33)) + } + val rootStorage = { + val levelDBInit = LevelDbFactory.factory.open(rootsDir, new Options) + RootNodesStorage[StorageKey, StorageValue](levelDBInit, settings.constants.MaxRollbackDepth, rootsDir) + } + UtxoState + .rollbackTo(VersionTag !@@ branchPoint, additionalBlocks, storage, rootStorage, settings.constants, influxRef) + .get + } } object NVHState extends StrictLogging { - import encry.view.state.avlTree.utils.implicits.Instances._ - sealed trait StateAction object StateAction { + case object StateStarted extends StateAction case class ModifierApplied(modifierId: PersistentModifier) extends StateAction case class Rollback(branchPoint: ModifierId) extends StateAction case class ApplyFailed(modifierId: PersistentModifier, errs: List[ModifierApplyError]) extends StateAction @@ -168,6 +207,8 @@ object NVHState extends StrictLogging { saveRootNodesFlag: Boolean, isFullChainSynced: Boolean) extends StateAction case object CreateTreeChunks extends StateAction + case object Restore extends StateAction + case class RollbackTo(branchPoint: ModifierId, safePointHeight: Int) extends StateAction case class TreeChunks(chunks: List[SnapshotChunk]) extends StateAction }