From ade5a6fe29975ace78706cdc5cbd1f915727452d Mon Sep 17 00:00:00 2001 From: Yiftach Kaplan Date: Tue, 23 Apr 2024 09:08:17 +0100 Subject: [PATCH 1/5] CORE-20347: Integrate new delivery tracker --- .../net/corda/p2p/linkmanager/LinkManager.kt | 10 +- .../linkmanager/common/CommonComponents.kt | 2 +- .../linkmanager/delivery/DeliveryTracker.kt | 29 +- .../linkmanager/delivery/ReplayScheduler.kt | 18 +- .../inbound/InboundMessageProcessor.kt | 22 +- .../outbound/OutboundLinkManager.kt | 37 +- .../p2p/linkmanager/tracker/AckProcessor.kt | 35 + .../linkmanager/tracker/DataMessageCache.kt | 227 ++--- .../tracker/DeliveryTrackerConfiguration.kt | 4 - .../tracker/DeliveryTrackerOffsetProvider.kt | 19 + .../tracker/DeliveryTrackerProcessor.kt | 16 +- .../p2p/linkmanager/tracker/MessageRecord.kt | 45 + .../linkmanager/tracker/MessageReplayer.kt | 38 + .../linkmanager/tracker/MessagesHandler.kt | 37 + .../p2p/linkmanager/tracker/PartitionState.kt | 115 ++- .../linkmanager/tracker/PartitionsStates.kt | 86 +- .../tracker/StatefulDeliveryTracker.kt | 91 +- .../tracker/TrackedMessageState.kt | 2 - .../exception/DeliveryTrackerExceptions.kt | 2 +- .../delivery/DeliveryTrackerTest.kt | 40 +- .../delivery/ReplaySchedulerTest.kt | 82 +- .../linkmanager/tracker/AckProcessorTest.kt | 137 +++ .../tracker/DataMessageCacheTest.kt | 476 ++++++++-- .../DeliveryTrackerConfigurationTest.kt | 10 - .../DeliveryTrackerOffsetProviderTest.kt | 37 + ...yTrackerPartitionAssignmentListenerTest.kt | 49 + .../tracker/DeliveryTrackerProcessorTest.kt | 85 +- .../tracker/MessageReplayerTest.kt | 68 ++ .../tracker/MessagesHandlerTest.kt | 124 +++ .../linkmanager/tracker/PartitionStateTest.kt | 740 +++++++-------- .../tracker/PartitionsStatesTest.kt | 543 ----------- .../tracker/PartitionsStatesTests.kt | 866 ++++++++++++++++++ gradle.properties | 2 +- .../schema/p2p/LinkManagerConfiguration.kt | 1 - 34 files changed, 2673 insertions(+), 1422 deletions(-) create mode 100644 components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/AckProcessor.kt create mode 100644 components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerOffsetProvider.kt create mode 100644 components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/MessageRecord.kt create mode 100644 components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/MessageReplayer.kt create mode 100644 components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/MessagesHandler.kt create mode 100644 components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/AckProcessorTest.kt create mode 100644 components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerOffsetProviderTest.kt create mode 100644 components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerPartitionAssignmentListenerTest.kt create mode 100644 components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/MessageReplayerTest.kt create mode 100644 components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/MessagesHandlerTest.kt delete mode 100644 components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStatesTest.kt create mode 100644 components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStatesTests.kt diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/LinkManager.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/LinkManager.kt index 1555289e9bf..537e87dc403 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/LinkManager.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/LinkManager.kt @@ -85,17 +85,9 @@ class LinkManager( commonComponents, ) private val outboundLinkManager = OutboundLinkManager( - lifecycleCoordinatorFactory = lifecycleCoordinatorFactory, commonComponents = commonComponents, - sessionComponents = sessionManagerCommonComponents, - linkManagerHostingMap = linkManagerHostingMap, - groupPolicyProvider = groupPolicyProvider, - membershipGroupReaderProvider = membershipGroupReaderProvider, - configurationReaderService = configurationReaderService, - subscriptionFactory = subscriptionFactory, - publisherFactory = publisherFactory, messagingConfiguration = messagingConfiguration, - clock = clock, + sessionComponents = sessionManagerCommonComponents, ) private val inboundLinkManager = InboundLinkManager( lifecycleCoordinatorFactory = lifecycleCoordinatorFactory, diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/common/CommonComponents.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/common/CommonComponents.kt index d7d3997bc94..3ea9512819e 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/common/CommonComponents.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/common/CommonComponents.kt @@ -40,8 +40,8 @@ internal class CommonComponents( membershipQueryClient: MembershipQueryClient, groupParametersReaderService: GroupParametersReaderService, internal val clock: Clock, - internal val schemaRegistry: AvroSchemaRegistry, internal val stateManager: StateManager, + internal val schemaRegistry: AvroSchemaRegistry, sessionEncryptionOpsClient: SessionEncryptionOpsClient, ) : LifecycleWithDominoTile { private companion object { diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTracker.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTracker.kt index 640011ba072..c51393bc155 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTracker.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTracker.kt @@ -1,6 +1,5 @@ package net.corda.p2p.linkmanager.delivery -import net.corda.configuration.read.ConfigurationReadService import net.corda.crypto.client.CryptoOpsClient import net.corda.data.p2p.AuthenticatedMessageAndKey import net.corda.data.p2p.AuthenticatedMessageDeliveryState @@ -25,49 +24,41 @@ import net.corda.messaging.api.publisher.config.PublisherConfig import net.corda.messaging.api.publisher.factory.PublisherFactory import net.corda.messaging.api.records.Record import net.corda.messaging.api.subscription.config.SubscriptionConfig -import net.corda.messaging.api.subscription.factory.SubscriptionFactory import net.corda.messaging.api.subscription.listener.StateAndEventListener import net.corda.metrics.CordaMetrics +import net.corda.p2p.linkmanager.common.CommonComponents import net.corda.p2p.linkmanager.sessions.SessionManager import net.corda.schema.Schemas.P2P.P2P_OUT_MARKERS import net.corda.utilities.debug -import net.corda.utilities.time.Clock import net.corda.virtualnode.toCorda import org.slf4j.LoggerFactory import java.time.Duration import java.time.Instant import java.util.concurrent.ConcurrentHashMap -@Suppress("LongParameterList") internal class DeliveryTracker( - coordinatorFactory: LifecycleCoordinatorFactory, - configReadService: ConfigurationReadService, - publisherFactory: PublisherFactory, + commonComponents: CommonComponents, messagingConfiguration: SmartConfig, - subscriptionFactory: SubscriptionFactory, sessionManager: SessionManager, - clock: Clock, processAuthenticatedMessage: (message: AuthenticatedMessageAndKey) -> List>, - ): LifecycleWithDominoTile { +): LifecycleWithDominoTile { private val appMessageReplayer = AppMessageReplayer( - coordinatorFactory, - publisherFactory, + commonComponents.lifecycleCoordinatorFactory, + commonComponents.publisherFactory, messagingConfiguration, processAuthenticatedMessage ) private val replayScheduler = ReplayScheduler( - coordinatorFactory, - configReadService, + commonComponents, true, appMessageReplayer::replayMessage, - clock = clock ) private val messageTracker = MessageTracker(replayScheduler) private val subscriptionConfig = SubscriptionConfig("message-tracker-group", P2P_OUT_MARKERS) private val messageTrackerSubscription = { - subscriptionFactory.createStateAndEventSubscription( + commonComponents.subscriptionFactory.createStateAndEventSubscription( subscriptionConfig, messageTracker.processor, messagingConfiguration, @@ -75,7 +66,7 @@ internal class DeliveryTracker( ) } private val messageTrackerSubscriptionTile = StateAndEventSubscriptionDominoTile( - coordinatorFactory, + commonComponents.lifecycleCoordinatorFactory, messageTrackerSubscription, subscriptionConfig, setOf( @@ -92,7 +83,9 @@ internal class DeliveryTracker( ) ) - override val dominoTile = ComplexDominoTile(this::class.java.simpleName, coordinatorFactory, + override val dominoTile = ComplexDominoTile( + this::class.java.simpleName, + commonComponents.lifecycleCoordinatorFactory, dependentChildren = setOf(messageTrackerSubscriptionTile.coordinatorName), managedChildren = setOf(messageTrackerSubscriptionTile.toNamedLifecycle()) ) diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/ReplayScheduler.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/ReplayScheduler.kt index ba61da27df4..a94a5438c8a 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/ReplayScheduler.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/delivery/ReplayScheduler.kt @@ -2,22 +2,20 @@ package net.corda.p2p.linkmanager.delivery import com.typesafe.config.Config import com.typesafe.config.ConfigException -import net.corda.configuration.read.ConfigurationReadService import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.BASE_REPLAY_PERIOD_KEY import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.MAX_REPLAYING_MESSAGES_PER_PEER import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.MESSAGE_REPLAY_PERIOD_KEY import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.REPLAY_ALGORITHM_KEY import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.REPLAY_PERIOD_CUTOFF_KEY -import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.lifecycle.domino.logic.ComplexDominoTile import net.corda.lifecycle.domino.logic.ConfigurationChangeHandler import net.corda.lifecycle.domino.logic.LifecycleWithDominoTile import net.corda.lifecycle.domino.logic.util.ResourcesHolder +import net.corda.p2p.linkmanager.common.CommonComponents import net.corda.p2p.linkmanager.sessions.SessionManager import net.corda.schema.configuration.ConfigKeys import net.corda.utilities.VisibleForTesting -import net.corda.utilities.time.Clock import org.slf4j.LoggerFactory import java.time.Duration import java.util.concurrent.CompletableFuture @@ -31,19 +29,16 @@ import java.util.concurrent.atomic.AtomicReference /** * This class keeps track of messages which may need to be replayed. */ -@Suppress("LongParameterList") internal class ReplayScheduler( - coordinatorFactory: LifecycleCoordinatorFactory, - private val configReadService: ConfigurationReadService, + private val commonComponents: CommonComponents, private val limitTotalReplays: Boolean, private val replayMessage: (message: M, messageId: MessageId) -> Unit, executorServiceFactory: () -> ScheduledExecutorService = { Executors.newSingleThreadScheduledExecutor() }, - private val clock: Clock - ) : LifecycleWithDominoTile { +) : LifecycleWithDominoTile { override val dominoTile = ComplexDominoTile( this::class.java.simpleName, - coordinatorFactory, + commonComponents.lifecycleCoordinatorFactory, onClose = { executorService.shutdownNow() }, configurationChangeHandler = ReplaySchedulerConfigurationChangeHandler() ) @@ -132,7 +127,8 @@ internal class ReplayScheduler( } } - inner class ReplaySchedulerConfigurationChangeHandler: ConfigurationChangeHandler(configReadService, + inner class ReplaySchedulerConfigurationChangeHandler: ConfigurationChangeHandler( + commonComponents.configurationReaderService, ConfigKeys.P2P_LINK_MANAGER_CONFIG, ::fromConfig) { override fun applyNewConfiguration( @@ -242,7 +238,7 @@ internal class ReplayScheduler( replayInfoPerMessageId.compute(messageId) { _, replayInfo -> replayInfo?.future?.cancel(false) val firstReplayPeriod = replayCalculator.get().calculateReplayInterval() - val delay = firstReplayPeriod.toMillis() + originalAttemptTimestamp - clock.instant().toEpochMilli() + val delay = firstReplayPeriod.toMillis() + originalAttemptTimestamp - commonComponents.clock.instant().toEpochMilli() val future = executorService.schedule({ replay(message, messageId) }, delay, TimeUnit.MILLISECONDS) ReplayInfo(firstReplayPeriod, future) } diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/inbound/InboundMessageProcessor.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/inbound/InboundMessageProcessor.kt index 58f9c3e4377..c7952ee44da 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/inbound/InboundMessageProcessor.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/inbound/InboundMessageProcessor.kt @@ -36,6 +36,7 @@ import net.corda.p2p.linkmanager.metrics.recordInboundSessionMessagesMetric import net.corda.p2p.linkmanager.metrics.recordOutboundSessionMessagesMetric import net.corda.p2p.linkmanager.sessions.StatefulSessionManagerImpl.Companion.LINK_MANAGER_SUBSYSTEM import net.corda.schema.Schemas +import net.corda.schema.Schemas.P2P.LINK_ACK_IN_TOPIC import net.corda.tracing.traceEventProcessing import net.corda.utilities.Either import net.corda.utilities.debug @@ -369,12 +370,21 @@ internal class InboundMessageProcessor( } } - private fun makeMarkerForAckMessage(message: AuthenticatedMessageAck): Record { - return Record( - Schemas.P2P.P2P_OUT_MARKERS, - message.messageId, - AppMessageMarker(LinkManagerReceivedMarker(), clock.instant().toEpochMilli()) - ) + private fun makeMarkerForAckMessage(message: AuthenticatedMessageAck): Record<*, *> { + return if (features.enableP2PStatefulDeliveryTracker) { + Record( + LINK_ACK_IN_TOPIC, + message.messageId, + null + ) + } else { + Record( + Schemas.P2P.P2P_OUT_MARKERS, + message.messageId, + AppMessageMarker(LinkManagerReceivedMarker(), clock.instant().toEpochMilli()) + ) + + } } private fun isCommunicationAllowed( diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/outbound/OutboundLinkManager.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/outbound/OutboundLinkManager.kt index 1e76b51ddf7..57351fda13c 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/outbound/OutboundLinkManager.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/outbound/OutboundLinkManager.kt @@ -1,40 +1,23 @@ package net.corda.p2p.linkmanager.outbound -import net.corda.configuration.read.ConfigurationReadService import net.corda.libs.configuration.SmartConfig -import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.lifecycle.domino.logic.ComplexDominoTile import net.corda.lifecycle.domino.logic.LifecycleWithDominoTile import net.corda.lifecycle.domino.logic.util.PublisherWithDominoLogic import net.corda.lifecycle.domino.logic.util.SubscriptionDominoTile -import net.corda.membership.grouppolicy.GroupPolicyProvider -import net.corda.membership.read.MembershipGroupReaderProvider import net.corda.messaging.api.publisher.config.PublisherConfig -import net.corda.messaging.api.publisher.factory.PublisherFactory import net.corda.messaging.api.subscription.config.SubscriptionConfig -import net.corda.messaging.api.subscription.factory.SubscriptionFactory import net.corda.p2p.linkmanager.common.CommonComponents import net.corda.p2p.linkmanager.delivery.DeliveryTracker -import net.corda.p2p.linkmanager.hosting.LinkManagerHostingMap import net.corda.p2p.linkmanager.sessions.SessionManagerCommonComponents import net.corda.p2p.linkmanager.tracker.StatefulDeliveryTracker import net.corda.schema.Schemas import net.corda.utilities.flags.Features -import net.corda.utilities.time.Clock -@Suppress("LongParameterList") internal class OutboundLinkManager( - lifecycleCoordinatorFactory: LifecycleCoordinatorFactory, commonComponents: CommonComponents, sessionComponents: SessionManagerCommonComponents, - linkManagerHostingMap: LinkManagerHostingMap, - groupPolicyProvider: GroupPolicyProvider, - membershipGroupReaderProvider: MembershipGroupReaderProvider, - configurationReaderService: ConfigurationReadService, - subscriptionFactory: SubscriptionFactory, - publisherFactory: PublisherFactory, messagingConfiguration: SmartConfig, - clock: Clock, features: Features = Features() ) : LifecycleWithDominoTile { companion object { @@ -42,27 +25,23 @@ internal class OutboundLinkManager( } private val outboundMessageProcessor = OutboundMessageProcessor( sessionComponents.sessionManager, - linkManagerHostingMap, - groupPolicyProvider, - membershipGroupReaderProvider, + commonComponents.linkManagerHostingMap, + commonComponents.groupPolicyProvider, + commonComponents.membershipGroupReaderProvider, commonComponents.messagesPendingSession, - clock, + commonComponents.clock, commonComponents.messageConverter, ) private val deliveryTracker = DeliveryTracker( - lifecycleCoordinatorFactory, - configurationReaderService, - publisherFactory, + commonComponents, messagingConfiguration, - subscriptionFactory, sessionComponents.sessionManager, - clock = clock ) { outboundMessageProcessor.processReplayedAuthenticatedMessage(it) } private val subscriptionConfig = SubscriptionConfig(OUTBOUND_MESSAGE_PROCESSOR_GROUP, Schemas.P2P.P2P_OUT_TOPIC) private val outboundMessageSubscription = { - subscriptionFactory.createEventLogSubscription( + commonComponents.subscriptionFactory.createEventLogSubscription( subscriptionConfig, outboundMessageProcessor, messagingConfiguration, @@ -88,7 +67,7 @@ internal class OutboundLinkManager( ) ComplexDominoTile( OUTBOUND_MESSAGE_PROCESSOR_GROUP, - coordinatorFactory = lifecycleCoordinatorFactory, + coordinatorFactory = commonComponents.lifecycleCoordinatorFactory, dependentChildren = listOf( statefulDeliveryTracker.dominoTile.coordinatorName, publisher.dominoTile.coordinatorName, @@ -100,7 +79,7 @@ internal class OutboundLinkManager( ) } else { SubscriptionDominoTile( - lifecycleCoordinatorFactory, + commonComponents.lifecycleCoordinatorFactory, outboundMessageSubscription, subscriptionConfig, dependentChildren = listOf( diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/AckProcessor.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/AckProcessor.kt new file mode 100644 index 00000000000..06ac8d52830 --- /dev/null +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/AckProcessor.kt @@ -0,0 +1,35 @@ +package net.corda.p2p.linkmanager.tracker + +import net.corda.messaging.api.processor.PubSubProcessor +import net.corda.messaging.api.records.Record +import net.corda.p2p.linkmanager.delivery.ReplayScheduler +import net.corda.p2p.linkmanager.sessions.SessionManager +import net.corda.virtualnode.toCorda +import java.util.concurrent.CompletableFuture +import java.util.concurrent.Future + +internal class AckProcessor( + private val partitionsStates: PartitionsStates, + private val cache: DataMessageCache, + private val replayScheduler: ReplayScheduler, +) : PubSubProcessor { + + override fun onNext(event: Record): Future { + val future = CompletableFuture.completedFuture(Unit) + val messageId = event.key + val messageRecord = cache.remove(messageId) ?: return future + partitionsStates.forget(messageRecord) + val counterparties = SessionManager.Counterparties( + ourId = messageRecord.message.header.source.toCorda(), + counterpartyId = messageRecord.message.header.destination.toCorda(), + ) + replayScheduler.removeFromReplay( + messageId, + counterparties, + ) + return CompletableFuture.completedFuture(Unit) + } + + override val keyClass = String::class.java + override val valueClass = String::class.java +} diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DataMessageCache.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DataMessageCache.kt index 6f972f99b39..c5af38acaa4 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DataMessageCache.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DataMessageCache.kt @@ -1,114 +1,88 @@ package net.corda.p2p.linkmanager.tracker -import com.github.benmanes.caffeine.cache.Cache -import com.github.benmanes.caffeine.cache.Caffeine -import net.corda.cache.caffeine.CacheFactoryImpl -import net.corda.data.p2p.app.AppMessage -import net.corda.libs.statemanager.api.State +import net.corda.data.p2p.app.AuthenticatedMessage import net.corda.libs.statemanager.api.StateManager -import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.lifecycle.domino.logic.ComplexDominoTile import net.corda.lifecycle.domino.logic.LifecycleWithDominoTile -import net.corda.messaging.api.records.EventLogRecord +import net.corda.p2p.linkmanager.common.CommonComponents import net.corda.p2p.linkmanager.tracker.exception.DataMessageCacheException -import net.corda.schema.registry.AvroSchemaRegistry -import net.corda.schema.registry.deserialize import org.slf4j.Logger import org.slf4j.LoggerFactory -import java.nio.ByteBuffer import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicReference +import java.util.concurrent.ConcurrentSkipListMap +import java.util.concurrent.atomic.AtomicLong internal class DataMessageCache( - coordinatorFactory: LifecycleCoordinatorFactory, - private val stateManager: StateManager, - private val schemaRegistry: AvroSchemaRegistry, + private val commonComponents: CommonComponents, private val config: DeliveryTrackerConfiguration, -) : LifecycleWithDominoTile, DeliveryTrackerConfiguration.ConfigurationChanged { + private val onOffsetsToReadFromChanged: (partitionsToLastPersistedOffset: Collection>) -> Unit, +) : LifecycleWithDominoTile { private companion object { const val NAME = "DataMessageCache" const val MAX_RETRIES = 3 val logger: Logger = LoggerFactory.getLogger(this::class.java.enclosingClass) } - fun get(key: String): AppMessage? { - failedCreates.getAndSet(ConcurrentHashMap())?.let { failed -> - if (failed.isNotEmpty()) { - messageCache.putAll(failed) - } - } - return messageCache.getIfPresent(key)?.message ?: stateManager.getIfPresent(key) + fun get(key: String): AuthenticatedMessage? { + return messageCache[key]?.message ?: commonComponents.stateManager.getIfPresent(key)?.message } - fun put(record: EventLogRecord) { - val message = record.value ?: return - messageCache.put(record.key, TrackedMessage(message, record.partition, record.offset)) - partitionOffsetTracker.compute(record.partition) { _, info -> - if (info == null) { - val offsetToMessageId = ConcurrentHashMap() - offsetToMessageId[record.offset] = record.key - PartitionInfo(record.offset, offsetToMessageId) - } else { - info.offsetToMessageId[record.offset] = record.key - if (record.offset > info.latestOffset) { - PartitionInfo(record.offset, info.offsetToMessageId) - } else { - info - } + fun put(messages: Collection) { + val toCache = messages.associateBy { message -> + message.message.header.messageId + } + if (toCache.isEmpty()) { + return + } + messageCache.putAll(toCache) + messages.groupBy { + it.partition + }.forEach { (partition, recordsInPartition) -> + partitionOffsetTracker.compute(partition) { _, info -> + val newInfo = info ?: PartitionInfo() + newInfo.offsetToMessageId.putAll( + recordsInPartition.associate { + it.offset to it.message.header.messageId + }, + ) + newInfo } } persistOlderCacheEntries() } - fun invalidate(key: String) { - when(messageCache.getIfPresent(key)) { - null -> { - logger.trace("Deleting delivery tracker entry from state manager for '{}'.", key) - stateManager.deleteIfPresent(key, MAX_RETRIES).apply { - if (!this) { - logger.warn("Failed to delete tracking state for '{}' after '{}' attempts.", key, MAX_RETRIES) - } + fun remove(key: String): MessageRecord? { + val record = messageCache.remove(key) + return if (record == null) { + logger.trace("Deleting delivery tracker entry from state manager for '{}'.", key) + commonComponents.stateManager.deleteIfPresent(key) + } else { + logger.trace("Discarding cached delivery tracker entry for '{}'.", key) + val partitionInfo = partitionOffsetTracker[record.partition] + if (partitionInfo != null) { + partitionInfo.offsetToMessageId.remove(record.offset) + + partitionInfo.readMessagesFromOffset.updateAndGet { + partitionInfo.offsetToMessageId.firstEntry()?.key ?: (record.offset + 1) } + onOffsetsToReadFromChanged(listOf(record.partition to partitionInfo.readMessagesFromOffset.get())) } - else -> { - logger.trace("Discarding cached delivery tracker entry for '{}'.", key) - messageCache.invalidate(key) - } + record } } - override fun changed() { - updateCacheSize(config.config.maxCacheSizeMegabytes) - } - override val dominoTile = ComplexDominoTile( componentName = NAME, - coordinatorFactory = coordinatorFactory, - dependentChildren = listOf(stateManager.name, config.dominoTile.coordinatorName), - onStart = ::onStart, - ) - - private fun onStart() { - config.lister(this) - } - - private data class TrackedMessage( - val message: AppMessage, - val partition: Int, - val offset: Long, + coordinatorFactory = commonComponents.lifecycleCoordinatorFactory, + dependentChildren = listOf(commonComponents.stateManager.name, config.dominoTile.coordinatorName), ) - /** - * Spill-over map for temporarily storing evicted cache entries that fail to be persisted. - */ - private val failedCreates = AtomicReference(ConcurrentHashMap()) - /** * Data class for storing the latest offset and an offset to message ID map for a partition. */ private data class PartitionInfo( - val latestOffset: Long, - val offsetToMessageId: ConcurrentHashMap + val offsetToMessageId: ConcurrentSkipListMap = ConcurrentSkipListMap(), + val readMessagesFromOffset: AtomicLong = AtomicLong(0), ) /** @@ -116,46 +90,15 @@ internal class DataMessageCache( */ private val partitionOffsetTracker = ConcurrentHashMap() - private val messageCache: Cache = - CacheFactoryImpl().build( - "P2P-delivery-tracker-cache", - Caffeine.newBuilder() - .maximumSize(config.config.maxCacheSizeMegabytes) - .removalListener { _, value, _ -> - value?.let { onRemoval(it) } - }.evictionListener { key, value, _ -> - if (key != null && value != null) { - onEviction(key, value) - } - } - ) - - private fun updateCacheSize(size: Long) { - messageCache.policy().eviction().ifPresent { - it.maximum = size - } - } - - private fun onEviction(messageId: String, trackedMessage: TrackedMessage) { - logger.trace("Handling cache eviction for message ID: '{}'", messageId) - val failed = stateManager.put(mapOf(messageId to trackedMessage.message), MAX_RETRIES) - if (failed.isNotEmpty()) { - logger.warn("Failed to persist tracking state for '{}' after '{}' attempts.", failed, MAX_RETRIES) - failedCreates.get()[messageId] = trackedMessage - } - partitionOffsetTracker[trackedMessage.partition]?.offsetToMessageId?.remove(trackedMessage.offset) - } - - private fun onRemoval(message: TrackedMessage) { - val offsetToMessageId = partitionOffsetTracker[message.partition]?.offsetToMessageId ?: return - logger.trace("Handling cache entry removal for message ID: '{}'", offsetToMessageId[message.offset]) - offsetToMessageId.remove(message.offset) - } + private val messageCache = ConcurrentHashMap() - private fun StateManager.getIfPresent(key: String): AppMessage? { + private fun StateManager.getIfPresent(key: String): MessageRecord? { return try { get(setOf(key)).values.firstOrNull()?.let { - schemaRegistry.deserialize(ByteBuffer.wrap(it.value)) + MessageRecord.fromState( + it, + commonComponents.schemaRegistry, + ) } } catch (e: Exception) { logger.warn("Unexpected error while trying to fetch message state for '{}'.", key, e) @@ -163,11 +106,12 @@ internal class DataMessageCache( } } - private fun StateManager.deleteIfPresent(key: String, remainingAttempts: Int): Boolean { + private fun StateManager.deleteIfPresent(key: String, remainingAttempts: Int = MAX_RETRIES): MessageRecord? { if (remainingAttempts <= 0) { - return false + logger.warn("Failed to delete tracking state for '{}' after '{}' attempts.", key, MAX_RETRIES) + return null } - val stateToDelete = get(setOf(key)).values.firstOrNull() ?: return true + val stateToDelete = get(setOf(key)).values.firstOrNull() ?: return null val failedDeletes = try { delete(setOf(stateToDelete)) } catch (e: Exception) { @@ -177,54 +121,63 @@ internal class DataMessageCache( return if (failedDeletes.isNotEmpty()) { deleteIfPresent(key, remainingAttempts - 1) } else { - true + MessageRecord.fromState(stateToDelete, commonComponents.schemaRegistry) } } - private fun StateManager.put(messages: Map, remainingAttempts: Int): Set { - if (remainingAttempts <= 0) { - return messages.keys - } + private fun StateManager.put( + messages: Map, + ) { val newStates = messages.mapValues { - State(it.key, schemaRegistry.serialize(it.value).array()) + it.value.toState(commonComponents.schemaRegistry) } val failedCreates = try { create(newStates.values) } catch (e: Exception) { val errorMessage = "Unexpected error while trying to persist message state to the state manager." logger.warn(errorMessage, e) - dominoTile.setError(DataMessageCacheException("$errorMessage. Cause: ${e.message}")) - emptySet() + dominoTile.setError(DataMessageCacheException("$errorMessage. Cause: ${e.message}", e)) + emptyList() } - return if (failedCreates.isNotEmpty()) { - put(messages.filterKeys { failedCreates.contains(it) }, remainingAttempts - 1) - } else { - emptySet() + + if (failedCreates.isNotEmpty()) { + logger.info("Failed to persist messages with IDs: $failedCreates") } } private fun persistOlderCacheEntries() { - val thresholds = partitionOffsetTracker.mapValues { - it.value.latestOffset - config.config.maxCacheOffsetAge - } - val messagesToPersist = partitionOffsetTracker.flatMap { (partition, info) -> - info.offsetToMessageId.filterKeys { it < (thresholds[partition] ?: 0) } - .mapNotNull { (_, messageId) -> - messageCache.getIfPresent(messageId)?.message?.let { + val messagesToPersist = partitionOffsetTracker.values.flatMap { info -> + val lastKey = info.offsetToMessageId.lastEntry()?.key + if (lastKey != null) { + val threshold = lastKey - config.config.maxCacheOffsetAge + val ids = info.offsetToMessageId.headMap(threshold) + ids.values.mapNotNull { messageId -> + messageCache.remove(messageId)?.let { messageId to it } + }.also { + ids.clear() + val oldestEntry = info.offsetToMessageId.firstEntry()?.key + if (oldestEntry != null) { + info.readMessagesFromOffset.set(oldestEntry) + } } + } else { + emptyList() + } }.toMap() if (messagesToPersist.isEmpty()) { return } logger.trace( "Messages with IDs: '{}' are older than the configured offset age and will be moved to the state manager.", - messagesToPersist.keys + messagesToPersist.keys, + ) + commonComponents.stateManager.put(messagesToPersist) + onOffsetsToReadFromChanged( + partitionOffsetTracker.map { + it.key to it.value.readMessagesFromOffset.get() + }, ) - stateManager.put(messagesToPersist, MAX_RETRIES).also { failed -> - logger.warn("Failed to persist tracking state for '{}' after '{}' attempts.", failed, MAX_RETRIES) - messageCache.invalidateAll((messagesToPersist - failed).keys) - } } } diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerConfiguration.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerConfiguration.kt index 6bff3c15bb6..54e0b36fc15 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerConfiguration.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerConfiguration.kt @@ -3,7 +3,6 @@ package net.corda.p2p.linkmanager.tracker import com.typesafe.config.Config import net.corda.configuration.read.ConfigurationReadService import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.DELIVERY_TRACKER_MAX_CACHE_OFFSET_AGE -import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.DELIVERY_TRACKER_MAX_CACHE_SIZE_MEGABYTES import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.DELIVERY_TRACKER_MAX_NUMBER_OF_PERSISTENCE_RETRIES import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.DELIVERY_TRACKER_OUTBOUND_BATCH_PROCESSING_TIMEOUT_SECONDS import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.DELIVERY_TRACKER_STATE_PERSISTENCE_PERIOD_SECONDS @@ -30,7 +29,6 @@ internal class DeliveryTrackerConfiguration( const val NAME = "DeliveryTrackerConfiguration" fun fromConfig(config: Config): Configuration { return Configuration( - maxCacheSizeMegabytes = config.getLong(DELIVERY_TRACKER_MAX_CACHE_SIZE_MEGABYTES), maxCacheOffsetAge = config.getLong(DELIVERY_TRACKER_MAX_CACHE_OFFSET_AGE), statePersistencePeriodSeconds = config.getDouble(DELIVERY_TRACKER_STATE_PERSISTENCE_PERIOD_SECONDS), outboundBatchProcessingTimeoutSeconds = config.getDouble(DELIVERY_TRACKER_OUTBOUND_BATCH_PROCESSING_TIMEOUT_SECONDS), @@ -39,7 +37,6 @@ internal class DeliveryTrackerConfiguration( } } data class Configuration( - val maxCacheSizeMegabytes: Long, val maxCacheOffsetAge: Long, val maxNumberOfPersistenceRetries: Int, val statePersistencePeriodSeconds: Double, @@ -51,7 +48,6 @@ internal class DeliveryTrackerConfiguration( private val configuration = AtomicReference( Configuration( - maxCacheSizeMegabytes = 100, maxCacheOffsetAge = 50000, statePersistencePeriodSeconds = 1.0, outboundBatchProcessingTimeoutSeconds = 30.0, diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerOffsetProvider.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerOffsetProvider.kt new file mode 100644 index 00000000000..ffbde5849c6 --- /dev/null +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerOffsetProvider.kt @@ -0,0 +1,19 @@ +package net.corda.p2p.linkmanager.tracker + +import net.corda.messaging.api.subscription.listener.ConsumerOffsetProvider + +internal class DeliveryTrackerOffsetProvider( + private val partitionStates: PartitionsStates, +) : ConsumerOffsetProvider { + override fun getStartingOffsets( + topicPartitions: Set>, + ): Map, Long> { + val topic = topicPartitions.map { it.first }.firstOrNull() ?: return emptyMap() + val partitionsIndices = topicPartitions.map { it.second }.toSet() + return partitionStates.loadPartitions(partitionsIndices).mapValues { (_, state) -> + state.readRecordsFromOffset + }.mapKeys { (partition, _) -> + topic to partition + } + } +} diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerProcessor.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerProcessor.kt index 64a00dc609f..d91b35c974e 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerProcessor.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerProcessor.kt @@ -2,24 +2,22 @@ package net.corda.p2p.linkmanager.tracker import net.corda.data.p2p.app.AppMessage import net.corda.lifecycle.domino.logic.util.PublisherWithDominoLogic -import net.corda.messaging.api.processor.EventLogProcessor +import net.corda.messaging.api.processor.EventSourceProcessor import net.corda.messaging.api.records.EventLogRecord -import net.corda.messaging.api.records.Record import net.corda.p2p.linkmanager.outbound.OutboundMessageProcessor internal class DeliveryTrackerProcessor( private val outboundMessageProcessor: OutboundMessageProcessor, - private val partitionsStates: PartitionsStates, + private val handler: MessagesHandler, private val publisher: PublisherWithDominoLogic, -) : EventLogProcessor { - override fun onNext(events: List>): List> { - partitionsStates.read(events) - val records = outboundMessageProcessor.onNext(events) +) : EventSourceProcessor { + override fun onNext(events: List>) { + val eventsToForward = handler.handleMessagesAndFilterRecords(events) + val records = outboundMessageProcessor.onNext(eventsToForward) publisher.publish(records).forEach { it.join() } - partitionsStates.sent(events) - return emptyList() + handler.handled(events) } override val keyClass = String::class.java diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/MessageRecord.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/MessageRecord.kt new file mode 100644 index 00000000000..18aa6bc258f --- /dev/null +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/MessageRecord.kt @@ -0,0 +1,45 @@ +package net.corda.p2p.linkmanager.tracker + +import net.corda.data.p2p.app.AuthenticatedMessage +import net.corda.libs.statemanager.api.Metadata +import net.corda.libs.statemanager.api.State +import net.corda.schema.registry.AvroSchemaRegistry +import net.corda.schema.registry.deserialize +import java.nio.ByteBuffer + +internal data class MessageRecord( + val message: AuthenticatedMessage, + val partition: Int, + val offset: Long, +) { + companion object { + private const val OFFSET = "OFFSET" + private const val PARTITION = "PARTITION" + fun fromState( + state: State, + schemaRegistry: AvroSchemaRegistry, + ): MessageRecord? { + val message = schemaRegistry.deserialize(ByteBuffer.wrap(state.value)) + val offset = (state.metadata.get(OFFSET) as? Number)?.toLong() ?: return null + val partition = (state.metadata.get(PARTITION) as? Number)?.toInt() ?: return null + return MessageRecord( + message, + offset = offset, + partition = partition, + ) + } + } + + fun toState( + schemaRegistry: AvroSchemaRegistry, + ) = State( + key = message.header.messageId, + value = schemaRegistry.serialize(message).array(), + metadata = Metadata( + mapOf( + OFFSET to offset, + PARTITION to partition, + ), + ), + ) +} diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/MessageReplayer.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/MessageReplayer.kt new file mode 100644 index 00000000000..1b5a059105f --- /dev/null +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/MessageReplayer.kt @@ -0,0 +1,38 @@ +package net.corda.p2p.linkmanager.tracker + +import net.corda.data.p2p.AuthenticatedMessageAndKey +import net.corda.lifecycle.domino.logic.util.PublisherWithDominoLogic +import net.corda.p2p.linkmanager.outbound.OutboundMessageProcessor +import org.slf4j.LoggerFactory + +internal class MessageReplayer( + private val publisher: PublisherWithDominoLogic, + private val outboundMessageProcessor: OutboundMessageProcessor, + private val cache: DataMessageCache, +) : (String, String) -> Unit { + private companion object { + val logger = LoggerFactory.getLogger(MessageReplayer::class.java) + } + override fun invoke( + messageId: String, + key: String, + ) { + logger.debug("Replaying message '$messageId' with key: '$key'") + val authenticatedMessage = cache.get(key) + if (authenticatedMessage == null) { + logger.warn("Cannot replay message '$messageId' with key: '$key', no such message") + return + } + + val messageAndKey = AuthenticatedMessageAndKey( + authenticatedMessage, + key, + ) + val records = outboundMessageProcessor.processReplayedAuthenticatedMessage(messageAndKey) + publisher.publish( + records, + ).forEach { + it.join() + } + } +} diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/MessagesHandler.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/MessagesHandler.kt new file mode 100644 index 00000000000..66e394e81e2 --- /dev/null +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/MessagesHandler.kt @@ -0,0 +1,37 @@ +package net.corda.p2p.linkmanager.tracker + +import net.corda.data.p2p.app.AppMessage +import net.corda.data.p2p.app.AuthenticatedMessage +import net.corda.messaging.api.records.EventLogRecord + +internal class MessagesHandler( + private val partitionsStates: PartitionsStates, + private val cache: DataMessageCache, +) { + fun handleMessagesAndFilterRecords( + events: List>, + ): List> { + val authenticatedMessages = events.mapNotNull { + val message = it.value?.message as? AuthenticatedMessage + if (message != null) { + MessageRecord( + message = message, + offset = it.offset, + partition = it.partition, + ) + } else { + null + } + } + cache.put(authenticatedMessages) + partitionsStates.read(authenticatedMessages) + + return partitionsStates.getEventToProcess(events) + } + + fun handled( + events: Collection>, + ) { + partitionsStates.handled(events) + } +} diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionState.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionState.kt index ba058e70f51..38bad8aade8 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionState.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionState.kt @@ -5,11 +5,13 @@ import com.fasterxml.jackson.core.JacksonException import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue -import net.corda.data.p2p.app.AppMessage +import net.corda.data.p2p.app.AuthenticatedMessage import net.corda.libs.statemanager.api.State import net.corda.libs.statemanager.api.StateOperationGroup -import net.corda.messaging.api.records.EventLogRecord +import net.corda.p2p.linkmanager.sessions.SessionManager import net.corda.v5.base.exceptions.CordaRuntimeException +import net.corda.v5.base.types.MemberX500Name +import net.corda.virtualnode.HoldingIdentity import java.time.Instant import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger @@ -41,30 +43,43 @@ internal class PartitionState( } @JsonProperty("messages") - private val trackedMessages = ConcurrentHashMap() + private val trackedMessages = + GroupToMessages() @JsonProperty("version") private val savedVersion = AtomicInteger(State.VERSION_INITIAL_VALUE) - private val _restartOffset = AtomicLong(0) - private val _lastSentOffset = AtomicLong(0) + private val _readRecordsFromOffset = AtomicLong(0) + private val _processRecordsFromOffset = AtomicLong(0) - var restartOffset: Long - get() { return _restartOffset.get() } - set(l) { _restartOffset.set(l) } - var lastSentOffset: Long - get() { return _lastSentOffset.get() } - set(l) { _lastSentOffset.set(l) } + var readRecordsFromOffset: Long + get() { return _readRecordsFromOffset.get() } + set(l) { _readRecordsFromOffset.set(l) } + var processRecordsFromOffset: Long + get() { return _processRecordsFromOffset.get() } + set(l) { _processRecordsFromOffset.set(l) } - fun addMessage(message: TrackedMessageState) { - trackedMessages[message.messageId] = message - } - - fun untrackMessage(messageId: String) { - trackedMessages.remove(messageId) + fun counterpartiesToMessages(): Collection>> { + return trackedMessages.flatMap { (groupId, ourNameToMessages) -> + ourNameToMessages.flatMap { (ourName, theirNameToMessages) -> + val ourId = HoldingIdentity( + MemberX500Name.parse(ourName), + groupId, + ) + theirNameToMessages.map { (theirName, messages) -> + val theirId = HoldingIdentity( + MemberX500Name.parse(theirName), + groupId, + ) + val counterparties = SessionManager.Counterparties( + ourId = ourId, + counterpartyId = theirId, + ) + counterparties to messages.values + } + } + } } - fun getTrackMessage(messageId: String): TrackedMessageState? = trackedMessages[messageId] - fun addToOperationGroup(group: StateOperationGroup) { val value = jsonParser.writeValueAsBytes(this) val version = savedVersion.get() @@ -83,36 +98,50 @@ internal class PartitionState( fun read( now: Instant, - records: List>, - ) { - val offset = records.onEach { record -> - val id = record.value?.id - if (id != null) { - trackedMessages[id] = TrackedMessageState( + records: Collection, + ): Collection { + return records.filter { record -> + val id = record.message.header.messageId + trackedMessages.getOrPut(record.message.header.source.groupId) { + OurNameToMessages() + }.getOrPut(record.message.header.source.x500Name) { + TheirNameToMessages() + }.getOrPut(record.message.header.destination.x500Name) { + MessageIdToMessage() + }.putIfAbsent( + id, + TrackedMessageState( messageId = id, timeStamp = now, - persisted = false, - ) - } - }.maxOfOrNull { record -> - record.offset - } ?: return - if (offset > lastSentOffset) { - lastSentOffset = offset - } - } - fun sent( - records: List>, - ) { - val offset = records.maxOfOrNull { - it.offset - } ?: return - if (offset > restartOffset) { - restartOffset = offset + ), + ) == null } } fun saved() { savedVersion.incrementAndGet() } + + fun trim() { + trackedMessages.entries.removeIf { (_, group) -> + group.entries.removeIf { (_, source) -> + source.entries.removeIf { (_, destination) -> + destination.isEmpty() + } + source.isEmpty() + } + group.isEmpty() + } + } + + fun forget(message: AuthenticatedMessage) { + trackedMessages[message.header.source.groupId] + ?.get(message.header.source.x500Name) + ?.get(message.header.destination.x500Name) + ?.remove(message.header.messageId) + } } +private typealias MessageIdToMessage = ConcurrentHashMap +private typealias TheirNameToMessages = ConcurrentHashMap +private typealias OurNameToMessages = ConcurrentHashMap +private typealias GroupToMessages = ConcurrentHashMap diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStates.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStates.kt index 70a9b2c9bdc..ab588bafc4c 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStates.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStates.kt @@ -6,8 +6,11 @@ import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.lifecycle.domino.logic.ComplexDominoTile import net.corda.lifecycle.domino.logic.LifecycleWithDominoTile import net.corda.messaging.api.records.EventLogRecord +import net.corda.p2p.linkmanager.delivery.ReplayScheduler +import net.corda.p2p.linkmanager.sessions.SessionManager import net.corda.p2p.linkmanager.tracker.PartitionState.Companion.stateKey import net.corda.utilities.time.Clock +import net.corda.virtualnode.toCorda import org.slf4j.LoggerFactory import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.Executors @@ -17,11 +20,13 @@ import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicReference +@Suppress("LongParameterList") internal class PartitionsStates( coordinatorFactory: LifecycleCoordinatorFactory, private val stateManager: StateManager, private val config: DeliveryTrackerConfiguration, private val clock: Clock, + private val replayScheduler: ReplayScheduler, private val executor: ScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(), ) : LifecycleWithDominoTile, DeliveryTrackerConfiguration.ConfigurationChanged { private companion object { @@ -31,22 +36,44 @@ internal class PartitionsStates( private val task = AtomicReference>() private val numberOfFailedRetries = AtomicInteger(0) - fun read(records: List>) { - val now = clock.instant() - records.groupBy { + fun getEventToProcess( + records: Collection>, + ): List> { + return records.groupBy { it.partition - }.forEach { (partition, partitionRecords) -> - partitions[partition]?.read(now, partitionRecords) + }.flatMap { (partition, records) -> + val processRecordsFromOffset = partitions[partition]?.processRecordsFromOffset ?: 0 + records.filter { + it.offset > processRecordsFromOffset + } } } - fun sent(records: List>) { + + fun read(records: Collection) { + val now = clock.instant() records.groupBy { it.partition }.forEach { (partition, partitionRecords) -> - partitions[partition]?.sent(partitionRecords) + partitions[partition]?.read(now, partitionRecords)?.forEach { messageRecord -> + val counterparties = SessionManager.Counterparties( + ourId = messageRecord.message.header.source.toCorda(), + counterpartyId = messageRecord.message.header.destination.toCorda(), + ) + replayScheduler.addForReplay( + originalAttemptTimestamp = now.toEpochMilli(), + messageId = messageRecord.message.header.messageId, + message = messageRecord.message.header.messageId, + counterparties = counterparties, + ) + } } } + fun forget(messageRecord: MessageRecord) { + val info = partitions[messageRecord.partition] + info?.forget(messageRecord.message) + } + private val partitions = ConcurrentHashMap() override val dominoTile = ComplexDominoTile( @@ -84,6 +111,7 @@ internal class PartitionsStates( val group = stateManager.createOperationGroup() partitions.values.forEach { stateToPersist -> + stateToPersist.trim() stateToPersist.addToOperationGroup(group) } val failedUpdates = try { @@ -123,21 +151,55 @@ internal class PartitionsStates( partitions.keys.removeAll(partitionsToForget) } - fun loadPartitions(partitionsToLoad: Set) { + fun loadPartitions(partitionsToLoad: Set): Map { val partitionsIndices = partitionsToLoad - partitions.keys + partitions.putAll( + loadPartitionsFromStateManager(partitionsIndices).also { loadedPartitions -> + loadedPartitions.values.flatMap { + it.counterpartiesToMessages() + }.forEach { (counterparties, messages) -> + messages.forEach { + replayScheduler.addForReplay( + it.timeStamp.toEpochMilli(), + it.messageId, + it.messageId, + counterparties, + ) + } + } + }, + ) + return partitionsToLoad.associateWith { + partitions[it] ?: PartitionState(it) + } + } + private fun loadPartitionsFromStateManager(partitionsIndices: Set): Map { if (partitionsIndices.isEmpty()) { - return + return emptyMap() } val keys = partitionsIndices.map { partitionIndex -> stateKey(partitionIndex) } val states = stateManager.get(keys) - partitionsIndices.forEach { partitionIndex -> + return partitionsIndices.associateWith { partitionIndex -> val state = states[stateKey(partitionIndex)] - partitions[partitionIndex] = PartitionState.fromState(partitionIndex, state) + PartitionState.fromState(partitionIndex, state) } } - fun get(partition: Int) = partitions[partition] + fun offsetsToReadFromChanged(partitionsToLastPersistedOffset: Collection>) { + partitionsToLastPersistedOffset.forEach { (partition, offset) -> + partitions[partition]?.readRecordsFromOffset = offset + } + } + + fun handled(events: Collection>) { + events.groupBy { + it.partition + }.forEach { (partition, records) -> + val offset = records.maxOf { it.offset } + partitions[partition]?.processRecordsFromOffset = offset + } + } } diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/StatefulDeliveryTracker.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/StatefulDeliveryTracker.kt index 4b0c5e54cf2..dfe758beef3 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/StatefulDeliveryTracker.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/StatefulDeliveryTracker.kt @@ -1,13 +1,18 @@ package net.corda.p2p.linkmanager.tracker import net.corda.libs.configuration.SmartConfig +import net.corda.lifecycle.domino.logic.ComplexDominoTile import net.corda.lifecycle.domino.logic.LifecycleWithDominoTile import net.corda.lifecycle.domino.logic.util.PublisherWithDominoLogic import net.corda.lifecycle.domino.logic.util.SubscriptionDominoTile import net.corda.messaging.api.subscription.config.SubscriptionConfig import net.corda.p2p.linkmanager.common.CommonComponents +import net.corda.p2p.linkmanager.delivery.ReplayScheduler import net.corda.p2p.linkmanager.outbound.OutboundMessageProcessor +import net.corda.p2p.linkmanager.sessions.SessionManager +import net.corda.schema.Schemas.P2P.LINK_ACK_IN_TOPIC import net.corda.schema.Schemas.P2P.P2P_OUT_TOPIC +import java.util.UUID internal class StatefulDeliveryTracker( private val commonComponents: CommonComponents, @@ -19,44 +24,124 @@ internal class StatefulDeliveryTracker( groupName = "stateless-delivery-tracker", eventTopic = P2P_OUT_TOPIC, ) + private val ackSubscriptionConfig = SubscriptionConfig( + groupName = "stateless-delivery-tracker-acks-${UUID.randomUUID()}", + eventTopic = LINK_ACK_IN_TOPIC, + ) private val config = DeliveryTrackerConfiguration( configurationReaderService = commonComponents.configurationReaderService, coordinatorFactory = commonComponents.lifecycleCoordinatorFactory, ) + private val cache = DataMessageCache( + commonComponents, + config, + ::onOffsetsToReadFromChanged, + ) + + private val replayer = MessageReplayer( + publisher = publisher, + outboundMessageProcessor = outboundMessageProcessor, + cache = cache, + ) + + private val replayScheduler = ReplayScheduler( + commonComponents = commonComponents, + limitTotalReplays = true, + replayer, + ) private val partitionsStates = PartitionsStates( coordinatorFactory = commonComponents.lifecycleCoordinatorFactory, stateManager = commonComponents.stateManager, config = config, clock = commonComponents.clock, + replayScheduler, ) + private val handler = MessagesHandler( + partitionsStates = partitionsStates, + cache = cache, + ) + + private val ackSubscription = { + commonComponents.subscriptionFactory.createPubSubSubscription( + subscriptionConfig = ackSubscriptionConfig, + processor = AckProcessor( + partitionsStates = partitionsStates, + cache = cache, + replayScheduler = replayScheduler, + ), + messagingConfig = messagingConfiguration, + ) + } + private val p2pOutSubscription = { - commonComponents.subscriptionFactory.createEventLogSubscription( + commonComponents.subscriptionFactory.createEventSourceSubscription( subscriptionConfig = subscriptionConfig, processor = DeliveryTrackerProcessor( outboundMessageProcessor, - partitionsStates, + handler, publisher, ), messagingConfig = messagingConfiguration, partitionAssignmentListener = DeliveryTrackerPartitionAssignmentListener( partitionsStates, ), + consumerOffsetProvider = DeliveryTrackerOffsetProvider( + partitionsStates, + ), ) } - override val dominoTile = SubscriptionDominoTile( + private fun onOffsetsToReadFromChanged(partitionsToLastPersistedOffset: Collection>) { + partitionsStates.offsetsToReadFromChanged(partitionsToLastPersistedOffset) + } + + private val p2pOutSubscriptionDominoTile = SubscriptionDominoTile( coordinatorFactory = commonComponents.lifecycleCoordinatorFactory, subscriptionGenerator = p2pOutSubscription, subscriptionConfig = subscriptionConfig, + managedChildren = emptySet(), + dependentChildren = setOf( + partitionsStates.dominoTile.coordinatorName, + config.dominoTile.coordinatorName, + publisher.dominoTile.coordinatorName, + cache.dominoTile.coordinatorName, + replayScheduler.dominoTile.coordinatorName, + ), + ) + private val ackSubscriptionDominoTile = SubscriptionDominoTile( + coordinatorFactory = commonComponents.lifecycleCoordinatorFactory, + subscriptionGenerator = ackSubscription, + subscriptionConfig = subscriptionConfig, + managedChildren = emptySet(), + dependentChildren = setOf( + partitionsStates.dominoTile.coordinatorName, + config.dominoTile.coordinatorName, + cache.dominoTile.coordinatorName, + replayScheduler.dominoTile.coordinatorName, + p2pOutSubscriptionDominoTile.coordinatorName, + ), + ) + + override val dominoTile = ComplexDominoTile( + coordinatorFactory = commonComponents.lifecycleCoordinatorFactory, dependentChildren = listOf( partitionsStates.dominoTile.coordinatorName, config.dominoTile.coordinatorName, publisher.dominoTile.coordinatorName, + cache.dominoTile.coordinatorName, + replayScheduler.dominoTile.coordinatorName, + p2pOutSubscriptionDominoTile.coordinatorName, + ackSubscriptionDominoTile.coordinatorName, ), managedChildren = listOf( partitionsStates.dominoTile.toNamedLifecycle(), config.dominoTile.toNamedLifecycle(), + cache.dominoTile.toNamedLifecycle(), + replayScheduler.dominoTile.toNamedLifecycle(), + p2pOutSubscriptionDominoTile.toNamedLifecycle(), + ackSubscriptionDominoTile.toNamedLifecycle(), ), + componentName = "StatefulDeliveryTracker", ) } diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/TrackedMessageState.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/TrackedMessageState.kt index 57c43bc2e6f..7163a4f1016 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/TrackedMessageState.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/TrackedMessageState.kt @@ -8,6 +8,4 @@ internal data class TrackedMessageState( val messageId: String, @JsonProperty("ts") val timeStamp: Instant, - @JsonProperty("p") - val persisted: Boolean, ) diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/exception/DeliveryTrackerExceptions.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/exception/DeliveryTrackerExceptions.kt index fdce8661d4f..3d08ea6d6aa 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/exception/DeliveryTrackerExceptions.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/exception/DeliveryTrackerExceptions.kt @@ -4,4 +4,4 @@ import net.corda.v5.base.exceptions.CordaRuntimeException class DataMessageStoreException(msg: String) : CordaRuntimeException(msg) -class DataMessageCacheException(msg: String) : CordaRuntimeException(msg) +class DataMessageCacheException(msg: String, cause: Throwable) : CordaRuntimeException(msg, cause) diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTrackerTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTrackerTest.kt index 9dbad6396b0..ff71edfb0bb 100644 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTrackerTest.kt +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/DeliveryTrackerTest.kt @@ -19,6 +19,8 @@ import net.corda.p2p.linkmanager.utilities.LoggingInterceptor import net.corda.data.p2p.markers.AppMessageMarker import net.corda.data.p2p.markers.LinkManagerProcessedMarker import net.corda.data.p2p.markers.LinkManagerReceivedMarker +import net.corda.p2p.linkmanager.common.CommonComponents +import net.corda.p2p.linkmanager.sessions.StatefulSessionManagerImpl import net.corda.test.util.identity.createTestHoldingIdentity import net.corda.virtualnode.toAvro import org.junit.jupiter.api.AfterEach @@ -131,34 +133,36 @@ class DeliveryTrackerTest { StateAndEventProcessor, StateAndEventListener > { - val publisherFactory = Mockito.mock(PublisherFactory::class.java) - - val subscriptionFactory = Mockito.mock(SubscriptionFactory::class.java) + val mockPublisherFactory = mock() val mockSubscription = MockStateAndEventSubscription() - Mockito.`when`(subscriptionFactory - .createStateAndEventSubscription(any(), any(), any(), any())) - .thenReturn(mockSubscription) + val mockSubscriptionFactory = mock { + on { + createStateAndEventSubscription(any(), any(), any(), any()) + } doReturn mockSubscription + } + val mockDominoTile = mock { + on { coordinatorName } doReturn mock() + } + val mockSessionManager = mock { + on { dominoTile } doReturn mockDominoTile + } + val commonComponents = mock { + on { publisherFactory } doReturn mockPublisherFactory + on { subscriptionFactory } doReturn mockSubscriptionFactory + on { lifecycleCoordinatorFactory } doReturn mock() + } val tracker = DeliveryTracker( + commonComponents, mock(), - mock(), - publisherFactory, - mock(), - subscriptionFactory, - mock { - val mockDominoTile = mock { - whenever(it.coordinatorName).doReturn(LifecycleCoordinatorName("", "")) - } - whenever(it.dominoTile).thenReturn(mockDominoTile) - }, - mock(), + mockSessionManager, ::processAuthenticatedMessage ) val processorCaptor = argumentCaptor>() val listenerCaptor = argumentCaptor>() - Mockito.verify(subscriptionFactory) + Mockito.verify(mockSubscriptionFactory) .createStateAndEventSubscription(anyOrNull(), processorCaptor.capture(), anyOrNull(), listenerCaptor.capture()) return Triple(tracker, processorCaptor.firstValue , listenerCaptor.firstValue) } diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/ReplaySchedulerTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/ReplaySchedulerTest.kt index 15f7303ff52..143025af8f6 100644 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/ReplaySchedulerTest.kt +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/delivery/ReplaySchedulerTest.kt @@ -8,6 +8,7 @@ import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.lifecycle.domino.logic.ComplexDominoTile import net.corda.lifecycle.domino.logic.util.ResourcesHolder +import net.corda.p2p.linkmanager.common.CommonComponents import net.corda.p2p.linkmanager.sessions.SessionManager import net.corda.p2p.linkmanager.utilities.LoggingInterceptor import net.corda.test.util.identity.createTestHoldingIdentity @@ -51,6 +52,12 @@ class ReplaySchedulerTest { private val service = mock() private val resourcesHolder = mock() private val configResourcesHolder = mock() + private val mockTimeFacilitiesProvider = MockTimeFacilitiesProvider() + private val commonComponents = mock { + on { configurationReaderService } doReturn service + on { lifecycleCoordinatorFactory } doReturn coordinatorFactory + on { clock } doReturn mockTimeFacilitiesProvider.clock + } private lateinit var configHandler: ReplayScheduler<*, *>.ReplaySchedulerConfigurationChangeHandler private val dominoTile = Mockito.mockConstruction(ComplexDominoTile::class.java) { mock, context -> @@ -58,7 +65,6 @@ class ReplaySchedulerTest { whenever(mock.withLifecycleLock(any<() -> Any>())).doAnswer { (it.arguments.first() as () -> Any).invoke() } configHandler = context.arguments()[6] as ReplayScheduler<*, *>.ReplaySchedulerConfigurationChangeHandler } - private val mockTimeFacilitiesProvider = MockTimeFacilitiesProvider() @AfterEach @@ -71,8 +77,8 @@ class ReplaySchedulerTest { @Test fun `The ReplayScheduler will not replay before start`() { - val replayManager = ReplayScheduler(coordinatorFactory, service, false, - { _: Any, _: String -> }, clock = mockTimeFacilitiesProvider.clock) + val replayManager = ReplayScheduler(commonComponents, false, + { _: Any, _: String -> },) assertThrows { replayManager.addForReplay(0,"", Any(), Mockito.mock(SessionManager.SessionCounterparties::class.java)) } @@ -81,11 +87,9 @@ class ReplaySchedulerTest { @Test fun `fromConfig correctly passes constant config`() { val replayScheduler = ReplayScheduler( - coordinatorFactory, - service, + commonComponents, false, { _: Any, _: String -> }, - clock = mockTimeFacilitiesProvider.clock ) val inner = ConfigFactory.empty() .withValue(LinkManagerConfiguration.MESSAGE_REPLAY_PERIOD_KEY, ConfigValueFactory.fromAnyRef(replayPeriod)) @@ -108,11 +112,9 @@ class ReplaySchedulerTest { fun `fromConfig correctly passes exponential backoff config`() { val cutOff = Duration.ofDays(6) val replayScheduler = ReplayScheduler( - coordinatorFactory, - service, + commonComponents, false, { _: Any, _: String -> }, - clock = mockTimeFacilitiesProvider.clock ) val inner = ConfigFactory.empty() .withValue(LinkManagerConfiguration.BASE_REPLAY_PERIOD_KEY, ConfigValueFactory.fromAnyRef(replayPeriod)) @@ -135,11 +137,9 @@ class ReplaySchedulerTest { @Test fun `on applyNewConfiguration completes the future exceptionally if config is invalid`() { ReplayScheduler( - coordinatorFactory, - service, + commonComponents, false, { _: Any, _: String -> }, - clock = mockTimeFacilitiesProvider.clock ) val future = configHandler.applyNewConfiguration( ReplayScheduler.ReplaySchedulerConfig.ExponentialBackoffReplaySchedulerConfig( @@ -158,11 +158,9 @@ class ReplaySchedulerTest { @Test fun `on applyNewConfiguration completes the future if config is valid`() { ReplayScheduler( - coordinatorFactory, - service, + commonComponents, false, { _: Any, _: String -> }, - clock = mockTimeFacilitiesProvider.clock ) val future = configHandler.applyNewConfiguration( ReplayScheduler.ReplaySchedulerConfig.ExponentialBackoffReplaySchedulerConfig( @@ -184,12 +182,10 @@ class ReplaySchedulerTest { val tracker = TrackReplayedMessages() val replayScheduler = ReplayScheduler( - coordinatorFactory, - service, + commonComponents, false, tracker::replayMessage, - {mockTimeFacilitiesProvider.mockScheduledExecutor}, - clock = mockTimeFacilitiesProvider.clock) + ) { mockTimeFacilitiesProvider.mockScheduledExecutor } setRunning() configHandler.applyNewConfiguration( ReplayScheduler.ReplaySchedulerConfig.ExponentialBackoffReplaySchedulerConfig( @@ -229,12 +225,10 @@ class ReplaySchedulerTest { val tracker = TrackReplayedMessages() val replayScheduler = ReplayScheduler( - coordinatorFactory, - service, + commonComponents, false, tracker::replayMessage, - {mockTimeFacilitiesProvider.mockScheduledExecutor}, - clock = mockTimeFacilitiesProvider.clock) + ) { mockTimeFacilitiesProvider.mockScheduledExecutor } setRunning() configHandler.applyNewConfiguration( ReplayScheduler.ReplaySchedulerConfig.ExponentialBackoffReplaySchedulerConfig( @@ -262,12 +256,10 @@ class ReplaySchedulerTest { val tracker = TrackReplayedMessages() val replayScheduler = ReplayScheduler( - coordinatorFactory, - service, + commonComponents, false, tracker::replayMessage, - {mockTimeFacilitiesProvider.mockScheduledExecutor}, - clock = mockTimeFacilitiesProvider.clock) + ) { mockTimeFacilitiesProvider.mockScheduledExecutor } setRunning() configHandler.applyNewConfiguration( ReplayScheduler.ReplaySchedulerConfig.ExponentialBackoffReplaySchedulerConfig( @@ -308,12 +300,10 @@ class ReplaySchedulerTest { } val replayScheduler = ReplayScheduler( - coordinatorFactory, - service, + commonComponents, false, ::replayMessage, - {mockTimeFacilitiesProvider.mockScheduledExecutor}, - clock = mockTimeFacilitiesProvider.clock) + ) { mockTimeFacilitiesProvider.mockScheduledExecutor } replayScheduler.start() setRunning() configHandler.applyNewConfiguration( @@ -340,12 +330,10 @@ class ReplaySchedulerTest { fun `The ReplayScheduler replays added messages after config update`() { val tracker = TrackReplayedMessages() val replayScheduler = ReplayScheduler( - coordinatorFactory, - service, + commonComponents, false, tracker::replayMessage, - {mockTimeFacilitiesProvider.mockScheduledExecutor}, - clock = mockTimeFacilitiesProvider.clock) + ) { mockTimeFacilitiesProvider.mockScheduledExecutor } replayScheduler.start() setRunning() configHandler.applyNewConfiguration( @@ -391,12 +379,10 @@ class ReplaySchedulerTest { val messageCap = 1 val tracker = TrackReplayedMessages() val replayScheduler = ReplayScheduler( - coordinatorFactory, - service, + commonComponents, true, tracker::replayMessage, - {mockTimeFacilitiesProvider.mockScheduledExecutor}, - clock = mockTimeFacilitiesProvider.clock) + ) { mockTimeFacilitiesProvider.mockScheduledExecutor } replayScheduler.start() setRunning() configHandler.applyNewConfiguration( @@ -431,12 +417,10 @@ class ReplaySchedulerTest { val messageCap = 3 val tracker = TrackReplayedMessages() val replayScheduler = ReplayScheduler( - coordinatorFactory, - service, + commonComponents, true, tracker::replayMessage, - {mockTimeFacilitiesProvider.mockScheduledExecutor}, - clock = mockTimeFacilitiesProvider.clock) + ) { mockTimeFacilitiesProvider.mockScheduledExecutor } replayScheduler.start() setRunning() configHandler.applyNewConfiguration( @@ -477,12 +461,10 @@ class ReplaySchedulerTest { val messageCap = 1 val tracker = TrackReplayedMessages() val replayScheduler = ReplayScheduler( - coordinatorFactory, - service, + commonComponents, true, tracker::replayMessage, - {mockTimeFacilitiesProvider.mockScheduledExecutor}, - clock = mockTimeFacilitiesProvider.clock) + ) { mockTimeFacilitiesProvider.mockScheduledExecutor } replayScheduler.start() setRunning() configHandler.applyNewConfiguration( @@ -525,12 +507,10 @@ class ReplaySchedulerTest { val messageCap = 1 val tracker = TrackReplayedMessages() val replayScheduler = ReplayScheduler( - coordinatorFactory, - service, + commonComponents, true, tracker::replayMessage, - {mockTimeFacilitiesProvider.mockScheduledExecutor}, - clock = mockTimeFacilitiesProvider.clock) + ) { mockTimeFacilitiesProvider.mockScheduledExecutor } replayScheduler.start() setRunning() configHandler.applyNewConfiguration( diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/AckProcessorTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/AckProcessorTest.kt new file mode 100644 index 00000000000..f9f43b99b1a --- /dev/null +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/AckProcessorTest.kt @@ -0,0 +1,137 @@ +package net.corda.p2p.linkmanager.tracker + +import net.corda.data.identity.HoldingIdentity +import net.corda.data.p2p.app.AuthenticatedMessageHeader +import net.corda.messaging.api.records.Record +import net.corda.p2p.linkmanager.delivery.ReplayScheduler +import net.corda.p2p.linkmanager.sessions.SessionManager +import net.corda.v5.base.types.MemberX500Name +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.mock +import org.mockito.kotlin.verify +import org.mockito.kotlin.verifyNoInteractions +import net.corda.virtualnode.HoldingIdentity as CordaHoldingIdentity + +class AckProcessorTest { + private val messageId = "messageId" + val mockHeader = mock { + on { source } doReturn HoldingIdentity( + "CN=Alice, O=Alice Corp, L=LDN, C=GB", + "group", + ) + on { destination } doReturn HoldingIdentity( + "CN=Bob, O=Bob Corp, L=LDN, C=GB", + "group", + ) + } + private val record = MessageRecord( + message = mock { + on { header } doReturn mockHeader + }, + partition = 20, + offset = 1020, + ) + private val partitionsStates = mock() + private val cache = mock { + on { remove(messageId) } doReturn record + } + private val replayScheduler = mock>() + + private val processor = AckProcessor( + partitionsStates = partitionsStates, + cache = cache, + replayScheduler = replayScheduler, + ) + + @Test + fun `unknown message will return a completed future`() { + val future = processor.onNext( + Record( + "topic", + "unknown-key", + "value", + ), + ) + + assertThat(future).isDone + } + + @Test + fun `unknown message will not interact with the states`() { + processor.onNext( + Record( + "topic", + "unknown-key", + "value", + ), + ) + + verifyNoInteractions(partitionsStates) + } + + @Test + fun `unknown message will not interact with the scheduler`() { + processor.onNext( + Record( + "topic", + "unknown-key", + "value", + ), + ) + + verifyNoInteractions(replayScheduler) + } + + @Test + fun `known message will not tell the states to forget it`() { + processor.onNext( + Record( + "topic", + messageId, + "value", + ), + ) + + verify(partitionsStates).forget(record) + } + + @Test + fun `known message will remove it from the scheduler`() { + processor.onNext( + Record( + "topic", + messageId, + "value", + ), + ) + + verify(replayScheduler).removeFromReplay( + messageId, + SessionManager.Counterparties( + CordaHoldingIdentity( + MemberX500Name.parse("CN=Alice, O=Alice Corp, L=LDN, C=GB"), + "group", + ), + CordaHoldingIdentity( + MemberX500Name.parse("CN=Bob, O=Bob Corp, L=LDN, C=GB"), + "group", + ), + ), + ) + } + + @Test + fun `known message will return a completed future`() { + val future = processor.onNext( + Record( + "topic", + messageId, + "value", + ), + ) + + assertThat(future).isDone + } +} diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DataMessageCacheTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DataMessageCacheTest.kt index 15dabfc4e18..39ed395bdcf 100644 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DataMessageCacheTest.kt +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DataMessageCacheTest.kt @@ -1,37 +1,53 @@ package net.corda.p2p.linkmanager.tracker -import net.corda.data.p2p.app.AppMessage +import net.corda.data.p2p.app.AuthenticatedMessage +import net.corda.data.p2p.app.AuthenticatedMessageHeader +import net.corda.libs.statemanager.api.Metadata import net.corda.libs.statemanager.api.State import net.corda.libs.statemanager.api.StateManager +import net.corda.lifecycle.ErrorEvent import net.corda.lifecycle.LifecycleCoordinator import net.corda.lifecycle.LifecycleCoordinatorFactory import net.corda.lifecycle.LifecycleEventHandler -import net.corda.lifecycle.LifecycleStatus import net.corda.lifecycle.RegistrationHandle -import net.corda.lifecycle.RegistrationStatusChangeEvent import net.corda.lifecycle.domino.logic.ComplexDominoTile -import net.corda.messaging.api.records.EventLogRecord +import net.corda.p2p.linkmanager.common.CommonComponents import net.corda.schema.registry.AvroSchemaRegistry import net.corda.schema.registry.deserialize +import net.corda.v5.base.exceptions.CordaRuntimeException import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatCode import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test import org.mockito.kotlin.any import org.mockito.kotlin.argumentCaptor import org.mockito.kotlin.doAnswer import org.mockito.kotlin.doReturn +import org.mockito.kotlin.doThrow +import org.mockito.kotlin.isA import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.times import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever import java.nio.ByteBuffer +import java.util.concurrent.ConcurrentLinkedQueue class DataMessageCacheTest { private val knownKey = "test" private val valueBytes = knownKey.toByteArray() + private val metadata = Metadata( + mapOf( + "OFFSET" to 1L, + "PARTITION" to 20, + ), + ) private val states = mapOf( - knownKey to mock { - on { key } doReturn knownKey - on { value } doReturn valueBytes - } + knownKey to State( + key = knownKey, + value = valueBytes, + metadata = metadata, + ), ) private val createCaptor = argumentCaptor>() private val deleteCaptor = argumentCaptor>() @@ -41,10 +57,10 @@ class DataMessageCacheTest { on { create(createCaptor.capture()) } doReturn emptySet() on { delete(deleteCaptor.capture()) } doReturn emptyMap() } - private val appMessage = mock() + private val message = mock() private val schemaRegistry = mock { on { serialize(any()) } doReturn ByteBuffer.wrap(valueBytes) - on { deserialize(ByteBuffer.wrap(valueBytes)) } doReturn appMessage + on { deserialize(ByteBuffer.wrap(valueBytes)) } doReturn message } private val configDominoTile = mock { on { coordinatorName } doReturn mock() @@ -52,8 +68,7 @@ class DataMessageCacheTest { private val config = mock { on { dominoTile } doReturn configDominoTile on { config } doReturn DeliveryTrackerConfiguration.Configuration( - maxCacheSizeMegabytes = 100, - maxCacheOffsetAge = 50000, + maxCacheOffsetAge = 300, statePersistencePeriodSeconds = 1.0, outboundBatchProcessingTimeoutSeconds = 30.0, maxNumberOfPersistenceRetries = 3, @@ -71,96 +86,447 @@ class DataMessageCacheTest { private val coordinatorFactory = mock { on { createCoordinator(any(), handler.capture()) } doReturn coordinator } + private val commonComponents = mock { + on { lifecycleCoordinatorFactory } doReturn coordinatorFactory + on { stateManager } doReturn stateManager + on { schemaRegistry } doReturn schemaRegistry + } + private val onOffsetsToReadFromChanged = ConcurrentLinkedQueue>() private val cache = DataMessageCache( - coordinatorFactory, - stateManager, - schemaRegistry, + commonComponents, config, - ) + ) { + onOffsetsToReadFromChanged.addAll(it) + } private fun createTestRecord( - key: String, - value: AppMessage, + id: String, partition: Int = 1, offset: Long = 1, - ) = EventLogRecord( - topic = "topic", - key = key, - value = value, - partition = partition, - offset = offset, - ) - - @Test - fun `onStart will listen to configuration changes`() { - followStatusChangesByNameHandlers.forEach { followStatusChangesByNameHandler -> - handler.firstValue.processEvent( - RegistrationStatusChangeEvent( - followStatusChangesByNameHandler, - LifecycleStatus.UP, - ), - coordinator, - ) + ): MessageRecord { + val headers = mock { + on { messageId } doReturn id } - - verify(config).lister(cache) + val message = mock { + on { header } doReturn headers + } + return MessageRecord( + message = message, + partition = partition, + offset = offset, + ) } @Nested inner class GetTests { @Test fun `get can return a value previously cached using put`() { - val testMessage = mock() val key = "key" - cache.put(createTestRecord(key, testMessage)) + val records = listOf(createTestRecord(key)) + cache.put(records) val result = cache.get(key) - assertThat(result).isEqualTo(testMessage) + assertThat(result).isEqualTo(records.first().message) + verify(stateManager, never()).get(setOf(knownKey)) } @Test fun `get will query the state manager if entry is not in the cache`() { val result = cache.get(knownKey) - assertThat(result).isEqualTo(appMessage) + assertThat(result).isEqualTo(message) verify(stateManager).get(setOf(knownKey)) } + + @Test + fun `get will return nothing for unknown key`() { + val result = cache.get("nop") + + assertThat(result).isNull() + } + + @Test + fun `get will return nothing for error during read`() { + whenever(stateManager.get(setOf("nop"))).doThrow(CordaRuntimeException("Ooops")) + + val result = cache.get("nop") + + assertThat(result).isNull() + } } @Nested inner class PutTests { @Test - fun `put will persist older cache entries to state manager`() { - val testMessage = mock() - val key1 = "key-1" - val key2 = "key-2" - cache.put(createTestRecord(key1, testMessage, 1, 10)) + fun `put without messages won't throw an exception`() { + assertThatCode { + cache.put(emptyList()) + }.doesNotThrowAnyException() + } + + @Test + fun `put will keep the records in cache`() { + cache.put( + listOf( + createTestRecord( + id = "one", + offset = 10, + partition = 1, + ), + createTestRecord( + id = "two", + offset = 10, + partition = 2, + ), + createTestRecord( + id = "three", + offset = 20, + partition = 1, + ), + ), + ) + + assertThat(cache.get("two")).isNotNull() + } - cache.put(createTestRecord(key2, testMessage, 1, 55000)) + @Test + fun `put will not persist if not needed`() { + cache.put( + listOf( + createTestRecord( + id = "one", + offset = 10, + partition = 1, + ), + createTestRecord( + id = "two", + offset = 10, + partition = 2, + ), + createTestRecord( + id = "three", + offset = 20, + partition = 1, + ), + ), + ) + + verify(stateManager, never()).create(any()) + } + + @Test + fun `put will persist the correct message`() { + val persisted = argumentCaptor>() + whenever(stateManager.create(persisted.capture())).doReturn(emptySet()) + cache.put( + listOf( + createTestRecord( + id = "1", + offset = 1, + partition = 1, + ), + createTestRecord( + id = "another-partition", + offset = 1, + partition = 2, + ), + ), + ) + + cache.put( + listOf( + createTestRecord( + id = "2", + offset = 2, + partition = 1, + ), + createTestRecord( + id = "3", + offset = 3, + partition = 1, + ), + createTestRecord( + id = "4", + offset = 4, + partition = 1, + ), + createTestRecord( + id = "299", + offset = 299, + partition = 1, + ), + createTestRecord( + id = "300", + offset = 300, + partition = 1, + ), + createTestRecord( + id = "301", + offset = 301, + partition = 1, + ), + createTestRecord( + id = "302", + offset = 302, + partition = 1, + ), + createTestRecord( + id = "303", + offset = 303, + partition = 1, + ), + createTestRecord( + id = "304", + offset = 304, + partition = 1, + ), + ), + ) + + val persistedIds = persisted.firstValue.map { it.key } + assertThat(persistedIds).contains( + "1", + "2", + "3", + ) + } + + @Test + fun `put will update the offsets to read from`() { + cache.put( + listOf( + createTestRecord( + id = "another-partition", + offset = 20, + partition = 2, + ), + createTestRecord( + id = "3", + offset = 3, + partition = 1, + ), + createTestRecord( + id = "4", + offset = 4, + partition = 1, + ), + createTestRecord( + id = "299", + offset = 299, + partition = 1, + ), + createTestRecord( + id = "300", + offset = 300, + partition = 1, + ), + createTestRecord( + id = "301", + offset = 301, + partition = 1, + ), + createTestRecord( + id = "302", + offset = 302, + partition = 1, + ), + createTestRecord( + id = "303", + offset = 303, + partition = 1, + ), + createTestRecord( + id = "504", + offset = 504, + partition = 1, + ), + ), + ) + + assertThat(onOffsetsToReadFromChanged).containsExactly( + 1 to 299, + 2 to 20, + ) + } + + @Test + fun `put will set the tile in error state in case of an exception`() { + val error = CordaRuntimeException("Ooops") + whenever(stateManager.create(any())).doThrow(error) + + cache.put( + listOf( + createTestRecord( + id = "3", + offset = 3, + partition = 1, + ), + createTestRecord( + id = "504", + offset = 504, + partition = 1, + ), + ), + ) - assertThat(createCaptor.firstValue.single().key).isEqualTo(key1) + verify(coordinator).postEvent(isA()) + } + + @Test + fun `put will ignore the state manager reply`() { + whenever(stateManager.create(any())).doReturn(setOf("3")) + + cache.put( + listOf( + createTestRecord( + id = "3", + offset = 3, + partition = 1, + ), + createTestRecord( + id = "504", + offset = 504, + partition = 1, + ), + ), + ) + + verify(coordinator, never()).postEvent(isA()) } } @Nested - inner class InvalidateTests { + inner class RemoveTests { @Test - fun `invalidate will delete from state manager if key not in cache`() { - cache.invalidate(knownKey) + fun `remove will delete from state manager if key not in cache`() { + cache.remove(knownKey) assertThat(deleteCaptor.firstValue.single().key).isEqualTo(knownKey) } @Test - fun `invalidate discards previously added cache entry`() { + fun `remove discards previously added cache entry`() { val key = "key" - cache.put(createTestRecord(key, mock())) + cache.put( + listOf( + createTestRecord(key), + ), + ) - cache.invalidate(key) + cache.remove(key) assertThat(cache.get(key)).isNull() } + + @Test + fun `remove will notify about offset to read change if changed`() { + cache.put( + listOf( + createTestRecord( + id = "2", + offset = 2, + partition = 1, + ), + createTestRecord( + id = "3", + offset = 3, + partition = 1, + ), + createTestRecord( + id = "4", + offset = 4, + partition = 1, + ), + ), + ) + + cache.remove("2") + + assertThat(onOffsetsToReadFromChanged).contains(1 to 3) + } + + @Test + fun `remove will notify about offset to read change if empty`() { + cache.put( + listOf( + createTestRecord( + id = "4", + offset = 4, + partition = 1, + ), + ), + ) + + cache.remove("4") + + assertThat(onOffsetsToReadFromChanged).contains(1 to 5) + } + + @Test + fun `remove will return the record if found`() { + cache.put( + listOf( + createTestRecord( + id = "4", + offset = 4, + partition = 1, + ), + ), + ) + + val record = cache.remove("4") + + assertThat(record).isNotNull + } + + @Test + fun `remove will return null if not found`() { + val record = cache.remove("key") + + assertThat(record).isNull() + } + + @Test + fun `remove will try again if the state manager failed the first time`() { + whenever(stateManager.delete(any())) + .thenReturn(mapOf("key" to mock())) + .thenReturn(emptyMap()) + + cache.remove(knownKey) + + verify(stateManager, times(2)).delete(states.values) + } + + @Test + fun `remove will stop trying after a while`() { + whenever(stateManager.delete(any())) + .doReturn(mapOf("key" to mock())) + + val record = cache.remove(knownKey) + + assertThat(record).isNull() + } + + @Test + fun `remove will try once if the state manager had an exception`() { + whenever(stateManager.delete(any())) + .thenThrow(CordaRuntimeException("oops")) + + cache.remove(knownKey) + + verify(stateManager).delete(states.values) + } + + @Test + fun `remove will return the state event if it failed to delete it`() { + whenever(stateManager.delete(any())) + .doThrow(CordaRuntimeException("Nop")) + + val record = cache.remove(knownKey) + + assertThat(record).isNotNull() + } } } diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerConfigurationTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerConfigurationTest.kt index 8c9e6e312dd..dd060bf5647 100644 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerConfigurationTest.kt +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerConfigurationTest.kt @@ -3,7 +3,6 @@ package net.corda.p2p.linkmanager.tracker import com.typesafe.config.ConfigFactory import com.typesafe.config.ConfigValueFactory import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.DELIVERY_TRACKER_MAX_CACHE_OFFSET_AGE -import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.DELIVERY_TRACKER_MAX_CACHE_SIZE_MEGABYTES import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.DELIVERY_TRACKER_MAX_NUMBER_OF_PERSISTENCE_RETRIES import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.DELIVERY_TRACKER_OUTBOUND_BATCH_PROCESSING_TIMEOUT_SECONDS import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.DELIVERY_TRACKER_STATE_PERSISTENCE_PERIOD_SECONDS @@ -25,7 +24,6 @@ class DeliveryTrackerConfigurationTest { assertThat(config).isEqualTo( DeliveryTrackerConfiguration.Configuration( - maxCacheSizeMegabytes = 100, maxCacheOffsetAge = 50000, statePersistencePeriodSeconds = 1.0, outboundBatchProcessingTimeoutSeconds = 30.0, @@ -37,7 +35,6 @@ class DeliveryTrackerConfigurationTest { @Test fun `applyNewConfiguration will change the configuration`() { val config = DeliveryTrackerConfiguration.Configuration( - maxCacheSizeMegabytes = 200, maxCacheOffsetAge = 300, statePersistencePeriodSeconds = 400.0, outboundBatchProcessingTimeoutSeconds = 500.0, @@ -56,7 +53,6 @@ class DeliveryTrackerConfigurationTest { @Test fun `applyNewConfiguration will complete the future`() { val config = DeliveryTrackerConfiguration.Configuration( - maxCacheSizeMegabytes = 200, maxCacheOffsetAge = 300, statePersistencePeriodSeconds = 400.0, outboundBatchProcessingTimeoutSeconds = 500.0, @@ -77,7 +73,6 @@ class DeliveryTrackerConfigurationTest { val listener = mock() configurationTile.lister(listener) val config = DeliveryTrackerConfiguration.Configuration( - maxCacheSizeMegabytes = 200, maxCacheOffsetAge = 300, statePersistencePeriodSeconds = 400.0, outboundBatchProcessingTimeoutSeconds = 500.0, @@ -109,10 +104,6 @@ class DeliveryTrackerConfigurationTest { @Test fun `fromConfig returns the correct configuration`() { val config = ConfigFactory.empty() - .withValue( - DELIVERY_TRACKER_MAX_CACHE_SIZE_MEGABYTES, - ConfigValueFactory.fromAnyRef(101), - ) .withValue( DELIVERY_TRACKER_MAX_CACHE_OFFSET_AGE, ConfigValueFactory.fromAnyRef(202), @@ -134,7 +125,6 @@ class DeliveryTrackerConfigurationTest { assertThat(configuration).isEqualTo( DeliveryTrackerConfiguration.Configuration( - maxCacheSizeMegabytes = 101, maxCacheOffsetAge = 202, statePersistencePeriodSeconds = 303.0, outboundBatchProcessingTimeoutSeconds = 404.0, diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerOffsetProviderTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerOffsetProviderTest.kt new file mode 100644 index 00000000000..5d47a7b2758 --- /dev/null +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerOffsetProviderTest.kt @@ -0,0 +1,37 @@ +package net.corda.p2p.linkmanager.tracker + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.mock + +class DeliveryTrackerOffsetProviderTest { + private val partitionsIndices = setOf(4, 6, 1) + private val stateOne = mock { + on { readRecordsFromOffset } doReturn 200 + } + private val stateTwo = mock { + on { readRecordsFromOffset } doReturn 310 + } + private val partitionStates = mock { + on { loadPartitions(partitionsIndices) } doReturn + mapOf( + 5 to stateOne, + 10 to stateTwo, + ) + } + + private val provider = DeliveryTrackerOffsetProvider(partitionStates) + + @Test + fun `getStartingOffsets return the correct offsets`() { + val offsets = provider.getStartingOffsets( + partitionsIndices.map { "topic" to it }.toSet(), + ) + + assertThat(offsets) + .hasSize(2) + .containsEntry(("topic" to 5), 200L) + .containsEntry(("topic" to 10), 310L) + } +} diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerPartitionAssignmentListenerTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerPartitionAssignmentListenerTest.kt new file mode 100644 index 00000000000..b2cc5a220a1 --- /dev/null +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerPartitionAssignmentListenerTest.kt @@ -0,0 +1,49 @@ +package net.corda.p2p.linkmanager.tracker + +import org.junit.jupiter.api.Test +import org.mockito.kotlin.mock +import org.mockito.kotlin.verify + +class DeliveryTrackerPartitionAssignmentListenerTest { + private val states = mock() + + private val listener = DeliveryTrackerPartitionAssignmentListener(states) + + @Test + fun `onPartitionsAssigned call the partition states`() { + listener.onPartitionsAssigned( + listOf( + "topic" to 4, + "topic" to 16, + "topic" to 8, + ), + ) + + verify(states).loadPartitions( + setOf( + 4, + 16, + 8, + ), + ) + } + + @Test + fun `onPartitionsUnassigned call the partition states`() { + listener.onPartitionsUnassigned( + listOf( + "topic" to 4, + "topic" to 16, + "topic" to 8, + ), + ) + + verify(states).forgetPartitions( + setOf( + 4, + 16, + 8, + ), + ) + } +} diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerProcessorTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerProcessorTest.kt index 42b1d2abc6a..83024e10f96 100644 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerProcessorTest.kt +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerProcessorTest.kt @@ -6,21 +6,13 @@ import net.corda.messaging.api.records.EventLogRecord import net.corda.messaging.api.records.Record import net.corda.p2p.linkmanager.outbound.OutboundMessageProcessor import org.junit.jupiter.api.Test +import org.mockito.kotlin.any import org.mockito.kotlin.doReturn import org.mockito.kotlin.mock import org.mockito.kotlin.verify -import org.mockito.kotlin.whenever import java.util.concurrent.CompletableFuture class DeliveryTrackerProcessorTest { - private val outboundMessageProcessor = mock {} - private val partitionsStates = mock {} - private val publisher = mock {} - private val processor = DeliveryTrackerProcessor( - outboundMessageProcessor, - partitionsStates, - publisher, - ) private val records = listOf( EventLogRecord( topic = "topic", @@ -37,47 +29,78 @@ class DeliveryTrackerProcessorTest { offset = 3003, ), ) + private val recordsToForward = listOf( + EventLogRecord( + topic = "topic", + key = "key-2", + value = mock(), + partition = 4, + offset = 104, + ), + EventLogRecord( + topic = "topic", + key = "key-3", + value = mock(), + partition = 5, + offset = 3002, + ), + ) + private val recordsToPublish = listOf( + Record( + topic = "topic", + key = "key-4", + value = 1000, + ), + ) + private val outboundMessageProcessor = mock { + on { onNext(any()) } doReturn recordsToPublish + } + private val future = mock> {} + private val publisher = mock { + on { publish(any()) } doReturn listOf(future) + } + private val handler = mock { + on { handleMessagesAndFilterRecords(any()) } doReturn recordsToForward + } + + private val processor = DeliveryTrackerProcessor( + outboundMessageProcessor, + handler, + publisher, + ) @Test - fun `onNext will send the messages to the outbound processor`() { + fun `onNext will send the messages to the handler`() { processor.onNext(records) - verify(outboundMessageProcessor).onNext(records) + verify(handler).handleMessagesAndFilterRecords(records) } @Test - fun `onNext will update the states before sending`() { + fun `onNext will send the messages to the processor`() { processor.onNext(records) - verify(partitionsStates).read(records) + verify(outboundMessageProcessor).onNext(recordsToForward) } @Test - fun `onNext will update the states after sending`() { + fun `onNext will publish the records`() { processor.onNext(records) - verify(partitionsStates).sent(records) + verify(publisher).publish(recordsToPublish) } @Test - fun `onNext will publish the records and wait publication`() { - val replies = listOf( - Record( - topic = "topic", - key = "key", - value = "", - ), - Record( - topic = "topic", - key = "key", - value = "another", - ), - ) - val future = mock>() - whenever(outboundMessageProcessor.onNext(records)).doReturn(replies) - whenever(publisher.publish(replies)).doReturn(listOf(future)) + fun `onNext will wait for published records`() { processor.onNext(records) verify(future).join() } + + @Test + fun `onNext will notify that it handled the messages`() { + processor.onNext(records) + + verify(handler).handled(records) + } } diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/MessageReplayerTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/MessageReplayerTest.kt new file mode 100644 index 00000000000..14c50176b21 --- /dev/null +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/MessageReplayerTest.kt @@ -0,0 +1,68 @@ +package net.corda.p2p.linkmanager.tracker + +import net.corda.data.p2p.AuthenticatedMessageAndKey +import net.corda.data.p2p.app.AuthenticatedMessage +import net.corda.lifecycle.domino.logic.util.PublisherWithDominoLogic +import net.corda.messaging.api.records.Record +import net.corda.p2p.linkmanager.outbound.OutboundMessageProcessor +import org.junit.jupiter.api.Test +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.mock +import org.mockito.kotlin.verify +import org.mockito.kotlin.verifyNoInteractions +import java.util.concurrent.CompletableFuture + +class MessageReplayerTest { + private val key = "key" + private val message = mock() + + private val cache = mock { + on { get(key) } doReturn message + } + private val records = listOf( + Record( + "topic", + "key1", + "value1", + ), + Record( + "topic", + "key2", + "value2", + ), + ) + private val outboundMessageProcessor = mock { + on { + processReplayedAuthenticatedMessage( + AuthenticatedMessageAndKey( + message, + key, + ), + ) + } doReturn records + } + private val future = mock> { } + private val publisher = mock { + on { publish(records) } doReturn listOf(future) + } + + private val replayer = MessageReplayer( + publisher, + outboundMessageProcessor, + cache, + ) + + @Test + fun `invoke will wait for the publisher to publish the records`() { + replayer.invoke(key, key) + + verify(future).join() + } + + @Test + fun `invoke will do nothing if a message can not be found`() { + replayer.invoke("id", "id") + + verifyNoInteractions(outboundMessageProcessor) + } +} diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/MessagesHandlerTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/MessagesHandlerTest.kt new file mode 100644 index 00000000000..4830a99d198 --- /dev/null +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/MessagesHandlerTest.kt @@ -0,0 +1,124 @@ +package net.corda.p2p.linkmanager.tracker + +import net.corda.data.p2p.app.AppMessage +import net.corda.data.p2p.app.AuthenticatedMessage +import net.corda.messaging.api.records.EventLogRecord +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.mock +import org.mockito.kotlin.verify + +class MessagesHandlerTest { + private val messageOne = mock() + private val messageTwo = mock() + private val events: List> = listOf( + EventLogRecord( + topic = "topic", + key = "key1", + value = mock { + on { message } doReturn messageOne + }, + offset = 100, + partition = 40, + ), + EventLogRecord( + topic = "topic", + key = "key2", + value = mock {}, + offset = 101, + partition = 30, + ), + EventLogRecord( + topic = "topic", + key = "key", + value = mock { + on { message } doReturn messageTwo + }, + offset = 102, + partition = 30, + ), + EventLogRecord( + topic = "topic", + key = "key", + value = null, + offset = 102, + partition = 30, + ), + ) + private val records: List> = listOf( + EventLogRecord( + topic = "topic", + key = "key1", + value = mock { }, + offset = 100, + partition = 30, + ), + EventLogRecord( + topic = "topic", + key = "key2", + value = null, + offset = 101, + partition = 30, + ), + ) + private val partitionsStates = mock { + on { getEventToProcess(events) } doReturn records + } + private val cache = mock {} + private val handler = MessagesHandler(partitionsStates, cache) + + @Test + fun `handleMessagesAndFilterRecords sends only the AuthenticatedMessage to the cache`() { + handler.handleMessagesAndFilterRecords(events) + + verify(cache).put( + listOf( + MessageRecord( + message = messageOne, + offset = 100, + partition = 40, + ), + MessageRecord( + message = messageTwo, + offset = 102, + partition = 30, + ), + ), + ) + } + + @Test + fun `handleMessagesAndFilterRecords sends only the AuthenticatedMessage to the states`() { + handler.handleMessagesAndFilterRecords(events) + + verify(partitionsStates).read( + listOf( + MessageRecord( + message = messageOne, + offset = 100, + partition = 40, + ), + MessageRecord( + message = messageTwo, + offset = 102, + partition = 30, + ), + ), + ) + } + + @Test + fun `handleMessagesAndFilterRecords return the filter events`() { + val returnedRecords = handler.handleMessagesAndFilterRecords(events) + + assertThat(returnedRecords).isEqualTo(records) + } + + @Test + fun `handled sends the events to the state`() { + handler.handled(events) + + verify(partitionsStates).handled(events) + } +} diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionStateTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionStateTest.kt index d199b2beab5..f62fa83f044 100644 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionStateTest.kt +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionStateTest.kt @@ -1,514 +1,400 @@ package net.corda.p2p.linkmanager.tracker -import net.corda.data.p2p.app.AppMessage import net.corda.data.p2p.app.AuthenticatedMessage import net.corda.data.p2p.app.AuthenticatedMessageHeader -import net.corda.data.p2p.app.InboundUnauthenticatedMessage -import net.corda.data.p2p.app.InboundUnauthenticatedMessageHeader -import net.corda.data.p2p.app.OutboundUnauthenticatedMessage -import net.corda.data.p2p.app.OutboundUnauthenticatedMessageHeader import net.corda.libs.statemanager.api.State import net.corda.libs.statemanager.api.StateOperationGroup -import net.corda.messaging.api.records.EventLogRecord +import net.corda.p2p.linkmanager.sessions.SessionManager +import net.corda.p2p.linkmanager.tracker.PartitionState.Companion.fromState import net.corda.p2p.linkmanager.tracker.PartitionState.Companion.stateKey import net.corda.v5.base.exceptions.CordaRuntimeException +import net.corda.v5.base.types.MemberX500Name +import net.corda.virtualnode.HoldingIdentity import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy import org.assertj.core.api.SoftAssertions.assertSoftly import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows +import org.mockito.kotlin.any import org.mockito.kotlin.argumentCaptor import org.mockito.kotlin.doReturn import org.mockito.kotlin.mock +import org.mockito.kotlin.verify import java.time.Instant +import net.corda.data.identity.HoldingIdentity as AvroHoldingIdentity class PartitionStateTest { - @Test - fun `restartOffset set works as expected`() { - val state = PartitionState(1) - - state.restartOffset = 400 - - assertThat(state.restartOffset).isEqualTo(400) - } - - @Test - fun `lastSentOffset set works as expected`() { - val state = PartitionState(1) - - state.lastSentOffset = 420 - - assertThat(state.lastSentOffset).isEqualTo(420) - } - - @Test - fun `addMessage works as expected`() { - val state = PartitionState(1) - val message = TrackedMessageState( - messageId = "id", - timeStamp = mock(), - persisted = false, - ) - - state.addMessage(message) - - assertThat(state.getTrackMessage("id")).isEqualTo(message) - } - - @Test - fun `untrackMessage removes message from list`() { - val state = PartitionState(1) - val message = TrackedMessageState( - messageId = "id", - timeStamp = mock(), - persisted = false, - ) - state.addMessage(message) - - state.untrackMessage(message.messageId) - - assertThat(state.getTrackMessage("id")).isNull() - } - @Nested - inner class FromStateTests { + inner class FromState { @Test - fun `init happy path`() { + fun `it reads the state correctly`() { val json = """ { - "partition": 1, - "restartOffset": 100000, - "lastSentOffset": 200000, + "processRecordsFromOffset": 300, + "readRecordsFromOffset": 204, "messages": { - "id1": { - "id": "id1", - "ts": 10, - "p": true - }, "id2": { - "id": "id2", - "ts": 10.002, - "p": false + "group-2": { + "CN=Alice, O=Alice Corp, L=LDN, C=GB": { + "CN=Bob, O=Bob Corp, L=LDN, C=GB": { + "id-2": { + "id": "id-2", + "ts": 10.231 + } + } + } + }, + "group": { + "CN=Alice, O=Alice Corp, L=LDN, C=GB": { + "CN=Carol, O=Carol Corp, L=LDN, C=GB": { + "id-3": { + "id": "id-3", + "ts": 10.231 + } + }, + "CN=Bob, O=Bob Corp, L=LDN, C=GB": { + "id-1": { + "id": "id-1", + "ts": 10.231 + } + } + } } - } + }, + "version": 12 } - """.trimIndent() - val state = State( - key = "", - value = json.toByteArray(), - version = 100, - ) + """ + val state = + State( + key = stateKey(1), + value = json.toByteArray(), + ) - val partitionState = PartitionState.fromState(3, state) + val partitionState = fromState(3, state) assertSoftly { - it.assertThat(partitionState.restartOffset).isEqualTo(100000) - it.assertThat(partitionState.lastSentOffset).isEqualTo(200000) - it.assertThat(partitionState.getTrackMessage("id1")).isEqualTo( - TrackedMessageState( - messageId = "id1", - timeStamp = Instant.ofEpochMilli(10000), - persisted = true, - ), - ) - it.assertThat(partitionState.getTrackMessage("id2")).isEqualTo( - TrackedMessageState( - messageId = "id2", - timeStamp = Instant.ofEpochMilli(10002), - persisted = false, - ), - ) + it.assertThat(partitionState.readRecordsFromOffset).isEqualTo(204) + it.assertThat(partitionState.processRecordsFromOffset).isEqualTo(300) + it.assertThat(partitionState.counterpartiesToMessages()).hasSize(3) } } @Test - fun `init works find for null state`() { - val partitionState = PartitionState.fromState(3, null) + fun `it reads null state correctly`() { + val partitionState = fromState(3, null) assertSoftly { - it.assertThat(partitionState.restartOffset).isEqualTo(0) - it.assertThat(partitionState.lastSentOffset).isEqualTo(0) + it.assertThat(partitionState.readRecordsFromOffset).isEqualTo(0) + it.assertThat(partitionState.processRecordsFromOffset).isEqualTo(0) + it.assertThat(partitionState.counterpartiesToMessages()).hasSize(0) } } - @Nested - inner class FailedInitTests { - @Test - fun `init fails with invalid JSON`() { - val json = """{"restartOffset": 100000,""" - val state = State( - key = "", - value = json.toByteArray(), - version = 100, - ) - - assertThrows { - PartitionState.fromState(3, state) - } - } - - @Test - fun `init fails with invalid field`() { - val json = """ -{ - "partition": "1", -} - """.trimIndent() - val state = State( - key = "", + @Test + fun `it throws expcetion for invalid JSON`() { + val json = "{" + val state = + State( + key = stateKey(1), value = json.toByteArray(), - version = 100, ) - assertThrows { - PartitionState.fromState(3, state) - } - } + assertThatThrownBy { + fromState(3, state) + }.isInstanceOf(CordaRuntimeException::class.java) } } @Test - fun `addToOperationGroup add the correct state to the created group`() { - val partitionState = PartitionState.fromState( - 1, - null, - ) - partitionState.restartOffset = 100 - partitionState.lastSentOffset = 400 - partitionState.addMessage( - TrackedMessageState( - messageId = "id1", - timeStamp = Instant.ofEpochMilli(401), - persisted = false, - ), - ) - partitionState.addMessage( - TrackedMessageState( - messageId = "id2", - timeStamp = Instant.ofEpochMilli(402), - persisted = false, - ), - ) - val state = argumentCaptor() - val group = mock { - on { create(state.capture()) } doReturn mock - } - - partitionState.addToOperationGroup(group) + fun `readRecordsFromOffset can be updated`() { + val state = PartitionState(1) - assertSoftly { - it.assertThat(state.firstValue.version).isEqualTo(State.VERSION_INITIAL_VALUE) - it.assertThat(state.firstValue.key).isEqualTo(stateKey(1)) - val created = PartitionState.fromState(1, state.firstValue) - it.assertThat(created.restartOffset).isEqualTo(partitionState.restartOffset) - it.assertThat(created.lastSentOffset).isEqualTo(partitionState.lastSentOffset) - it.assertThat(created.getTrackMessage("id1")).isEqualTo(partitionState.getTrackMessage("id1")) - it.assertThat(created.getTrackMessage("id2")).isEqualTo(partitionState.getTrackMessage("id2")) - } - } + state.readRecordsFromOffset = 132 - @Test - fun `addToOperationGroup add the correct state to the updated group`() { - val partitionState = PartitionState.fromState( - 1, - null, - ) - partitionState.restartOffset = 100 - partitionState.lastSentOffset = 400 - partitionState.addMessage( - TrackedMessageState( - messageId = "id1", - timeStamp = Instant.ofEpochMilli(401), - persisted = false, - ), - ) - partitionState.addMessage( - TrackedMessageState( - messageId = "id2", - timeStamp = Instant.ofEpochMilli(402), - persisted = false, - ), - ) - val state = argumentCaptor() - val group = mock { - on { update(state.capture()) } doReturn mock - } - partitionState.saved() - - partitionState.addToOperationGroup(group) - - assertSoftly { - it.assertThat(state.firstValue.version).isEqualTo(State.VERSION_INITIAL_VALUE + 1) - it.assertThat(state.firstValue.key).isEqualTo(stateKey(1)) - val created = PartitionState.fromState(1, state.firstValue) - it.assertThat(created.restartOffset).isEqualTo(partitionState.restartOffset) - it.assertThat(created.lastSentOffset).isEqualTo(partitionState.lastSentOffset) - it.assertThat(created.getTrackMessage("id1")).isEqualTo(partitionState.getTrackMessage("id1")) - it.assertThat(created.getTrackMessage("id2")).isEqualTo(partitionState.getTrackMessage("id2")) - } + assertThat(state.readRecordsFromOffset).isEqualTo(132) } @Test - fun `read will update the offset`() { - val now = Instant.ofEpochMilli(1000) - val records = listOf( - EventLogRecord( - partition = 1, - offset = 200, - key = "", - value = mock(), - topic = "", - ), - EventLogRecord( - partition = 1, - offset = 201, - key = "", - value = mock(), - topic = "", - ), - EventLogRecord( - partition = 1, - offset = 202, - key = "", - value = mock(), - topic = "", - ), - EventLogRecord( - partition = 1, - offset = 100, - key = "", - value = mock(), - topic = "", - ), - ) + fun `processRecordsFromOffset can be updated`() { val state = PartitionState(1) - state.read(now, records) + state.processRecordsFromOffset = 132 - assertThat(state.lastSentOffset).isEqualTo(202) + assertThat(state.processRecordsFromOffset).isEqualTo(132) } @Test - fun `read will not update the offset if not more then the max`() { - val now = Instant.ofEpochMilli(1000) - val records = listOf( - EventLogRecord( - partition = 1, - offset = 200, - key = "", - value = mock(), - topic = "", - ), - EventLogRecord( - partition = 1, - offset = 201, - key = "", - value = mock(), - topic = "", - ), - EventLogRecord( - partition = 1, - offset = 202, - key = "", - value = mock(), - topic = "", - ), - EventLogRecord( - partition = 1, - offset = 100, - key = "", - value = mock(), - topic = "", - ), - ) + fun `counterpartiesToMessages returns all the messages`() { val state = PartitionState(1) - state.lastSentOffset = 1000 + state.read( + Instant.ofEpochMilli(100), + (1..4).flatMap { i -> + val source = "CN=Alice-$i, O=Alice Corp, L=LDN, C=GB" + (1..3).flatMap { j -> + val target = "CN=Bob-$j, O=Bob Corp, L=LDN, C=GB" + (1..2).map { k -> + MessageRecord( + offset = i * 1000L + j * 100 + k, + partition = 1, + message = createMessage( + id = "id-$i-$j-$k", + group = "group-$i", + from = source, + to = target, + ), + ) + } + } + }, + ) - state.read(now, records) + val counterpartiesToMessages = state.counterpartiesToMessages() - assertThat(state.lastSentOffset).isEqualTo(1000) + assertThat(counterpartiesToMessages) + .anyMatch { (counterParties, messages) -> + counterParties == SessionManager.Counterparties( + ourId = HoldingIdentity( + groupId = "group-1", + x500Name = MemberX500Name.parse("CN=Alice-1, O=Alice Corp, L=LDN, C=GB"), + ), + counterpartyId = HoldingIdentity( + groupId = "group-1", + x500Name = MemberX500Name.parse("CN=Bob-2, O=Bob Corp, L=LDN, C=GB"), + ), + ) && messages.contains( + TrackedMessageState(messageId = "id-1-2-1", timeStamp = Instant.ofEpochMilli(100)), + ) && messages.contains( + TrackedMessageState(messageId = "id-1-2-2", timeStamp = Instant.ofEpochMilli(100)), + ) && messages.size == 2 + } + .hasSize(12) } - @Test - fun `read will save the messages if the message is AuthenticatedMessage`() { - val now = Instant.ofEpochMilli(1000) - val messageHeader = mock { - on { messageId } doReturn "message ID" - } - val authenticatedMessage = mock { - on { header } doReturn messageHeader - } - val value = mock { - on { message } doReturn authenticatedMessage - } - val records = listOf( - EventLogRecord( - partition = 1, - offset = 200, - key = "a", - value = value, - topic = "", - ), - ) - val state = PartitionState(1) - state.lastSentOffset = 1000 + @Nested + inner class AddToOperationGroupTest { + @Test + fun `it create the correct JSON`() { + val state = PartitionState(1) + state.read( + Instant.ofEpochMilli(100), + (1..4).flatMap { i -> + val source = "CN=Alice-$i, O=Alice Corp, L=LDN, C=GB" + (1..3).flatMap { j -> + val target = "CN=Bob-$j, O=Bob Corp, L=LDN, C=GB" + (1..2).map { k -> + MessageRecord( + offset = i * 1000L + j * 100 + k, + partition = 1, + message = createMessage( + id = "id-$i-$j-$k", + group = "group-$i", + from = source, + to = target, + ), + ) + } + } + }, + ) + state.readRecordsFromOffset = 2003 + state.processRecordsFromOffset = 2001 + val captureState = argumentCaptor() + val group = mock { + on { create(captureState.capture()) } doReturn mock + } - state.read(now, records) + state.addToOperationGroup(group) - assertThat(state.getTrackMessage("message ID")).isEqualTo( - TrackedMessageState( - "message ID", - now, - false, - ), - ) - } + val savedState = fromState(1, captureState.firstValue) - @Test - fun `read will save the messages if the message is OutboundUnauthenticatedMessage`() { - val now = Instant.ofEpochMilli(1000) - val messageHeader = mock { - on { messageId } doReturn "message ID" - } - val outboundUnauthenticatedMessage = mock { - on { header } doReturn messageHeader + assertSoftly { softly -> + softly.assertThat(savedState.readRecordsFromOffset).isEqualTo(state.readRecordsFromOffset) + softly.assertThat(savedState.processRecordsFromOffset).isEqualTo(state.processRecordsFromOffset) + softly.assertThat( + savedState.counterpartiesToMessages() + .toMap() + .mapValues { + it.value.toSet() + }, + ) + .containsAllEntriesOf( + state.counterpartiesToMessages() + .toMap() + .mapValues { + it.value.toSet() + }, + ) + } } - val value = mock { - on { message } doReturn outboundUnauthenticatedMessage + + @Test + fun `it create a new state when needed`() { + val state = PartitionState(1) + val group = mock { + on { create(any()) } doReturn mock + } + + state.addToOperationGroup(group) + + verify(group).create(any()) } - val records = listOf( - EventLogRecord( - partition = 1, - offset = 200, - key = "a", - value = value, - topic = "", - ), - ) - val state = PartitionState(1) - state.lastSentOffset = 1000 - state.read(now, records) + @Test + fun `it update a state when needed`() { + val state = PartitionState(1) + state.saved() + val group = mock { + on { update(any()) } doReturn mock + } - assertThat(state.getTrackMessage("message ID")).isEqualTo( - TrackedMessageState( - "message ID", - now, - false, - ), - ) + state.addToOperationGroup(group) + + verify(group).update(any()) + } } @Test - fun `read will save the messages if the message is InboundUnauthenticatedMessage`() { - val now = Instant.ofEpochMilli(1000) - val messageHeader = mock { - on { messageId } doReturn "message ID" - } - val inboundUnauthenticatedMessage = mock { - on { header } doReturn messageHeader - } - val value = mock { - on { message } doReturn inboundUnauthenticatedMessage - } - val records = listOf( - EventLogRecord( - partition = 1, - offset = 200, - key = "a", - value = value, - topic = "", + fun `read return only the new messages`() { + val state = PartitionState(1) + state.read( + Instant.ofEpochMilli(100), + listOf( + MessageRecord( + offset = 3, + partition = 3, + message = createMessage( + id = "id-3", + group = "group-1", + from = "CN=Alice, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob, O=Bob Corp, L=LDN, C=GB", + ), + ), + MessageRecord( + offset = 4, + partition = 4, + message = createMessage( + id = "id-4", + group = "group-1", + from = "CN=Alice, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob, O=Bob Corp, L=LDN, C=GB", + ), + ), ), ) - val state = PartitionState(1) - state.lastSentOffset = 1000 - - state.read(now, records) - assertThat(state.getTrackMessage("message ID")).isEqualTo( - TrackedMessageState( - "message ID", - now, - false, + val newMessages = state.read( + Instant.ofEpochMilli(100), + listOf( + MessageRecord( + offset = 1, + partition = 1, + message = createMessage( + id = "id-1", + group = "group-1", + from = "CN=Alice, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob, O=Bob Corp, L=LDN, C=GB", + ), + ), + MessageRecord( + offset = 2, + partition = 2, + message = createMessage( + id = "id-2", + group = "group-1", + from = "CN=Alice, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob, O=Bob Corp, L=LDN, C=GB", + ), + ), + MessageRecord( + offset = 3, + partition = 3, + message = createMessage( + id = "id-3", + group = "group-1", + from = "CN=Alice, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob, O=Bob Corp, L=LDN, C=GB", + ), + ), + MessageRecord( + offset = 4, + partition = 4, + message = createMessage( + id = "id-4", + group = "group-1", + from = "CN=Alice, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob, O=Bob Corp, L=LDN, C=GB", + ), + ), ), ) + + assertThat(newMessages.map { it.offset }) + .contains( + 1, + 2, + ) + .hasSize(2) } @Test - fun `sent will update the offset`() { - val records = listOf( - EventLogRecord( - partition = 1, - offset = 200, - key = "", - value = mock(), - topic = "", - ), - EventLogRecord( - partition = 1, - offset = 201, - key = "", - value = mock(), - topic = "", - ), - EventLogRecord( - partition = 1, - offset = 202, - key = "", - value = mock(), - topic = "", - ), - EventLogRecord( - partition = 1, - offset = 100, - key = "", - value = mock(), - topic = "", + fun `forget will remove the messages`() { + val state = PartitionState(1) + val message = createMessage( + id = "id-3", + group = "group-1", + from = "CN=Alice, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob, O=Bob Corp, L=LDN, C=GB", + ) + state.read( + Instant.ofEpochMilli(100), + listOf( + MessageRecord( + offset = 3, + partition = 3, + message = message, + ), + MessageRecord( + offset = 4, + partition = 4, + message = createMessage( + id = "id-4", + group = "group-1", + from = "CN=Alice, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob, O=Bob Corp, L=LDN, C=GB", + ), + ), ), ) - val state = PartitionState(1) - state.sent(records) + state.forget(message) - assertThat(state.restartOffset).isEqualTo(202) + assertThat( + state.counterpartiesToMessages() + .flatMap { + it.second + }, + ).hasSize(1) } - @Test - fun `sent will not update the offset if not more then the max`() { - val records = listOf( - EventLogRecord( - partition = 1, - offset = 200, - key = "", - value = mock(), - topic = "", - ), - EventLogRecord( - partition = 1, - offset = 201, - key = "", - value = mock(), - topic = "", - ), - EventLogRecord( - partition = 1, - offset = 202, - key = "", - value = mock(), - topic = "", - ), - EventLogRecord( - partition = 1, - offset = 100, - key = "", - value = mock(), - topic = "", - ), - ) - val state = PartitionState(1) - state.restartOffset = 1000 - - state.sent(records) + private fun createMessage( + id: String, + group: String, + from: String, + to: String, + ): AuthenticatedMessage { + val headers = mock { + on { messageId } doReturn id + on { source } doReturn AvroHoldingIdentity( + from, + group, + ) + on { destination } doReturn AvroHoldingIdentity( + to, + group, + ) + } - assertThat(state.restartOffset).isEqualTo(1000) + return mock { + on { header } doReturn headers + } } } diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStatesTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStatesTest.kt deleted file mode 100644 index 7dfc7fa8a42..00000000000 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStatesTest.kt +++ /dev/null @@ -1,543 +0,0 @@ -package net.corda.p2p.linkmanager.tracker - -import net.corda.data.p2p.app.AppMessage -import net.corda.libs.statemanager.api.State -import net.corda.libs.statemanager.api.StateManager -import net.corda.libs.statemanager.api.StateOperationGroup -import net.corda.lifecycle.ErrorEvent -import net.corda.lifecycle.LifecycleCoordinator -import net.corda.lifecycle.LifecycleCoordinatorFactory -import net.corda.lifecycle.LifecycleEvent -import net.corda.lifecycle.LifecycleEventHandler -import net.corda.lifecycle.LifecycleStatus -import net.corda.lifecycle.RegistrationHandle -import net.corda.lifecycle.RegistrationStatusChangeEvent -import net.corda.lifecycle.domino.logic.ComplexDominoTile -import net.corda.messaging.api.records.EventLogRecord -import net.corda.p2p.linkmanager.tracker.PartitionState.Companion.stateKey -import net.corda.utilities.time.Clock -import net.corda.v5.base.exceptions.CordaRuntimeException -import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.SoftAssertions.assertSoftly -import org.junit.jupiter.api.Nested -import org.junit.jupiter.api.Test -import org.mockito.ArgumentMatchers.eq -import org.mockito.Mockito.times -import org.mockito.kotlin.any -import org.mockito.kotlin.argThat -import org.mockito.kotlin.argumentCaptor -import org.mockito.kotlin.doAnswer -import org.mockito.kotlin.doReturn -import org.mockito.kotlin.doThrow -import org.mockito.kotlin.mock -import org.mockito.kotlin.never -import org.mockito.kotlin.verify -import org.mockito.kotlin.whenever -import java.time.Instant -import java.util.concurrent.ScheduledExecutorService -import java.util.concurrent.ScheduledFuture -import java.util.concurrent.TimeUnit - -class PartitionsStatesTest { - private val handler = argumentCaptor() - private val followStatusChangesByNameHandlers = mutableSetOf() - private val coordinator = mock { - on { followStatusChangesByName(any()) } doAnswer { - val handler = mock() - followStatusChangesByNameHandlers.add(handler) - handler - } - } - private val coordinatorFactory = mock { - on { createCoordinator(any(), handler.capture()) } doReturn coordinator - } - private val operationGroup = mock { - on { execute() } doReturn emptyMap() - } - private val stateManager = mock { - on { name } doReturn mock() - on { createOperationGroup() } doReturn operationGroup - } - private val configDominoTile = mock { - on { coordinatorName } doReturn mock() - } - private val config = mock { - on { dominoTile } doReturn configDominoTile - on { config } doReturn DeliveryTrackerConfiguration.Configuration( - maxCacheSizeMegabytes = 100, - maxCacheOffsetAge = 50000, - statePersistencePeriodSeconds = 1.0, - outboundBatchProcessingTimeoutSeconds = 30.0, - maxNumberOfPersistenceRetries = 3, - ) - } - private val ends = mock() - private val now = mock { - on { plusMillis(any()) } doReturn ends - } - private val clock = mock { - on { instant() } doReturn now - } - private val future = mock>() - private val persist = argumentCaptor() - private val executor = mock { - on { - scheduleAtFixedRate( - persist.capture(), - any(), - any(), - any(), - ) - } doReturn future - } - - private val states = PartitionsStates( - coordinatorFactory, - stateManager, - config, - clock, - executor, - ) - - @Test - fun `onStart will listen to configuration changes`() { - followStatusChangesByNameHandlers.forEach { followStatusChangesByNameHandler -> - handler.firstValue.processEvent( - RegistrationStatusChangeEvent( - followStatusChangesByNameHandler, - LifecycleStatus.UP, - ), - coordinator, - ) - } - - verify(config).lister(states) - } - - @Test - fun `onStart will start the task`() { - followStatusChangesByNameHandlers.forEach { followStatusChangesByNameHandler -> - handler.firstValue.processEvent( - RegistrationStatusChangeEvent( - followStatusChangesByNameHandler, - LifecycleStatus.UP, - ), - coordinator, - ) - } - - verify(executor).scheduleAtFixedRate( - any(), - eq(1000L), - eq(1000L), - eq(TimeUnit.MILLISECONDS), - ) - } - - @Test - fun `onStop will cancel the task`() { - followStatusChangesByNameHandlers.forEach { followStatusChangesByNameHandler -> - handler.firstValue.processEvent( - RegistrationStatusChangeEvent( - followStatusChangesByNameHandler, - LifecycleStatus.UP, - ), - coordinator, - ) - } - - states.close() - - verify(future).cancel(false) - } - - @Test - fun `onStop will not cancel anything if not started before`() { - states.close() - - verify(future, never()).cancel(false) - } - - @Test - fun `loadPartitions will create new partition if partition is not in the state manager`() { - states.loadPartitions( - setOf(1, 3), - ) - - val partition = states.get(3) - - assertThat(partition?.restartOffset).isEqualTo(0) - } - - @Test - fun `loadPartitions will read partition from the state manager`() { - val key = stateKey(3) - val json = """ - { - "restartOffset": 301, - "lastSentOffset": 403, - "partition": 3, - "messages": {} - } - """.trimIndent() - whenever( - stateManager.get( - argThat { - contains(key) - }, - ), - ).doReturn( - mapOf( - key to State( - key = key, - value = json.toByteArray(), - ), - ), - ) - states.loadPartitions( - setOf( - 1, - 3, - ), - ) - - val partition = states.get(3) - - assertThat(partition?.restartOffset).isEqualTo(301) - } - - @Test - fun `forgetPartitions will forget the partition details`() { - states.loadPartitions( - setOf(1, 3), - ) - states.forgetPartitions( - setOf(2, 3), - ) - - val partition = states.get(3) - - assertThat(partition).isNull() - } - - @Test - fun `read will update the offsets`() { - states.loadPartitions( - setOf( - 1, - 3, - ), - ) - val records: List> = listOf( - EventLogRecord( - topic = "topic", - key = "key", - value = null, - partition = 1, - offset = 100, - ), - EventLogRecord( - topic = "topic", - key = "key", - value = null, - partition = 1, - offset = 200, - ), - EventLogRecord( - topic = "topic", - key = "key", - value = null, - partition = 1, - offset = 101, - ), - EventLogRecord( - topic = "topic", - key = "key", - value = null, - partition = 3, - offset = 303, - ), - EventLogRecord( - topic = "topic", - key = "key", - value = null, - partition = 1, - offset = 102, - ), - ) - - states.read(records) - - assertSoftly { - val partitionOne = states.get(1) - it.assertThat(partitionOne?.lastSentOffset).isEqualTo(200) - val partitionThree = states.get(3) - it.assertThat(partitionThree?.lastSentOffset).isEqualTo(303) - } - } - - @Test - fun `sent will update the offsets`() { - states.loadPartitions( - setOf( - 1, - 3, - ), - ) - val records = listOf( - EventLogRecord( - topic = "topic", - key = "key", - value = null, - partition = 1, - offset = 100, - ), - EventLogRecord( - topic = "topic", - key = "key", - value = null, - partition = 1, - offset = 200, - ), - EventLogRecord( - topic = "topic", - key = "key", - value = null, - partition = 1, - offset = 101, - ), - EventLogRecord( - topic = "topic", - key = "key", - value = null, - partition = 3, - offset = 303, - ), - EventLogRecord( - topic = "topic", - key = "key", - value = null, - partition = 1, - offset = 102, - ), - ) - - states.sent(records) - - assertSoftly { - val partitionOne = states.get(1) - it.assertThat(partitionOne?.restartOffset).isEqualTo(200) - val partitionThree = states.get(3) - it.assertThat(partitionThree?.restartOffset).isEqualTo(303) - } - } - - @Test - fun `changed will recreate the task`() { - states.changed() - states.changed() - - verify(executor, times(2)).scheduleAtFixedRate( - any(), - eq(1000L), - eq(1000L), - eq(TimeUnit.MILLISECONDS), - ) - } - - @Test - fun `changed will cancel the previous task`() { - states.changed() - states.changed() - - verify(future, times(1)).cancel(false) - } - - @Nested - inner class PersistTests { - @Test - fun `persist will persist the data`() { - states.changed() - val key = stateKey(3) - val json = """ - { - "restartOffset": 301, - "lastSentOffset": 403, - "partition": 3, - "version": 12, - "messages": {} - } - """.trimIndent() - whenever( - stateManager.get( - argThat { - contains(key) - }, - ), - ).doReturn( - mapOf( - key to State( - key = key, - value = json.toByteArray(), - version = 12, - ), - ), - ) - states.loadPartitions( - setOf(1, 3), - ) - val records: List> = listOf( - EventLogRecord( - topic = "topic", - key = "key", - value = null, - partition = 1, - offset = 101, - ), - EventLogRecord( - topic = "topic", - key = "key", - value = null, - partition = 3, - offset = 3003, - ), - ) - states.read(records) - - persist.firstValue.run() - - verify(operationGroup).execute() - verify(operationGroup, times(1)).create(any()) - verify(operationGroup, times(1)).update(any()) - } - - @Test - fun `persist will not run anything if not needed`() { - states.changed() - - persist.firstValue.run() - - verify(operationGroup, never()).execute() - } - - @Test - fun `persist failure will set as error if the state manager is out of sync`() { - whenever(operationGroup.execute()).doReturn(mapOf(stateKey(3) to mock())) - val events = argumentCaptor() - whenever(coordinator.postEvent(events.capture())).doAnswer { } - states.changed() - states.loadPartitions( - setOf( - 1, - 3, - ), - ) - val records: List> = listOf( - EventLogRecord( - topic = "topic", - key = "key", - value = null, - partition = 1, - offset = 101, - ), - EventLogRecord( - topic = "topic", - key = "key", - value = null, - partition = 3, - offset = 3003, - ), - ) - states.read(records) - - persist.firstValue.run() - - val error = (events.firstValue as? ErrorEvent)?.cause - assertThat(error).isExactlyInstanceOf(IllegalStateException::class.java) - } - - @Test - fun `persist exceptional failure will not post an error for the first time`() { - whenever(operationGroup.execute()) - .doThrow(CordaRuntimeException("Ooops")) - states.changed() - states.loadPartitions( - setOf( - 1, - 3, - ), - ) - - persist.firstValue.run() - - verify(coordinator, never()).postEvent(any()) - } - - @Test - fun `persist exceptional failure will post an error after the Nth time`() { - val e = CordaRuntimeException("Ooops") - whenever(operationGroup.execute()) - .doThrow(e) - val events = argumentCaptor() - whenever(coordinator.postEvent(events.capture())).doAnswer { } - states.changed() - states.loadPartitions( - setOf( - 1, - 3, - ), - ) - - persist.firstValue.run() - persist.firstValue.run() - persist.firstValue.run() - - val error = (events.firstValue as? ErrorEvent)?.cause - assertThat(error).isSameAs(e) - } - - @Test - fun `persist exceptional failure will not post an error after the Nth time if it had any success in between`() { - whenever(operationGroup.execute()) - .thenThrow(CordaRuntimeException("Ooops")) - .thenThrow(CordaRuntimeException("Ooops")) - .thenReturn(emptyMap()) - .thenThrow(CordaRuntimeException("Ooops")) - .thenThrow(CordaRuntimeException("Ooops")) - .thenReturn(emptyMap()) - states.changed() - states.loadPartitions( - setOf( - 1, - 3, - ), - ) - - persist.firstValue.run() - persist.firstValue.run() - persist.firstValue.run() - persist.firstValue.run() - persist.firstValue.run() - - verify(coordinator, never()).postEvent(any()) - } - - @Test - fun `persist will update the state`() { - states.changed() - states.loadPartitions( - setOf( - 1, - 3, - ), - ) - - persist.firstValue.run() - - val info = states.get(3) - val group = mock() - info?.addToOperationGroup(group) - verify(group).update(any()) - } - } -} diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStatesTests.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStatesTests.kt new file mode 100644 index 00000000000..4fe450e8d39 --- /dev/null +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStatesTests.kt @@ -0,0 +1,866 @@ +package net.corda.p2p.linkmanager.tracker + +import net.corda.data.p2p.app.AppMessage +import net.corda.data.p2p.app.AuthenticatedMessage +import net.corda.data.p2p.app.AuthenticatedMessageHeader +import net.corda.libs.statemanager.api.State +import net.corda.libs.statemanager.api.StateManager +import net.corda.libs.statemanager.api.StateOperationGroup +import net.corda.lifecycle.ErrorEvent +import net.corda.lifecycle.LifecycleCoordinator +import net.corda.lifecycle.LifecycleCoordinatorFactory +import net.corda.lifecycle.LifecycleEvent +import net.corda.lifecycle.LifecycleEventHandler +import net.corda.lifecycle.LifecycleStatus +import net.corda.lifecycle.RegistrationHandle +import net.corda.lifecycle.RegistrationStatusChangeEvent +import net.corda.lifecycle.domino.logic.ComplexDominoTile +import net.corda.messaging.api.records.EventLogRecord +import net.corda.p2p.linkmanager.delivery.ReplayScheduler +import net.corda.p2p.linkmanager.sessions.SessionManager +import net.corda.p2p.linkmanager.tracker.PartitionState.Companion.stateKey +import net.corda.utilities.time.Clock +import net.corda.v5.base.exceptions.CordaRuntimeException +import net.corda.v5.base.types.MemberX500Name +import net.corda.virtualnode.HoldingIdentity +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatCode +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.Test +import org.mockito.ArgumentMatchers.eq +import org.mockito.kotlin.any +import org.mockito.kotlin.argThat +import org.mockito.kotlin.argumentCaptor +import org.mockito.kotlin.doAnswer +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.doThrow +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.times +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import java.time.Instant +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit +import net.corda.data.identity.HoldingIdentity as AvroHoldingIdentity + +class PartitionsStatesTests { + private companion object { + const val STATE_JSON = """ +{ + "processRecordsFromOffset": 300, + "readRecordsFromOffset": 204, + "messages": { + "group-2": { + "CN=Alice, O=Alice Corp, L=LDN, C=GB": { + "CN=Bob, O=Bob Corp, L=LDN, C=GB": { + "id-2": { + "id": "id-2", + "ts": 10.231 + } + } + } + }, + "group": { + "CN=Alice, O=Alice Corp, L=LDN, C=GB": { + "CN=Carol, O=Carol Corp, L=LDN, C=GB": { + "id-3": { + "id": "id-3", + "ts": 10.231 + } + }, + "CN=Bob, O=Bob Corp, L=LDN, C=GB": { + "id-1": { + "id": "id-1", + "ts": 10.231 + } + } + } + } + }, + "version": 0 +} + """ + private fun createMessage( + id: String, + group: String, + from: String, + to: String, + ): AuthenticatedMessage { + val headers = mock { + on { messageId } doReturn id + on { source } doReturn AvroHoldingIdentity( + from, + group, + ) + on { destination } doReturn AvroHoldingIdentity( + to, + group, + ) + } + + return mock { + on { header } doReturn headers + } + } + } + private val partitionsIndices = setOf( + 1, + 5, + 12, + ) + private val savedState = + State( + key = stateKey(1), + value = STATE_JSON.toByteArray(), + ) + + private val handler = argumentCaptor() + private val followStatusChangesByNameHandlers = mutableSetOf() + private val coordinator = mock { + on { followStatusChangesByName(any()) } doAnswer { + val handler = mock() + followStatusChangesByNameHandlers.add(handler) + handler + } + } + private val coordinatorFactory = mock { + on { createCoordinator(any(), handler.capture()) } doReturn coordinator + } + private val operationGroup = mock { + on { execute() } doReturn emptyMap() + } + private val stateManager = mock { + on { createOperationGroup() } doReturn operationGroup + on { + get( + partitionsIndices.map { + stateKey(it) + }, + ) + } doReturn mapOf(stateKey(5) to savedState) + + on { name } doReturn mock() + } + private val configDominoTile = mock { + on { coordinatorName } doReturn mock() + } + private val config = mock { + on { dominoTile } doReturn configDominoTile + on { config } doReturn DeliveryTrackerConfiguration.Configuration( + maxCacheOffsetAge = 50000, + statePersistencePeriodSeconds = 1.0, + outboundBatchProcessingTimeoutSeconds = 30.0, + maxNumberOfPersistenceRetries = 3, + ) + } + private val now = Instant.ofEpochMilli(2001) + private val clock = mock { + on { instant() } doReturn now + } + private val replayScheduler = mock>() + private val future = mock>() + private val persist = argumentCaptor() + private val executor = mock { + on { + scheduleAtFixedRate( + persist.capture(), + any(), + any(), + any(), + ) + } doReturn future + } + + private val partitionsStates = PartitionsStates( + coordinatorFactory, + stateManager, + config, + clock, + replayScheduler, + executor, + ) + + @Test + fun `getEventToProcess return the events to process`() { + partitionsStates.loadPartitions(partitionsIndices) + val records = listOf( + EventLogRecord( + topic = "topic", + key = "key1", + value = mock { }, + offset = 100, + partition = 5, + ), + EventLogRecord( + topic = "topic", + key = "key1", + value = mock { }, + offset = 299, + partition = 5, + ), + EventLogRecord( + topic = "topic", + key = "key1", + value = mock { }, + offset = 300, + partition = 5, + ), + EventLogRecord( + topic = "topic", + key = "key1", + value = mock { }, + offset = 301, + partition = 5, + ), + EventLogRecord( + topic = "topic", + key = "key1", + value = mock { }, + offset = 302, + partition = 5, + ), + EventLogRecord( + topic = "topic", + key = "key1", + value = mock { }, + offset = 12, + partition = 1, + ), + EventLogRecord( + topic = "topic", + key = "key1", + value = mock { }, + offset = 13, + partition = 1, + ), + ) + + val returnedRecords = partitionsStates.getEventToProcess(records) + + val partitionsAndOffset = returnedRecords.map { + it.partition to it.offset + } + assertThat(partitionsAndOffset) + .hasSize(4) + .contains(5 to 301) + .contains(5 to 302) + .contains(1 to 12) + .contains(1 to 13) + } + + @Test + fun `read will replay the messages`() { + partitionsStates.loadPartitions(partitionsIndices) + val records = listOf( + MessageRecord( + message = createMessage( + id = "id-2", + from = "CN=Alice, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob, O=Bob Corp, L=LDN, C=GB", + group = "group-2", + ), + offset = 400, + partition = 5, + ), + MessageRecord( + message = createMessage( + id = "id-12", + from = "CN=Alice, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob, O=Bob Corp, L=LDN, C=GB", + group = "group-2", + ), + offset = 404, + partition = 5, + ), + MessageRecord( + message = createMessage( + id = "id-22", + from = "CN=Alice, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob, O=Bob Corp, L=LDN, C=GB", + group = "group-3", + ), + offset = 500, + partition = 12, + ), + ) + + partitionsStates.read(records) + + verify(replayScheduler).addForReplay( + originalAttemptTimestamp = now.toEpochMilli(), + messageId = "id-12", + message = "id-12", + counterparties = SessionManager.Counterparties( + ourId = HoldingIdentity( + groupId = "group-2", + x500Name = MemberX500Name.parse("CN=Alice, O=Alice Corp, L=LDN, C=GB"), + ), + counterpartyId = HoldingIdentity( + groupId = "group-2", + x500Name = MemberX500Name.parse("CN=Bob, O=Bob Corp, L=LDN, C=GB"), + ), + ), + ) + verify(replayScheduler).addForReplay( + originalAttemptTimestamp = now.toEpochMilli(), + messageId = "id-22", + message = "id-22", + counterparties = SessionManager.Counterparties( + ourId = HoldingIdentity( + groupId = "group-3", + x500Name = MemberX500Name.parse("CN=Alice, O=Alice Corp, L=LDN, C=GB"), + ), + counterpartyId = HoldingIdentity( + groupId = "group-3", + x500Name = MemberX500Name.parse("CN=Bob, O=Bob Corp, L=LDN, C=GB"), + ), + ), + ) + } + + @Test + fun `read will not replay messages that had been replayed`() { + partitionsStates.loadPartitions(partitionsIndices) + + val records = listOf( + MessageRecord( + message = createMessage( + id = "id-2", + from = "CN=Alice, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob, O=Bob Corp, L=LDN, C=GB", + group = "group-2", + ), + offset = 400, + partition = 5, + ), + ) + + partitionsStates.read(records) + + verify(replayScheduler, never()).addForReplay( + originalAttemptTimestamp = now.toEpochMilli(), + messageId = "id-2", + message = "id-2", + counterparties = SessionManager.Counterparties( + ourId = HoldingIdentity( + groupId = "group-2", + x500Name = MemberX500Name.parse("CN=Alice, O=Alice Corp, L=LDN, C=GB"), + ), + counterpartyId = HoldingIdentity( + groupId = "group-2", + x500Name = MemberX500Name.parse("CN=Bob, O=Bob Corp, L=LDN, C=GB"), + ), + ), + ) + } + + @Test + fun `forget will remove the message from the state`() { + partitionsStates.loadPartitions(partitionsIndices) + + partitionsStates.forget( + MessageRecord( + partition = 5, + offset = 1002, + message = createMessage( + id = "id-2", + from = "CN=Alice, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob, O=Bob Corp, L=LDN, C=GB", + group = "group-2", + ), + ), + ) + + partitionsStates.read( + listOf( + MessageRecord( + message = createMessage( + id = "id-2", + from = "CN=Alice, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob, O=Bob Corp, L=LDN, C=GB", + group = "group-2", + ), + offset = 400, + partition = 5, + ), + ), + ) + verify(replayScheduler).addForReplay( + originalAttemptTimestamp = now.toEpochMilli(), + messageId = "id-2", + message = "id-2", + counterparties = SessionManager.Counterparties( + ourId = HoldingIdentity( + groupId = "group-2", + x500Name = MemberX500Name.parse("CN=Alice, O=Alice Corp, L=LDN, C=GB"), + ), + counterpartyId = HoldingIdentity( + groupId = "group-2", + x500Name = MemberX500Name.parse("CN=Bob, O=Bob Corp, L=LDN, C=GB"), + ), + ), + ) + } + + @Test + fun `forget will not throw an exception for unknown message partition`() { + partitionsStates.loadPartitions(partitionsIndices) + + assertThatCode { + partitionsStates.forget( + MessageRecord( + partition = 105, + offset = 1002, + message = createMessage( + id = "id-2", + from = "CN=Alice, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob, O=Bob Corp, L=LDN, C=GB", + group = "group-2", + ), + ), + ) + }.doesNotThrowAnyException() + } + + @Test + fun `onStart will listen to configuration changes`() { + followStatusChangesByNameHandlers.forEach { followStatusChangesByNameHandler -> + handler.firstValue.processEvent( + RegistrationStatusChangeEvent( + followStatusChangesByNameHandler, + LifecycleStatus.UP, + ), + coordinator, + ) + } + + verify(config).lister(partitionsStates) + } + + @Test + fun `onStart will start the task`() { + followStatusChangesByNameHandlers.forEach { followStatusChangesByNameHandler -> + handler.firstValue.processEvent( + RegistrationStatusChangeEvent( + followStatusChangesByNameHandler, + LifecycleStatus.UP, + ), + coordinator, + ) + } + + verify(executor).scheduleAtFixedRate( + any(), + eq(1000L), + eq(1000L), + eq(TimeUnit.MILLISECONDS), + ) + } + + @Test + fun `onStop will cancel the task`() { + followStatusChangesByNameHandlers.forEach { followStatusChangesByNameHandler -> + handler.firstValue.processEvent( + RegistrationStatusChangeEvent( + followStatusChangesByNameHandler, + LifecycleStatus.UP, + ), + coordinator, + ) + } + + partitionsStates.close() + + verify(future).cancel(false) + } + + @Test + fun `onStop will not cancel anything if not started before`() { + partitionsStates.close() + + verify(future, never()).cancel(false) + } + + @Test + fun `changed will recreate the task`() { + partitionsStates.changed() + partitionsStates.changed() + + verify(executor, times(2)).scheduleAtFixedRate( + any(), + eq(1000L), + eq(1000L), + eq(TimeUnit.MILLISECONDS), + ) + } + + @Test + fun `changed will cancel the previous task`() { + partitionsStates.changed() + partitionsStates.changed() + + verify(future, times(1)).cancel(false) + } + + @Nested + inner class PersistTests { + @Test + fun `persist will persist the data`() { + partitionsStates.changed() + val key = stateKey(3) + val json = """ + { + "processRecordsFromOffset": 300, + "readRecordsFromOffset": 204, + "messages": {}, + "version": 1 + } + """.trimIndent() + whenever( + stateManager.get( + argThat { + contains(key) + }, + ), + ).doReturn( + mapOf( + key to State( + key = key, + value = json.toByteArray(), + version = 12, + ), + ), + ) + partitionsStates.loadPartitions( + setOf(1, 3), + ) + val records = listOf( + MessageRecord( + message = createMessage( + id = "id-1", + from = "CN=Alice-1, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob-1, O=Bob Corp, L=LDN, C=GB", + group = "group-1", + ), + partition = 1, + offset = 101, + ), + MessageRecord( + message = createMessage( + id = "id-3", + from = "CN=Alice-3, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob-3, O=Bob Corp, L=LDN, C=GB", + group = "group-3", + ), + partition = 3, + offset = 3003, + ), + ) + partitionsStates.read(records) + + persist.firstValue.run() + + verify(operationGroup).execute() + verify(operationGroup, times(1)).create(any()) + verify(operationGroup, times(1)).update(any()) + } + + @Test + fun `persist will not run anything if not needed`() { + partitionsStates.changed() + + persist.firstValue.run() + + verify(operationGroup, never()).execute() + } + + @Test + fun `persist failure will set as error if the state manager is out of sync`() { + whenever(operationGroup.execute()).doReturn(mapOf(stateKey(3) to mock())) + val events = argumentCaptor() + whenever(coordinator.postEvent(events.capture())).doAnswer { } + partitionsStates.changed() + partitionsStates.loadPartitions( + setOf( + 1, + 3, + ), + ) + val records = listOf( + MessageRecord( + message = createMessage( + id = "id-1", + from = "CN=Alice-1, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob-1, O=Bob Corp, L=LDN, C=GB", + group = "group-1", + ), + partition = 1, + offset = 101, + ), + MessageRecord( + message = createMessage( + id = "id-3", + from = "CN=Alice-3, O=Alice Corp, L=LDN, C=GB", + to = "CN=Bob-3, O=Bob Corp, L=LDN, C=GB", + group = "group-3", + ), + partition = 3, + offset = 3003, + ), + ) + partitionsStates.read(records) + + persist.firstValue.run() + + val error = (events.firstValue as? ErrorEvent)?.cause + assertThat(error).isExactlyInstanceOf(IllegalStateException::class.java) + } + + @Test + fun `persist exceptional failure will not post an error for the first time`() { + whenever(operationGroup.execute()) + .doThrow(CordaRuntimeException("Ooops")) + partitionsStates.changed() + partitionsStates.loadPartitions( + setOf( + 1, + 3, + ), + ) + + persist.firstValue.run() + + verify(coordinator, never()).postEvent(any()) + } + + @Test + fun `persist exceptional failure will post an error after the Nth time`() { + val e = CordaRuntimeException("Ooops") + whenever(operationGroup.execute()) + .doThrow(e) + val events = argumentCaptor() + whenever(coordinator.postEvent(events.capture())).doAnswer { } + partitionsStates.changed() + partitionsStates.loadPartitions( + setOf( + 1, + 3, + ), + ) + + persist.firstValue.run() + persist.firstValue.run() + persist.firstValue.run() + + val error = (events.firstValue as? ErrorEvent)?.cause + assertThat(error).isSameAs(e) + } + + @Test + fun `persist exceptional failure will not post an error after the Nth time if it had any success in between`() { + whenever(operationGroup.execute()) + .thenThrow(CordaRuntimeException("Ooops")) + .thenThrow(CordaRuntimeException("Ooops")) + .thenReturn(emptyMap()) + .thenThrow(CordaRuntimeException("Ooops")) + .thenThrow(CordaRuntimeException("Ooops")) + .thenReturn(emptyMap()) + partitionsStates.changed() + partitionsStates.loadPartitions( + setOf( + 1, + 3, + ), + ) + + persist.firstValue.run() + persist.firstValue.run() + persist.firstValue.run() + persist.firstValue.run() + persist.firstValue.run() + + verify(coordinator, never()).postEvent(any()) + } + } + + @Test + fun `forgetPartitions forget the partitions`() { + partitionsStates.changed() + partitionsStates.loadPartitions( + setOf( + 1, + 2, + 3, + 4, + ), + ) + + partitionsStates.forgetPartitions( + setOf( + 2, + 4, + ), + ) + + persist.firstValue.run() + + verify(operationGroup, times(2)).create(any()) + } + + @Test + fun `loadPartitions return the states`() { + val states = partitionsStates.loadPartitions(partitionsIndices) + + assertThat(states) + .hasSize(3) + .anySatisfy { index, state -> + assertThat(index) + .isEqualTo(1) + assertThat(state.readRecordsFromOffset).isEqualTo(0) + } + .anySatisfy { index, state -> + assertThat(index) + .isEqualTo(12) + assertThat(state.readRecordsFromOffset).isEqualTo(0) + } + .anySatisfy { index, state -> + assertThat(index) + .isEqualTo(5) + assertThat(state.readRecordsFromOffset).isEqualTo(204) + } + } + + @Test + fun `loadPartitions will replay messages`() { + partitionsStates.loadPartitions(partitionsIndices) + + verify(replayScheduler).addForReplay( + originalAttemptTimestamp = 10231, + message = "id-2", + messageId = "id-2", + counterparties = SessionManager.Counterparties( + ourId = HoldingIdentity( + x500Name = MemberX500Name.parse("CN=Alice, O=Alice Corp, L=LDN, C=GB"), + groupId = "group-2", + ), + counterpartyId = HoldingIdentity( + x500Name = MemberX500Name.parse("CN=Bob, O=Bob Corp, L=LDN, C=GB"), + groupId = "group-2", + ), + ), + ) + verify(replayScheduler).addForReplay( + originalAttemptTimestamp = 10231, + message = "id-3", + messageId = "id-3", + counterparties = SessionManager.Counterparties( + ourId = HoldingIdentity( + x500Name = MemberX500Name.parse("CN=Alice, O=Alice Corp, L=LDN, C=GB"), + groupId = "group", + ), + counterpartyId = HoldingIdentity( + x500Name = MemberX500Name.parse("CN=Carol, O=Carol Corp, L=LDN, C=GB"), + groupId = "group", + ), + ), + ) + verify(replayScheduler).addForReplay( + originalAttemptTimestamp = 10231, + message = "id-1", + messageId = "id-1", + counterparties = SessionManager.Counterparties( + ourId = HoldingIdentity( + x500Name = MemberX500Name.parse("CN=Alice, O=Alice Corp, L=LDN, C=GB"), + groupId = "group", + ), + counterpartyId = HoldingIdentity( + x500Name = MemberX500Name.parse("CN=Bob, O=Bob Corp, L=LDN, C=GB"), + groupId = "group", + ), + ), + ) + } + + @Test + fun `loadPartitions will not throw an exception for empty list`() { + assertThatCode { + partitionsStates.loadPartitions(emptySet()) + }.doesNotThrowAnyException() + } + + @Test + fun `offsetsToReadFromChanged update the states`() { + partitionsStates.loadPartitions(partitionsIndices) + + partitionsStates.offsetsToReadFromChanged( + listOf( + 1 to 1000L, + 5 to 3000L, + 7 to 7000L, + ), + ) + + val states = partitionsStates.loadPartitions(partitionsIndices) + + assertThat(states[1]?.readRecordsFromOffset).isEqualTo(1000L) + } + + @Test + fun `handled update the states`() { + partitionsStates.loadPartitions(partitionsIndices) + val records = listOf( + EventLogRecord( + topic = "topic", + key = "key1", + value = mock { }, + offset = 1000L, + partition = 5, + ), + EventLogRecord( + topic = "topic", + key = "key1", + value = mock { }, + offset = 1001L, + partition = 5, + ), + EventLogRecord( + topic = "topic", + key = "key1", + value = mock { }, + offset = 999L, + partition = 5, + ), + EventLogRecord( + topic = "topic", + key = "key1", + value = mock { }, + offset = 301L, + partition = 5, + ), + EventLogRecord( + topic = "topic", + key = "key1", + value = mock { }, + offset = 13L, + partition = 1, + ), + EventLogRecord( + topic = "topic", + key = "key1", + value = mock { }, + offset = 24L, + partition = 100, + ), + ) + + partitionsStates.handled( + records, + ) + + val states = partitionsStates.loadPartitions(partitionsIndices) + + assertThat(states[5]?.processRecordsFromOffset).isEqualTo(1001L) + } +} diff --git a/gradle.properties b/gradle.properties index 8738d741411..5cf8be1f37c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -39,7 +39,7 @@ commonsLangVersion = 3.12.0 commonsTextVersion = 1.10.0 # Corda API libs revision (change in 4th digit indicates a breaking change) # Change to 5.3.0.xx-SNAPSHOT to pick up maven local published copy -cordaApiVersion=5.3.0.12-beta+ +cordaApiVersion=5.3.0.212-alpha-1713800316025 disruptorVersion=3.4.4 felixConfigAdminVersion=1.9.26 diff --git a/libs/configuration/configuration-schema/p2p/src/main/kotlin/net/corda/libs/configuration/schema/p2p/LinkManagerConfiguration.kt b/libs/configuration/configuration-schema/p2p/src/main/kotlin/net/corda/libs/configuration/schema/p2p/LinkManagerConfiguration.kt index 98048188948..8a0d9cb6be1 100644 --- a/libs/configuration/configuration-schema/p2p/src/main/kotlin/net/corda/libs/configuration/schema/p2p/LinkManagerConfiguration.kt +++ b/libs/configuration/configuration-schema/p2p/src/main/kotlin/net/corda/libs/configuration/schema/p2p/LinkManagerConfiguration.kt @@ -13,7 +13,6 @@ class LinkManagerConfiguration { const val REVOCATION_CHECK_KEY = "revocationCheck.mode" const val INBOUND_SESSIONS_CACHE_SIZE = "sessionCache.inboundSessionsCacheSize" const val OUTBOUND_SESSIONS_CACHE_SIZE = "sessionCache.outboundSessionsCacheSize" - const val DELIVERY_TRACKER_MAX_CACHE_SIZE_MEGABYTES = "deliveryTracker.maxCacheSizeMegabytes" const val DELIVERY_TRACKER_MAX_CACHE_OFFSET_AGE = "deliveryTracker.maxCacheOffsetAge" const val DELIVERY_TRACKER_STATE_PERSISTENCE_PERIOD_SECONDS = "deliveryTracker.statePersistencePeriodSeconds" const val DELIVERY_TRACKER_OUTBOUND_BATCH_PROCESSING_TIMEOUT_SECONDS = "deliveryTracker.outboundBatchProcessingTimeoutSeconds" From f93f6450c2106961ffd7f75c4178de72d360cf84 Mon Sep 17 00:00:00 2001 From: Yiftach Kaplan Date: Tue, 23 Apr 2024 09:49:40 +0100 Subject: [PATCH 2/5] Fix issues --- .../inbound/InboundMessageProcessor.kt | 31 ++++++++++--------- .../p2p/linkmanager/tracker/PartitionState.kt | 4 +-- .../linkmanager/tracker/PartitionsStates.kt | 2 +- .../net/corda/p2p/P2PLayerEndToEndTest.kt | 20 ++++++++++++ .../linkmanager/tracker/PartitionStateTest.kt | 4 +-- .../tracker/PartitionsStatesTests.kt | 4 +-- 6 files changed, 44 insertions(+), 21 deletions(-) diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/inbound/InboundMessageProcessor.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/inbound/InboundMessageProcessor.kt index c7952ee44da..ec51d1cf2da 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/inbound/InboundMessageProcessor.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/inbound/InboundMessageProcessor.kt @@ -210,7 +210,7 @@ internal class InboundMessageProcessor( is SessionManager.SessionDirection.Outbound -> processOutboundDataMessage(sessionIdAndMessage, sessionDirection)?.let { ItemWithSource( InboundResponse( - listOf(it), + it, ), sessionIdAndMessage.message.source, ) @@ -244,7 +244,7 @@ internal class InboundMessageProcessor( private fun processOutboundDataMessage( sessionIdAndMessage: SessionIdAndMessage, sessionDirection: SessionManager.SessionDirection.Outbound - ): Record<*, *>? { + ): List>? { return if (isCommunicationAllowed(sessionDirection.counterparties)) { MessageConverter.extractPayload( sessionDirection.session, @@ -256,8 +256,7 @@ internal class InboundMessageProcessor( is AuthenticatedMessageAck -> { logger.debug { "Processing ack for message ${ack.messageId} from session $sessionIdAndMessage." } sessionManager.messageAcknowledged(sessionIdAndMessage.sessionId) - val record = makeMarkerForAckMessage(ack) - record + makeMarkerForAckMessage(ack) } else -> { logger.warn("Received an inbound message with unexpected type for SessionId = $sessionIdAndMessage.") @@ -370,20 +369,24 @@ internal class InboundMessageProcessor( } } - private fun makeMarkerForAckMessage(message: AuthenticatedMessageAck): Record<*, *> { - return if (features.enableP2PStatefulDeliveryTracker) { - Record( - LINK_ACK_IN_TOPIC, - message.messageId, - null - ) - } else { + private fun makeMarkerForAckMessage(message: AuthenticatedMessageAck): + List> { + return listOf( Record( Schemas.P2P.P2P_OUT_MARKERS, message.messageId, - AppMessageMarker(LinkManagerReceivedMarker(), clock.instant().toEpochMilli()) + AppMessageMarker(LinkManagerReceivedMarker(), clock.instant().toEpochMilli()), ) - + ) + if (features.enableP2PStatefulDeliveryTracker) { + listOf( + Record( + LINK_ACK_IN_TOPIC, + message.messageId, + null, + ) + ) + } else { + emptyList() } } diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionState.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionState.kt index 38bad8aade8..6ee048ffdea 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionState.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionState.kt @@ -48,8 +48,8 @@ internal class PartitionState( @JsonProperty("version") private val savedVersion = AtomicInteger(State.VERSION_INITIAL_VALUE) - private val _readRecordsFromOffset = AtomicLong(0) - private val _processRecordsFromOffset = AtomicLong(0) + private val _readRecordsFromOffset = AtomicLong(-1) + private val _processRecordsFromOffset = AtomicLong(-1) var readRecordsFromOffset: Long get() { return _readRecordsFromOffset.get() } diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStates.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStates.kt index ab588bafc4c..104882dfbe1 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStates.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStates.kt @@ -42,7 +42,7 @@ internal class PartitionsStates( return records.groupBy { it.partition }.flatMap { (partition, records) -> - val processRecordsFromOffset = partitions[partition]?.processRecordsFromOffset ?: 0 + val processRecordsFromOffset = partitions[partition]?.processRecordsFromOffset ?: -1 records.filter { it.offset > processRecordsFromOffset } diff --git a/components/link-manager/src/nonOsgiIntegrationTest/kotlin/net/corda/p2p/P2PLayerEndToEndTest.kt b/components/link-manager/src/nonOsgiIntegrationTest/kotlin/net/corda/p2p/P2PLayerEndToEndTest.kt index 6a5db280703..f8d7805ef40 100644 --- a/components/link-manager/src/nonOsgiIntegrationTest/kotlin/net/corda/p2p/P2PLayerEndToEndTest.kt +++ b/components/link-manager/src/nonOsgiIntegrationTest/kotlin/net/corda/p2p/P2PLayerEndToEndTest.kt @@ -34,6 +34,10 @@ import net.corda.libs.configuration.SmartConfig import net.corda.libs.configuration.SmartConfigFactory import net.corda.libs.configuration.merger.impl.ConfigMergerImpl import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration +import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.DELIVERY_TRACKER_MAX_CACHE_OFFSET_AGE +import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.DELIVERY_TRACKER_MAX_NUMBER_OF_PERSISTENCE_RETRIES +import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.DELIVERY_TRACKER_OUTBOUND_BATCH_PROCESSING_TIMEOUT_SECONDS +import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.DELIVERY_TRACKER_STATE_PERSISTENCE_PERIOD_SECONDS import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.INBOUND_SESSIONS_CACHE_SIZE import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.MAX_MESSAGE_SIZE_KEY import net.corda.libs.configuration.schema.p2p.LinkManagerConfiguration.Companion.MAX_REPLAYING_MESSAGES_PER_PEER @@ -624,6 +628,22 @@ class P2PLayerEndToEndTest { replayConfig.root() ).root()) .withValue(REVOCATION_CHECK_KEY, ConfigValueFactory.fromAnyRef(RevocationCheckMode.OFF.toString())) + .withValue( + DELIVERY_TRACKER_MAX_CACHE_OFFSET_AGE, + ConfigValueFactory.fromAnyRef(Duration.ofSeconds(300)), + ) + .withValue( + DELIVERY_TRACKER_STATE_PERSISTENCE_PERIOD_SECONDS, + ConfigValueFactory.fromAnyRef(Duration.ofSeconds(300)), + ) + .withValue( + DELIVERY_TRACKER_OUTBOUND_BATCH_PROCESSING_TIMEOUT_SECONDS, + ConfigValueFactory.fromAnyRef(Duration.ofSeconds(300)), + ) + .withValue( + DELIVERY_TRACKER_MAX_NUMBER_OF_PERSISTENCE_RETRIES, + ConfigValueFactory.fromAnyRef(Duration.ofSeconds(20)), + ) } private val replayConfig by lazy { ConfigFactory.empty() diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionStateTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionStateTest.kt index f62fa83f044..6a898791cfa 100644 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionStateTest.kt +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionStateTest.kt @@ -83,8 +83,8 @@ class PartitionStateTest { val partitionState = fromState(3, null) assertSoftly { - it.assertThat(partitionState.readRecordsFromOffset).isEqualTo(0) - it.assertThat(partitionState.processRecordsFromOffset).isEqualTo(0) + it.assertThat(partitionState.readRecordsFromOffset).isEqualTo(-1) + it.assertThat(partitionState.processRecordsFromOffset).isEqualTo(-1) it.assertThat(partitionState.counterpartiesToMessages()).hasSize(0) } } diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStatesTests.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStatesTests.kt index 4fe450e8d39..e6663c688be 100644 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStatesTests.kt +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionsStatesTests.kt @@ -718,12 +718,12 @@ class PartitionsStatesTests { .anySatisfy { index, state -> assertThat(index) .isEqualTo(1) - assertThat(state.readRecordsFromOffset).isEqualTo(0) + assertThat(state.readRecordsFromOffset).isEqualTo(-1) } .anySatisfy { index, state -> assertThat(index) .isEqualTo(12) - assertThat(state.readRecordsFromOffset).isEqualTo(0) + assertThat(state.readRecordsFromOffset).isEqualTo(-1) } .anySatisfy { index, state -> assertThat(index) From 339246f62e349a720f4d6e32baf61005fe469b6b Mon Sep 17 00:00:00 2001 From: Yiftach Kaplan Date: Tue, 23 Apr 2024 12:11:06 +0100 Subject: [PATCH 3/5] Offset must be zero or more --- .../p2p/linkmanager/tracker/DeliveryTrackerOffsetProvider.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerOffsetProvider.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerOffsetProvider.kt index ffbde5849c6..e99b8a4c3f3 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerOffsetProvider.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/DeliveryTrackerOffsetProvider.kt @@ -1,6 +1,7 @@ package net.corda.p2p.linkmanager.tracker import net.corda.messaging.api.subscription.listener.ConsumerOffsetProvider +import kotlin.math.max internal class DeliveryTrackerOffsetProvider( private val partitionStates: PartitionsStates, @@ -11,7 +12,7 @@ internal class DeliveryTrackerOffsetProvider( val topic = topicPartitions.map { it.first }.firstOrNull() ?: return emptyMap() val partitionsIndices = topicPartitions.map { it.second }.toSet() return partitionStates.loadPartitions(partitionsIndices).mapValues { (_, state) -> - state.readRecordsFromOffset + max(state.readRecordsFromOffset, 0) }.mapKeys { (partition, _) -> topic to partition } From d8d4f4bb8eb6451bc735f30070889de2bb0dea14 Mon Sep 17 00:00:00 2001 From: Yiftach Kaplan Date: Tue, 23 Apr 2024 13:56:49 +0100 Subject: [PATCH 4/5] Fix domino tile --- .../outbound/OutboundLinkManager.kt | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/outbound/OutboundLinkManager.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/outbound/OutboundLinkManager.kt index 57351fda13c..663723408e6 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/outbound/OutboundLinkManager.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/outbound/OutboundLinkManager.kt @@ -32,22 +32,6 @@ internal class OutboundLinkManager( commonComponents.clock, commonComponents.messageConverter, ) - private val deliveryTracker = DeliveryTracker( - commonComponents, - messagingConfiguration, - sessionComponents.sessionManager, - ) { outboundMessageProcessor.processReplayedAuthenticatedMessage(it) } - - private val subscriptionConfig = SubscriptionConfig(OUTBOUND_MESSAGE_PROCESSOR_GROUP, Schemas.P2P.P2P_OUT_TOPIC) - - private val outboundMessageSubscription = { - commonComponents.subscriptionFactory.createEventLogSubscription( - subscriptionConfig, - outboundMessageProcessor, - messagingConfiguration, - partitionAssignmentListener = null - ) - } override val dominoTile = if (features.enableP2PStatefulDeliveryTracker) { val publisher = PublisherWithDominoLogic( @@ -78,6 +62,23 @@ internal class OutboundLinkManager( ), ) } else { + val deliveryTracker = DeliveryTracker( + commonComponents, + messagingConfiguration, + sessionComponents.sessionManager, + ) { outboundMessageProcessor.processReplayedAuthenticatedMessage(it) } + + val subscriptionConfig = SubscriptionConfig(OUTBOUND_MESSAGE_PROCESSOR_GROUP, Schemas.P2P.P2P_OUT_TOPIC) + + val outboundMessageSubscription = { + commonComponents.subscriptionFactory.createEventLogSubscription( + subscriptionConfig, + outboundMessageProcessor, + messagingConfiguration, + partitionAssignmentListener = null + ) + } + SubscriptionDominoTile( commonComponents.lifecycleCoordinatorFactory, outboundMessageSubscription, From 2bdb962cdf0015e631ecd8576a5ebaaa0974993a Mon Sep 17 00:00:00 2001 From: Yiftach Kaplan Date: Thu, 25 Apr 2024 11:36:56 +0100 Subject: [PATCH 5/5] Fix changes from issues found while testing --- .../outbound/OutboundMessageProcessor.kt | 62 +++++++++---- .../p2p/linkmanager/tracker/PartitionState.kt | 15 +++- .../outbound/OutboundMessageProcessorTest.kt | 88 ++++++++++++++++++- .../linkmanager/tracker/PartitionStateTest.kt | 4 +- 4 files changed, 147 insertions(+), 22 deletions(-) diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/outbound/OutboundMessageProcessor.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/outbound/OutboundMessageProcessor.kt index 2ae6be0af7b..d806e1c102d 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/outbound/OutboundMessageProcessor.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/outbound/OutboundMessageProcessor.kt @@ -43,6 +43,8 @@ import net.corda.metrics.CordaMetrics import net.corda.p2p.linkmanager.TraceableItem import net.corda.p2p.linkmanager.metrics.recordOutboundMessagesMetric import net.corda.p2p.linkmanager.metrics.recordOutboundSessionMessagesMetric +import net.corda.schema.Schemas.P2P.LINK_ACK_IN_TOPIC +import net.corda.utilities.flags.Features @Suppress("LongParameterList", "TooManyFunctions") internal class OutboundMessageProcessor( @@ -55,6 +57,7 @@ internal class OutboundMessageProcessor( private val messageConverter: MessageConverter, private val networkMessagingValidator: NetworkMessagingValidator = NetworkMessagingValidator(membershipGroupReaderProvider), + private val features: Features = Features(), ) : EventLogProcessor { override val keyClass = String::class.java @@ -301,12 +304,12 @@ internal class OutboundMessageProcessor( } if (ttlExpired(messageAndKey.message.header.ttl)) { - val expiryMarker = recordForTTLExpiredMarker(messageAndKey.message.header.messageId) + val expiryMarkers = recordForTTLExpiredMarker(messageAndKey.message.header.messageId) return if (isReplay) { - ValidateAuthenticatedMessageResult.NoSessionNeeded(listOf(expiryMarker)) + ValidateAuthenticatedMessageResult.NoSessionNeeded(expiryMarkers) } else { ValidateAuthenticatedMessageResult.NoSessionNeeded( - listOf(recordForLMProcessedMarker(messageAndKey, messageAndKey.message.header.messageId), expiryMarker) + expiryMarkers + recordForLMProcessedMarker(messageAndKey, messageAndKey.message.header.messageId) ) } } @@ -331,18 +334,16 @@ internal class OutboundMessageProcessor( recordInboundMessagesMetric(messageAndKey.message) return if (isReplay) { ValidateAuthenticatedMessageResult.NoSessionNeeded( - listOf( - Record(Schemas.P2P.P2P_IN_TOPIC, messageAndKey.key, AppMessage(messageAndKey.message)), - recordForLMReceivedMarker(messageAndKey.message.header.messageId) - ) + recordForLMReceivedMarker(messageAndKey.message.header.messageId) + + Record(Schemas.P2P.P2P_IN_TOPIC, messageAndKey.key, AppMessage(messageAndKey.message)) ) } else { ValidateAuthenticatedMessageResult.NoSessionNeeded( - listOf( - Record(Schemas.P2P.P2P_IN_TOPIC, messageAndKey.key, AppMessage(messageAndKey.message)), - recordForLMProcessedMarker(messageAndKey, messageAndKey.message.header.messageId), - recordForLMReceivedMarker(messageAndKey.message.header.messageId) - ) + recordForLMReceivedMarker(messageAndKey.message.header.messageId) + + listOf( + Record(Schemas.P2P.P2P_IN_TOPIC, messageAndKey.key, AppMessage(messageAndKey.message)), + recordForLMProcessedMarker(messageAndKey, messageAndKey.message.header.messageId), + ) ) } } else { @@ -416,14 +417,43 @@ internal class OutboundMessageProcessor( return Record(Schemas.P2P.P2P_OUT_MARKERS, messageId, marker) } - private fun recordForLMReceivedMarker(messageId: String): Record { + private fun recordForLMReceivedMarker(messageId: String): List> { val marker = AppMessageMarker(LinkManagerReceivedMarker(), clock.instant().toEpochMilli()) - return Record(Schemas.P2P.P2P_OUT_MARKERS, messageId, marker) + val markerRecord = listOf( + Record(Schemas.P2P.P2P_OUT_MARKERS, messageId, marker) + ) + val linkInRecords = if (features.enableP2PStatefulDeliveryTracker) { + listOf( + Record( + LINK_ACK_IN_TOPIC, + messageId, + null, + ) + ) + } else { + emptyList() + } + return markerRecord + linkInRecords } - private fun recordForTTLExpiredMarker(messageId: String): Record { + private fun recordForTTLExpiredMarker(messageId: String): + List> { val marker = AppMessageMarker(TtlExpiredMarker(Component.LINK_MANAGER), clock.instant().toEpochMilli()) - return Record(Schemas.P2P.P2P_OUT_MARKERS, messageId, marker) + val markerRecords = listOf( + Record(Schemas.P2P.P2P_OUT_MARKERS, messageId, marker) + ) + val linkInRecords = if (features.enableP2PStatefulDeliveryTracker) { + listOf( + Record( + LINK_ACK_IN_TOPIC, + messageId, + null, + ) + ) + } else { + emptyList() + } + return markerRecords + linkInRecords } private fun recordForLMDiscardedMarker(message: AuthenticatedMessageAndKey, diff --git a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionState.kt b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionState.kt index 6ee048ffdea..eb5312de783 100644 --- a/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionState.kt +++ b/components/link-manager/src/main/kotlin/net/corda/p2p/linkmanager/tracker/PartitionState.kt @@ -1,5 +1,6 @@ package net.corda.p2p.linkmanager.tracker +import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.core.JacksonException import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule @@ -14,12 +15,15 @@ import net.corda.v5.base.types.MemberX500Name import net.corda.virtualnode.HoldingIdentity import java.time.Instant import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicLong internal class PartitionState( @JsonProperty("partition") private val partition: Int, + @JsonIgnore + private val toCreate: AtomicBoolean = AtomicBoolean(false) ) { companion object { private val jsonParser = jacksonObjectMapper() @@ -37,7 +41,10 @@ internal class PartitionState( throw CordaRuntimeException("Can not read state json.", e) } } else { - PartitionState(partition) + PartitionState( + partition, + AtomicBoolean(true) + ) } } } @@ -89,7 +96,7 @@ internal class PartitionState( key = key, version = version, ) - if (version == State.VERSION_INITIAL_VALUE) { + if (toCreate.get()) { group.create(state) } else { group.update(state) @@ -119,7 +126,9 @@ internal class PartitionState( } fun saved() { - savedVersion.incrementAndGet() + if (!toCreate.getAndSet(false)) { + savedVersion.incrementAndGet() + } } fun trim() { diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/outbound/OutboundMessageProcessorTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/outbound/OutboundMessageProcessorTest.kt index 68237575914..95e3840338f 100644 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/outbound/OutboundMessageProcessorTest.kt +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/outbound/OutboundMessageProcessorTest.kt @@ -30,9 +30,11 @@ import net.corda.p2p.linkmanager.sessions.PendingSessionMessageQueues import net.corda.p2p.linkmanager.sessions.SessionManager import net.corda.p2p.linkmanager.utilities.mockMembersAndGroups import net.corda.schema.Schemas +import net.corda.schema.Schemas.P2P.LINK_ACK_IN_TOPIC import net.corda.test.util.identity.createTestHoldingIdentity import net.corda.test.util.time.MockTimeFacilitiesProvider import net.corda.utilities.Either +import net.corda.utilities.flags.Features import net.corda.utilities.seconds import net.corda.virtualnode.toAvro import org.assertj.core.api.Assertions.assertThat @@ -81,6 +83,9 @@ class OutboundMessageProcessorTest { on { validateInbound(any(), any()) } doReturn Either.Left(Unit) on { validateOutbound(any(), any()) } doReturn Either.Left(Unit) } + private val features = mock { + on { enableP2PStatefulDeliveryTracker } doReturn false + } private val processor = OutboundMessageProcessor( sessionManager, @@ -90,7 +95,8 @@ class OutboundMessageProcessorTest { messagesPendingSession, mockTimeFacilitiesProvider.clock, messageConverter, - networkMessagingValidator + networkMessagingValidator, + features = features, ) private fun setupSessionManager(response: SessionManager.SessionState) { @@ -999,6 +1005,47 @@ class OutboundMessageProcessorTest { } } + @Test + fun `processReplayedAuthenticatedMessage will send a message to the link ack in topic when the destination is locally hosted`() { + whenever(features.enableP2PStatefulDeliveryTracker).doReturn(true) + val destination = membersAndGroups.first.getGroupReader(localIdentity).lookup(myIdentity.x500Name)!! + whenever(hostingMap.isHostedLocallyAndSessionKeyMatch(destination)).doReturn(true) + val payload = "test" + val authenticatedMsg = AuthenticatedMessage( + AuthenticatedMessageHeader( + myIdentity.toAvro(), + localIdentity.toAvro(), + null, "message-id", "trace-id", "system-1", MembershipStatusFilter.ACTIVE + ), + ByteBuffer.wrap(payload.toByteArray()) + ) + val authenticatedMessageAndKey = AuthenticatedMessageAndKey(authenticatedMsg, "key") + + val records = processor.processReplayedAuthenticatedMessage(authenticatedMessageAndKey) + + assertSoftly { softAssertions -> + softAssertions.assertThat(records).hasSize(3) + val markers = records.filter { it.topic == Schemas.P2P.P2P_OUT_MARKERS }.map { it.value } + .filterIsInstance() + softAssertions.assertThat(markers).hasSize(1) + + val receivedMarkers = markers.map { it.marker }.filterIsInstance() + softAssertions.assertThat(receivedMarkers).hasSize(1) + + val messages = records + .filter { + it.topic == Schemas.P2P.P2P_IN_TOPIC + }.filter { + it.key == "key" + }.map { it.value }.filterIsInstance() + softAssertions.assertThat(messages).hasSize(1) + softAssertions.assertThat(messages.first().message).isEqualTo(authenticatedMessageAndKey.message) + + val ackMessage = records.firstOrNull { it.topic == LINK_ACK_IN_TOPIC } + softAssertions.assertThat(ackMessage?.key).isEqualTo("message-id") + } + } + @Test fun `processReplayedAuthenticatedMessage will not write any records if destination is not in the members map or locally hosted`() { setupSessionManager(SessionManager.SessionState.SessionEstablished(authenticatedSession, sessionCounterparties)) @@ -1239,6 +1286,45 @@ class OutboundMessageProcessorTest { } } + @Test + fun `processReplayedAuthenticatedMessage will send a record to the link ack in topic when needed`() { + whenever(features.enableP2PStatefulDeliveryTracker).doReturn(true) + setupSessionManager(SessionManager.SessionState.SessionAlreadyPending(sessionCounterparties)) + val authenticatedMsg = AuthenticatedMessage( + AuthenticatedMessageHeader( + remoteIdentity.toAvro(), + localIdentity.toAvro(), + null, "MessageId", "trace-id", "system-1", MembershipStatusFilter.ACTIVE + ), + ByteBuffer.wrap("payload".toByteArray()) + ) + val authenticatedMessageAndKey = AuthenticatedMessageAndKey( + authenticatedMsg, + "key" + ) + authenticatedMessageAndKey.message.header.ttl = Instant.ofEpochMilli(0) + + val records = processor.processReplayedAuthenticatedMessage(authenticatedMessageAndKey) + + val markers = records.filter { it.value is AppMessageMarker } + assertSoftly { + it.assertThat(records).hasSize(2) + + it.assertThat(markers.map { it.key }).allMatch { + it.equals("MessageId") + } + it.assertThat(markers).hasSize(1) + it.assertThat(markers.map { it.value as AppMessageMarker }.filter { it.marker is TtlExpiredMarker }) + .hasSize(1) + it.assertThat(markers.map { it.value as AppMessageMarker } + .filter { it.marker is LinkManagerReceivedMarker }).isEmpty() + it.assertThat(markers.map { it.topic }.distinct()).containsOnly(Schemas.P2P.P2P_OUT_MARKERS) + + val ackMessage = records.firstOrNull { record -> record.topic == LINK_ACK_IN_TOPIC } + it.assertThat(ackMessage?.key).isEqualTo("MessageId") + } + } + @Test fun `OutboundMessageProcessor produces TtlExpiredMarker and LinkManagerProcessedMarker if TTL expiry is true and replay is false`() { val authenticatedMsg = AuthenticatedMessage( diff --git a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionStateTest.kt b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionStateTest.kt index 6a898791cfa..34bd5dd073b 100644 --- a/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionStateTest.kt +++ b/components/link-manager/src/test/kotlin/net/corda/p2p/linkmanager/tracker/PartitionStateTest.kt @@ -199,7 +199,7 @@ class PartitionStateTest { state.processRecordsFromOffset = 2001 val captureState = argumentCaptor() val group = mock { - on { create(captureState.capture()) } doReturn mock + on { update(captureState.capture()) } doReturn mock } state.addToOperationGroup(group) @@ -228,7 +228,7 @@ class PartitionStateTest { @Test fun `it create a new state when needed`() { - val state = PartitionState(1) + val state = fromState (1, null) val group = mock { on { create(any()) } doReturn mock }