From 688f3d5e3855b0c079d16659b6b824685d2a3033 Mon Sep 17 00:00:00 2001 From: Cody Littley <56973212+cody-littley@users.noreply.github.com> Date: Thu, 11 Jan 2024 16:52:29 -0600 Subject: [PATCH] feat: PCES can use either birth round or generation (#10808) Signed-off-by: Austin Littley Signed-off-by: Cody Littley Co-authored-by: Austin Littley --- .../pool/AccountTransactionFactoryTests.java | 2 + .../com/swirlds/platform/SwirldsPlatform.java | 19 +- .../swirlds/platform/event/GossipEvent.java | 3 + .../preconsensus/BestEffortPcesFileCopy.java | 56 +- .../event/preconsensus/PcesConfig.java | 128 ++--- .../platform/event/preconsensus/PcesFile.java | 240 +++++--- .../event/preconsensus/PcesFileIterator.java | 29 +- .../event/preconsensus/PcesFileManager.java | 103 ++-- .../event/preconsensus/PcesFileReader.java | 38 +- .../event/preconsensus/PcesFileTracker.java | 92 +-- .../event/preconsensus/PcesMetrics.java | 47 +- .../preconsensus/PcesMultiFileIterator.java | 28 +- .../event/preconsensus/PcesMutableFile.java | 55 +- .../event/preconsensus/PcesUtilities.java | 79 +-- .../event/preconsensus/PcesWriter.java | 149 ++--- .../recovery/EventRecoveryWorkflow.java | 5 + .../wiring/LinkedEventIntakeWiring.java | 19 +- .../platform/wiring/PlatformWiring.java | 17 +- .../wiring/components/PcesWriterWiring.java | 30 +- .../platform/wiring/diagram-commands.txt | 13 +- .../test/fixtures/event/RandomEventUtils.java | 10 +- .../generator/AbstractGraphGenerator.java | 13 - .../event/source/AbstractEventSource.java | 35 +- .../preconsensus/PcesFileManagerTests.java | 149 ++--- .../preconsensus/PcesFileReaderTests.java | 540 +++++++++++------- .../event/preconsensus/PcesFileTests.java | 184 +++--- .../preconsensus/PcesReadWriteTests.java | 277 +++++---- .../preconsensus/PcesUtilitiesTests.java | 139 +++-- .../event/preconsensus/PcesWriterTests.java | 446 +++++++++------ 29 files changed, 1709 insertions(+), 1236 deletions(-) diff --git a/platform-sdk/platform-apps/tests/PlatformTestingTool/src/test/java/com/swirlds/demo/virtualmerkle/transaction/pool/AccountTransactionFactoryTests.java b/platform-sdk/platform-apps/tests/PlatformTestingTool/src/test/java/com/swirlds/demo/virtualmerkle/transaction/pool/AccountTransactionFactoryTests.java index d2659de0e967..a5fd09f6fd8f 100644 --- a/platform-sdk/platform-apps/tests/PlatformTestingTool/src/test/java/com/swirlds/demo/virtualmerkle/transaction/pool/AccountTransactionFactoryTests.java +++ b/platform-sdk/platform-apps/tests/PlatformTestingTool/src/test/java/com/swirlds/demo/virtualmerkle/transaction/pool/AccountTransactionFactoryTests.java @@ -27,8 +27,10 @@ import com.swirlds.merkle.map.test.pta.MapKey; import java.util.ArrayList; import java.util.List; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +@Disabled("flaky") class AccountTransactionFactoryTests { /** diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/SwirldsPlatform.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/SwirldsPlatform.java index 67d01411d0c2..30ac2e1d4ee9 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/SwirldsPlatform.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/SwirldsPlatform.java @@ -79,6 +79,7 @@ import com.swirlds.platform.dispatch.DispatchConfiguration; import com.swirlds.platform.dispatch.triggers.flow.DiskStateLoadedTrigger; import com.swirlds.platform.dispatch.triggers.flow.ReconnectStateLoadedTrigger; +import com.swirlds.platform.event.AncientMode; import com.swirlds.platform.event.EventCounter; import com.swirlds.platform.event.GossipEvent; import com.swirlds.platform.event.creation.EventCreationManager; @@ -440,15 +441,25 @@ public class SwirldsPlatform implements Platform { try { final Path databaseDirectory = getDatabaseDirectory(platformContext, selfId); + // When we perform the migration to using birth round bounding, we will need to read + // the old type and start writing the new type. + final AncientMode currentFileType = platformContext + .getConfiguration() + .getConfigData(EventConfig.class) + .useBirthRoundAncientThreshold() + ? AncientMode.BIRTH_ROUND_THRESHOLD + : AncientMode.GENERATION_THRESHOLD; + initialPcesFiles = PcesFileReader.readFilesFromDisk( platformContext, recycleBin, databaseDirectory, initialState.getRound(), - preconsensusEventStreamConfig.permitGaps()); + preconsensusEventStreamConfig.permitGaps(), + currentFileType); - preconsensusEventFileManager = new PcesFileManager( - platformContext, Time.getCurrent(), initialPcesFiles, selfId, initialState.getRound()); + preconsensusEventFileManager = + new PcesFileManager(platformContext, initialPcesFiles, selfId, initialState.getRound()); } catch (final IOException e) { throw new UncheckedIOException(e); } @@ -791,7 +802,6 @@ public class SwirldsPlatform implements Platform { platformWiring.updateNonAncientEventWindow(NonAncientEventWindow.createUsingPlatformContext( initialState.getRound(), initialMinimumGenerationNonAncient, platformContext)); - platformWiring.updateMinimumGenerationNonAncient(initialState.getMinRoundGeneration()); // We don't want to invoke these callbacks until after we are starting up. final long round = initialState.getRound(); @@ -977,7 +987,6 @@ private void loadReconnectState(final SignedState signedState) { signedState.getState().getPlatformState().getAddressBook())); platformWiring.updateNonAncientEventWindow(NonAncientEventWindow.createUsingPlatformContext( signedState.getRound(), signedState.getMinRoundGeneration(), platformContext)); - platformWiring.updateMinimumGenerationNonAncient(signedState.getMinRoundGeneration()); consensusRoundHandler.loadDataFromSignedState(signedState, true); diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/GossipEvent.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/GossipEvent.java index 9845629fa10c..5e3e3dcf480d 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/GossipEvent.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/GossipEvent.java @@ -202,6 +202,9 @@ public void buildDescriptor() { this.descriptor = hashedData.createEventDescriptor(); } + /** + * {@inheritDoc} + */ @Override public long getGeneration() { return hashedData.getGeneration(); diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/BestEffortPcesFileCopy.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/BestEffortPcesFileCopy.java index a5a2f5c43e8d..5b49e5955f1f 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/BestEffortPcesFileCopy.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/BestEffortPcesFileCopy.java @@ -57,18 +57,18 @@ private BestEffortPcesFileCopy() {} * fail as a result. This method retries several times if a failure is encountered. Success is not guaranteed, but * success or failure is atomic and will not throw an exception. * - * @param platformContext the platform context - * @param selfId the id of this node - * @param destinationDirectory the directory where the state is being written - * @param minimumGenerationNonAncient the minimum generation of events that are not ancient, with respect to the - * state that is being written - * @param round the round of the state that is being written + * @param platformContext the platform context + * @param selfId the id of this node + * @param destinationDirectory the directory where the state is being written + * @param lowerBound the lower bound of events that are not ancient, with respect to the state that is + * being written + * @param round the round of the state that is being written */ public static void copyPcesFilesRetryOnFailure( @NonNull final PlatformContext platformContext, @NonNull final NodeId selfId, @NonNull final Path destinationDirectory, - final long minimumGenerationNonAncient, + final long lowerBound, final long round) { final boolean copyPreconsensusStream = platformContext @@ -89,8 +89,7 @@ public static void copyPcesFilesRetryOnFailure( try { executeAndRename( pcesDestination, - temporaryDirectory -> copyPcesFiles( - platformContext, selfId, temporaryDirectory, minimumGenerationNonAncient)); + temporaryDirectory -> copyPcesFiles(platformContext, selfId, temporaryDirectory, lowerBound)); return; } catch (final IOException | UncheckedIOException e) { @@ -122,17 +121,17 @@ public static void copyPcesFilesRetryOnFailure( * real production states and streams, in the short term. In the longer term we should consider alternate and * cleaner strategies. * - * @param platformContext the platform context - * @param selfId the id of this node - * @param destinationDirectory the directory where the PCES files should be written - * @param minimumGenerationNonAncient the minimum generation of events that are not ancient, with respect to the - * state that is being written + * @param platformContext the platform context + * @param selfId the id of this node + * @param destinationDirectory the directory where the PCES files should be written + * @param lowerBound the lower bound of events that are not ancient, with respect to the state that is + * being written */ private static void copyPcesFiles( @NonNull final PlatformContext platformContext, @NonNull final NodeId selfId, @NonNull final Path destinationDirectory, - final long minimumGenerationNonAncient) + final long lowerBound) throws IOException { final List allFiles = gatherPcesFilesOnDisk(selfId, platformContext); @@ -144,7 +143,7 @@ private static void copyPcesFiles( Collections.sort(allFiles); // Discard all files that either have an incorrect origin or that do not contain non-ancient events. - final List filesToCopy = getRequiredPcesFiles(allFiles, minimumGenerationNonAncient); + final List filesToCopy = getRequiredPcesFiles(allFiles, lowerBound); if (filesToCopy.isEmpty()) { return; } @@ -156,20 +155,18 @@ private static void copyPcesFiles( * Get the preconsensus files that we need to copy to a state. We need any file that has a matching origin and that * contains non-ancient events (w.r.t. the state). * - * @param allFiles all PCES files on disk - * @param minimumGenerationNonAncient the minimum generation of events that are not ancient, with respect to the - * state that is being written + * @param allFiles all PCES files on disk + * @param lowerBound the lower bound of events that are not ancient, with respect to the state that is being + * written * @return the list of files to copy */ @NonNull - private static List getRequiredPcesFiles( - @NonNull final List allFiles, final long minimumGenerationNonAncient) { + private static List getRequiredPcesFiles(@NonNull final List allFiles, final long lowerBound) { final List filesToCopy = new ArrayList<>(); final PcesFile lastFile = allFiles.get(allFiles.size() - 1); for (final PcesFile file : allFiles) { - if (file.getOrigin() == lastFile.getOrigin() - && file.getMaximumGeneration() >= minimumGenerationNonAncient) { + if (file.getOrigin() == lastFile.getOrigin() && file.getUpperBound() >= lowerBound) { filesToCopy.add(file); } } @@ -177,30 +174,29 @@ private static List getRequiredPcesFiles( if (filesToCopy.isEmpty()) { logger.warn( STATE_TO_DISK.getMarker(), - "No preconsensus event files meeting specified criteria found to copy. " - + "Minimum generation non-ancient: {}", - minimumGenerationNonAncient); + "No preconsensus event files meeting specified criteria found to copy. Lower bound: {}", + lowerBound); } else if (filesToCopy.size() == 1) { logger.info( STATE_TO_DISK.getMarker(), """ Found 1 preconsensus event file meeting specified criteria to copy. - Minimum generation non-ancient: {} + Lower bound: {} File: {} """, - minimumGenerationNonAncient, + lowerBound, filesToCopy.get(0).getPath()); } else { logger.info( STATE_TO_DISK.getMarker(), """ Found {} preconsensus event files meeting specified criteria to copy. - Minimum generation non-ancient: {} + Lower bound: {} First file to copy: {} Last file to copy: {} """, filesToCopy.size(), - minimumGenerationNonAncient, + lowerBound, filesToCopy.get(0).getPath(), filesToCopy.get(filesToCopy.size() - 1).getPath()); } diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesConfig.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesConfig.java index dacdbadcf7f0..ebfc34c6977a 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesConfig.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesConfig.java @@ -25,83 +25,69 @@ /** * Configuration for preconsensus event storage. * - * @param writeQueueCapacity the queue capacity for preconsensus events waiting to be - * written to disk - * @param minimumRetentionPeriod the minimum amount of time that preconsensus events should be - * stored on disk. At a minimum, should exceed the length of time - * between state saving. - * @param preferredFileSizeMegabytes the preferred file size for preconsensus event files. Not a - * strong guarantee on file size, more of a suggestion. - * @param bootstrapGenerationalSpan when first starting up a preconsensus event file manager, the - * running average for the generational utilization will not have - * any values in it. Use this value until the running average - * represents real data. Once the running average for the - * generational utilization becomes available, then this value is - * ignored. - * @param generationalUtilizationSpanRunningAverageLength the preconsensus event stream tracks the running average of - * the generational utilization in event stream files. It uses - * this running average in a heuristic when deciding the maximum - * generation permitted in a new file. This value controls the - * number of recent files that are considered when computing the - * running average. - * @param bootstrapGenerationalSpanOverlapFactor when choosing the generation span for a new file during the - * bootstrapping phase, multiply the running average of previous - * file generational utilization by this factor. Choosing a value - * too large will cause unnecessary un-utilized generational span - * if the node crashes. Choosing a value too small may case event - * files to be smaller than the preferred size. Should be larger - * than the non-boostrap span overlap factor to allow for faster - * convergence on a sane file size at startup time. - * @param generationalSpanOverlapFactor when choosing the generation span for a new file, multiply the - * running average of previous file generational utilization by - * this factor. Choosing a value too large will cause unnecessary - * un-utilized generational span if the node crashes. Choosing a - * value too small may case event files to be smaller than the - * preferred size. - * @param minimumGenerationalCapacity when creating a new file, make sure it at least has the - * capacity for this many generations after the generation of the - * first event in the file. This is puts a sane "floor" on the - * generational span heuristic, so that we never attempt to open - * a file that doesn't have capacity for events, regardless of - * the required generational span to do so. If properly - * configured and under steady state operation, this capacity is - * unlikely to be a limiting factor on the generational span of - * files. - * @param permitGaps if false (default) then throw an exception if we attempt to - * load preconsensus events and notice gaps in the file sequence. - * This is only possible if a preconsensus event file has been - * deleted out of band. This setting is present only to allow - * emergency manual action. In general, allowing gaps is likely - * to either lead to ISSes or, more likely, cause events to be - * added to the hashgraph without their parents being added. Or - * Both. Use this with caution. - * @param databaseDirectory the directory where preconsensus events will be stored, - * relative to - * {@link - * com.swirlds.common.config.StateConfig#savedStateDirectory()}. - * @param replayQueueSize the size of the queue used for holding preconsensus events - * that are waiting to be replayed - * @param replayHashPoolSize the number of threads used for hashing events during replay - * @param copyRecentStreamToStateSnapshots if true, then copy recent PCES files into the saved state - * snapshot directories every time we take a state snapshot. The - * files copied are guaranteed to contain all non-ancient events - * w.r.t. the state snapshot. - * @param compactLastFileOnStartup if true, then compact the last file's generational span on - * startup. - * @param forceIgnorePcesSignatures if true, then ignore the signatures on preconsensus events. - * Note: This is a TEST ONLY setting. It must never be enabled - * in production. + * @param writeQueueCapacity the queue capacity for preconsensus events waiting to be written to disk + * @param minimumRetentionPeriod the minimum amount of time that preconsensus events should be stored on + * disk. At a minimum, should exceed the length of time between state + * saving. + * @param preferredFileSizeMegabytes the preferred file size for preconsensus event files. Not a strong + * guarantee on file size, more of a suggestion. + * @param bootstrapSpan when first starting up a preconsensus event file manager, the running + * average for the span utilization will not have any values in it. Use this + * value until the running average represents real data. Once the running + * average for the span utilization becomes available, then this value is + * ignored. + * @param spanUtilizationRunningAverageLength the preconsensus event stream tracks the running average of the span + * utilization in event stream files. It uses this running average in a + * heuristic when deciding the upper bound of a new file. This value controls + * the number of recent files that are considered when computing the running + * average. + * @param bootstrapSpanOverlapFactor when choosing the pan for a new file during the bootstrapping phase, + * multiply the running average of previous file span utilization by this + * factor. Choosing a value too large will cause unnecessary un-utilized span + * if the node crashes. Choosing a value too small may case event files to be + * smaller than the preferred size. Should be larger than the non-boostrap + * span overlap factor to allow for faster convergence on a sane file size at + * startup time. + * @param spanOverlapFactor when choosing the span for a new file, multiply the running average of + * previous file span utilization by this factor. Choosing a value too large + * will cause unnecessary un-utilized span if the node crashes. Choosing a + * value too small may case event files to be smaller than the preferred + * size. + * @param minimumSpan when creating a new file, make sure it at least has this much available + * span. This is puts a sane "floor" on the span heuristic, so that we never + * attempt to open a file that doesn't have capacity for events, regardless + * of the required span to do so. If properly configured and under steady + * state operation, this capacity is unlikely to be a limiting factor on the + * span of files. + * @param permitGaps if false (default) then throw an exception if we attempt to load + * preconsensus events and notice gaps in the file sequence. This is only + * possible if a preconsensus event file has been deleted out of band. This + * setting is present only to allow emergency manual action. In general, + * allowing gaps is likely to either lead to ISSes or, more likely, cause + * events to be added to the hashgraph without their parents being added. Or + * Both. Use this with caution. + * @param databaseDirectory the directory where preconsensus events will be stored, relative to + * {@link com.swirlds.common.config.StateConfig#savedStateDirectory()}. + * @param replayQueueSize the size of the queue used for holding preconsensus events that are + * waiting to be replayed + * @param replayHashPoolSize the number of threads used for hashing events during replay + * @param copyRecentStreamToStateSnapshots if true, then copy recent PCES files into the saved state snapshot + * directories every time we take a state snapshot. The files copied are + * guaranteed to contain all non-ancient events w.r.t. the state snapshot. + * @param compactLastFileOnStartup if true, then compact the last file's span on startup. + * @param forceIgnorePcesSignatures if true, then ignore the signatures on preconsensus events. Note: This is + * a TEST ONLY setting. It must never be enabled in production. */ @ConfigData("event.preconsensus") public record PcesConfig( @ConfigProperty(defaultValue = "1000") int writeQueueCapacity, @ConfigProperty(defaultValue = "1h") Duration minimumRetentionPeriod, @ConfigProperty(defaultValue = "10") int preferredFileSizeMegabytes, - @ConfigProperty(defaultValue = "50") int bootstrapGenerationalSpan, - @ConfigProperty(defaultValue = "5") int generationalUtilizationSpanRunningAverageLength, - @Min(1) @ConfigProperty(defaultValue = "10") double bootstrapGenerationalSpanOverlapFactor, - @Min(1) @ConfigProperty(defaultValue = "1.2") double generationalSpanOverlapFactor, - @ConfigProperty(defaultValue = "5") int minimumGenerationalCapacity, + @ConfigProperty(defaultValue = "50") int bootstrapSpan, + @ConfigProperty(defaultValue = "5") int spanUtilizationRunningAverageLength, + @Min(1) @ConfigProperty(defaultValue = "10") double bootstrapSpanOverlapFactor, + @Min(1) @ConfigProperty(defaultValue = "1.2") double spanOverlapFactor, + @ConfigProperty(defaultValue = "5") int minimumSpan, @ConfigProperty(defaultValue = "false") boolean permitGaps, @ConfigProperty(defaultValue = "preconsensus-events") Path databaseDirectory, @ConfigProperty(defaultValue = "1024") int replayQueueSize, diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFile.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFile.java index ca14858c94c8..e2ea43688f8d 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFile.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFile.java @@ -18,9 +18,12 @@ import static com.swirlds.common.formatting.StringFormattingUtils.parseSanitizedTimestamp; import static com.swirlds.common.formatting.StringFormattingUtils.sanitizeTimestamp; +import static com.swirlds.platform.event.AncientMode.BIRTH_ROUND_THRESHOLD; +import static com.swirlds.platform.event.AncientMode.GENERATION_THRESHOLD; import com.swirlds.common.io.utility.RecycleBin; import com.swirlds.common.utility.NonCryptographicHashing; +import com.swirlds.platform.event.AncientMode; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.io.IOException; @@ -39,13 +42,20 @@ *

* *

- * Files have the following format. Deviation from this format is not allowed. A {@link PcesFileManager} - * will be unable to correctly read files with a different format. + * Prior to the birth round migration, files have the following format. Deviation from this format is not allowed. A + * {@link PcesFileManager} will be unable to correctly read files with a different format. *

* 
  * [Instant.toString().replace(":", "+")]-seq[sequence number]-ming[minimum legal generation]-maxg[maximum legal generation]-orgn[origin round].pces
  *
*

+ * After the birth round migration, files have the following format. Deviation from this format is not allowed. A + * {@link PcesFileManager} will be unable to correctly read files with a different format. + *

+ * 
+ * [Instant.toString().replace(":", "+")]-seq[sequence number]-minr[minimum legal birth round]-maxr[maximum legal birth round]-orgn[origin round].pces
+ *
+ *

* By default, files are stored with the following directory structure. Note that files are not required to be stored * with this directory structure in order to be read by a {@link PcesFileManager}. *

@@ -75,11 +85,21 @@ public final class PcesFile implements Comparable { */ public static final String MINIMUM_GENERATION_PREFIX = "ming"; + /** + * Written before the minimum birth round in the file name. Improves readability for humans. + */ + public static final String MINIMUM_BIRTH_ROUND_PREFIX = "minr"; + /** * Written before the maximum generation in the file name. Improves readability for humans. */ public static final String MAXIMUM_GENERATION_PREFIX = "maxg"; + /** + * Written before the maximum birth round in the file name. Improves readability for humans. + */ + public static final String MAXIMUM_BIRTH_ROUND_PREFIX = "maxr"; + /** * Written before the origin round. Improves readability for humans. */ @@ -97,14 +117,16 @@ public final class PcesFile implements Comparable { private final long sequenceNumber; /** - * The minimum generation of events that are permitted to be in this file. + * The lower bound for events in the PCES file. Will be either a generation or a birth round depending on the + * {@link AncientMode}. */ - private final long minimumGeneration; + private final long lowerBound; /** - * The maximum generation of events that are permitted to be in this file. + * The upper bound for events that are permitted to be in this file. Will be either a generation or a birth round + * depending on the {@link AncientMode}. */ - private final long maximumGeneration; + private final long upperBound; /** * The round number from which an unbroken stream of events has been written. If two sequential files have different @@ -122,49 +144,59 @@ public final class PcesFile implements Comparable { */ private final Path path; + /** + * The type of the file. + */ + private final AncientMode fileType; + /** * Construct a new PreConsensusEventFile. * - * @param timestamp the timestamp of when the writing of this file began - * @param sequenceNumber the sequence number of the file. All file sequence numbers are unique. Sequence numbers - * are allocated in monotonically increasing order. - * @param minimumGeneration the minimum generation of events that are permitted to be in this file - * @param maximumGeneration the maximum generation of events that are permitted to be in this file - * @param origin the origin of the stream file, signals the round from which the stream is unbroken - * @param path the location where this file can be found + * @param fileType the type of this PCES file + * @param timestamp the timestamp of when the writing of this file began + * @param sequenceNumber the sequence number of the file. All file sequence numbers are unique. Sequence numbers are + * allocated in monotonically increasing order. + * @param lowerBound the upper bound of events that are permitted to be in this file, either a generation or a + * birth round depending on the {@link AncientMode}. + * @param upperBound the lower bound of events that are permitted to be in this file, either a generation or a + * birth round depending on the {@link AncientMode}. + * @param origin the origin of the stream file, signals the round from which the stream is unbroken + * @param path the location where this file can be found */ private PcesFile( + @NonNull final AncientMode fileType, @NonNull final Instant timestamp, final long sequenceNumber, - final long minimumGeneration, - final long maximumGeneration, + final long lowerBound, + final long upperBound, final long origin, @NonNull final Path path) { if (sequenceNumber < 0) { - throw new IllegalArgumentException("sequence number " + minimumGeneration + " is negative"); + throw new IllegalArgumentException("sequence number " + lowerBound + " is negative"); } - if (minimumGeneration < 0) { - throw new IllegalArgumentException("minimum generation " + minimumGeneration + " is negative"); + if (lowerBound < 0) { + throw new IllegalArgumentException("lower bound " + lowerBound + " is negative"); } - if (maximumGeneration < 0) { - throw new IllegalArgumentException("maximum generation " + maximumGeneration + " is negative"); + if (upperBound < 0) { + throw new IllegalArgumentException("upper bound " + upperBound + " is negative"); } if (origin < 0) { throw new IllegalArgumentException("origin " + origin + " is negative"); } - if (maximumGeneration < minimumGeneration) { - throw new IllegalArgumentException("maximum generation " + maximumGeneration - + " is less than minimum generation " + minimumGeneration); + if (upperBound < lowerBound) { + throw new IllegalArgumentException( + "upper bound " + upperBound + " is less than the lower bound " + lowerBound); } + this.fileType = Objects.requireNonNull(fileType); this.sequenceNumber = sequenceNumber; - this.minimumGeneration = minimumGeneration; - this.maximumGeneration = maximumGeneration; + this.lowerBound = lowerBound; + this.upperBound = upperBound; this.origin = origin; this.timestamp = Objects.requireNonNull(timestamp); this.path = Objects.requireNonNull(path); @@ -173,28 +205,32 @@ private PcesFile( /** * Create a new event file descriptor. * - * @param timestamp the timestamp when this file was created (wall clock time) - * @param sequenceNumber the sequence number of the descriptor - * @param minimumGeneration the minimum event generation permitted to be in this file (inclusive) - * @param maximumGeneration the maximum event generation permitted to be in this file (inclusive) - * @param origin the origin round number, i.e. the round after which the stream is unbroken - * @param rootDirectory the directory where event stream files are stored + * @param fileType the type of this PCES file + * @param timestamp the timestamp when this file was created (wall clock time) + * @param sequenceNumber the sequence number of the descriptor + * @param minimumBound the minimum event bound permitted to be in this file (inclusive), either a generation or a + * birth round depending on the {@link AncientMode}. + * @param maximumBound the maximum event bound permitted to be in this file (inclusive), either a generation or a + * birth round depending on the {@link AncientMode}. + * @param origin the origin round number, i.e. the round after which the stream is unbroken + * @param rootDirectory the directory where event stream files are stored * @return a description of the file */ @NonNull public static PcesFile of( + @NonNull final AncientMode fileType, @NonNull final Instant timestamp, final long sequenceNumber, - final long minimumGeneration, - final long maximumGeneration, + final long minimumBound, + final long maximumBound, final long origin, @NonNull final Path rootDirectory) { final Path parentDirectory = buildParentDirectory(rootDirectory, timestamp); - final String fileName = buildFileName(timestamp, sequenceNumber, minimumGeneration, maximumGeneration, origin); + final String fileName = buildFileName(fileType, timestamp, sequenceNumber, minimumBound, maximumBound, origin); final Path path = parentDirectory.resolve(fileName); - return new PcesFile(timestamp, sequenceNumber, minimumGeneration, maximumGeneration, origin, path); + return new PcesFile(fileType, timestamp, sequenceNumber, minimumBound, maximumBound, origin, path); } /** @@ -213,6 +249,7 @@ public static PcesFile of(@NonNull final Path filePath) throws IOException { } final String fileName = filePath.getFileName().toString(); + final AncientMode fileType = determineFileType(fileName); final String[] elements = fileName.substring(0, fileName.length() - EVENT_FILE_EXTENSION.length()) .split(EVENT_FILE_SEPARATOR); @@ -223,10 +260,11 @@ public static PcesFile of(@NonNull final Path filePath) throws IOException { try { return new PcesFile( + fileType, parseSanitizedTimestamp(elements[0]), Long.parseLong(elements[1].replace(SEQUENCE_NUMBER_PREFIX, "")), - Long.parseLong(elements[2].replace(MINIMUM_GENERATION_PREFIX, "")), - Long.parseLong(elements[3].replace(MAXIMUM_GENERATION_PREFIX, "")), + Long.parseLong(elements[2].replace(getLowerBoundPrefix(fileType), "")), + Long.parseLong(elements[3].replace(getUpperBoundPrefix(fileType), "")), Long.parseLong(elements[4].replace(ORIGIN_PREFIX, "")), filePath); } catch (final DateTimeParseException | IllegalArgumentException ex) { @@ -234,30 +272,72 @@ public static PcesFile of(@NonNull final Path filePath) throws IOException { } } + /** + * Determine the type of the pces file based on its file name. + * + * @param fileName the name of the file + * @return the type of the file + * @throws IOException if the file type could not be determined + */ + @NonNull + private static AncientMode determineFileType(@NonNull final String fileName) throws IOException { + if (fileName.contains(MINIMUM_GENERATION_PREFIX) && fileName.contains(MAXIMUM_GENERATION_PREFIX)) { + return GENERATION_THRESHOLD; + } else if (fileName.contains(MINIMUM_BIRTH_ROUND_PREFIX) && fileName.contains(MAXIMUM_BIRTH_ROUND_PREFIX)) { + return BIRTH_ROUND_THRESHOLD; + } else { + throw new IOException("Unable to determine file type from " + fileName); + } + } + + /** + * Get the prefix used to identify the lower bound in the file name. + * + * @param fileType the type of the file + * @return the prefix used to identify the lower bound in the file name + */ + @NonNull + private static String getLowerBoundPrefix(@NonNull final AncientMode fileType) { + return fileType == GENERATION_THRESHOLD ? MINIMUM_GENERATION_PREFIX : MINIMUM_BIRTH_ROUND_PREFIX; + } + + /** + * Get the prefix used to identify the upper bound in the file name. + * + * @param fileType the type of the file + * @return the prefix used to identify the upper bound in the file name + */ + @NonNull + private static String getUpperBoundPrefix(@NonNull final AncientMode fileType) { + return fileType == GENERATION_THRESHOLD ? MAXIMUM_GENERATION_PREFIX : MAXIMUM_BIRTH_ROUND_PREFIX; + } + /** * Create a new event file descriptor for span compaction. * - * @param maximumGenerationInFile the maximum generation that is actually in the file + * @param maximumBoundaryValueInFile the maximum boundary value for events actually present in the file. Will be + * either a generation or a birth round depending on the {@link AncientMode}. * @return a description of the new file */ @NonNull - public PcesFile buildFileWithCompressedSpan(final long maximumGenerationInFile) { - if (maximumGenerationInFile < minimumGeneration) { - throw new IllegalArgumentException("maximumGenerationInFile " + maximumGenerationInFile - + " is less than minimumGeneration " + minimumGeneration); + public PcesFile buildFileWithCompressedSpan(final long maximumBoundaryValueInFile) { + if (maximumBoundaryValueInFile < lowerBound) { + throw new IllegalArgumentException("maximumBoundaryValueInFile " + maximumBoundaryValueInFile + + " is less than lowerBound " + lowerBound); } - if (maximumGenerationInFile > maximumGeneration) { - throw new IllegalArgumentException("maximumGenerationInFile " + maximumGenerationInFile - + " is greater than maximumGeneration " + maximumGeneration); + if (maximumBoundaryValueInFile > upperBound) { + throw new IllegalArgumentException("maximumBoundaryValueInFile " + maximumBoundaryValueInFile + + " is greater than upperBound " + upperBound); } final Path parentDirectory = path.getParent(); final String fileName = - buildFileName(timestamp, sequenceNumber, minimumGeneration, maximumGenerationInFile, origin); + buildFileName(fileType, timestamp, sequenceNumber, lowerBound, maximumBoundaryValueInFile, origin); final Path newPath = parentDirectory.resolve(fileName); - return new PcesFile(timestamp, sequenceNumber, minimumGeneration, maximumGenerationInFile, origin, newPath); + return new PcesFile( + fileType, timestamp, sequenceNumber, lowerBound, maximumBoundaryValueInFile, origin, newPath); } /** @@ -277,17 +357,19 @@ public long getSequenceNumber() { } /** - * @return the minimum event generation permitted to be in this file (inclusive) + * @return the minimum event bound permitted to be in this file (inclusive), either a generation or a birth round + * depending on the {@link AncientMode}. */ - public long getMinimumGeneration() { - return minimumGeneration; + public long getLowerBound() { + return lowerBound; } /** - * @return the maximum event generation permitted to be in this file (inclusive) + * @return the maximum event generation permitted to be in this file (inclusive), either a generation or a birth + * round depending on the {@link AncientMode}. */ - public long getMaximumGeneration() { - return maximumGeneration; + public long getUpperBound() { + return upperBound; } /** @@ -370,15 +452,16 @@ public void deleteFile(@NonNull final Path rootDirectory, @Nullable final Recycl } /** - * Get an iterator that walks over the events in this file. The iterator will only return events that have a - * generation equal to or greater to the minimum generation. + * Get an iterator that walks over the events in this file. The iterator will only return events that have an + * ancient indicator that is greater than or equal to the lower bound. * - * @param minimumGeneration the minimum generation of the events to return + * @param lowerBound lower bound of the events to return, will be either a generation or a birth round depending on + * the {@link AncientMode}. * @return an iterator over the events in this file */ @NonNull - public PcesFileIterator iterator(final long minimumGeneration) throws IOException { - return new PcesFileIterator(this, minimumGeneration); + public PcesFileIterator iterator(final long lowerBound) throws IOException { + return new PcesFileIterator(this, lowerBound, fileType); } /** @@ -400,19 +483,21 @@ private static Path buildParentDirectory(@NonNull final Path rootDirectory, @Non /** * Derive the name for this file. * - * @param timestamp the timestamp of when the file was created - * @param sequenceNumber the sequence number of the file - * @param minimumGeneration the minimum generation of events permitted in this file - * @param maximumGeneration the maximum generation of events permitted in this file - * @param origin the origin round number, i.e. the round after which the stream is unbroken + * @param fileType the type of this PCES file + * @param timestamp the timestamp of when the file was created + * @param sequenceNumber the sequence number of the file + * @param minimumBound the minimum bound of events permitted in this file + * @param maximumBound the maximum bound of events permitted in this file + * @param origin the origin round number, i.e. the round after which the stream is unbroken * @return the file name */ @NonNull private static String buildFileName( + @NonNull final AncientMode fileType, @NonNull final Instant timestamp, final long sequenceNumber, - final long minimumGeneration, - final long maximumGeneration, + final long minimumBound, + final long maximumBound, final long origin) { return new StringBuilder(STRING_BUILDER_INITIAL_CAPACITY) @@ -421,11 +506,11 @@ private static String buildFileName( .append(SEQUENCE_NUMBER_PREFIX) .append(sequenceNumber) .append(EVENT_FILE_SEPARATOR) - .append(MINIMUM_GENERATION_PREFIX) - .append(minimumGeneration) + .append(getLowerBoundPrefix(fileType)) + .append(minimumBound) .append(EVENT_FILE_SEPARATOR) - .append(MAXIMUM_GENERATION_PREFIX) - .append(maximumGeneration) + .append(getUpperBoundPrefix(fileType)) + .append(maximumBound) .append(EVENT_FILE_SEPARATOR) .append(ORIGIN_PREFIX) .append(origin) @@ -443,14 +528,25 @@ public String getFileName() { return path.getFileName().toString(); } + /** + * Get the type of this file. + * + * @return the type of this file + */ + @NonNull + public AncientMode getFileType() { + return fileType; + } + /** * Check if it is legal for the file described by this object to contain a particular event. * - * @param generation the generation of the event in question + * @param eventSequenceNumber a sequence number that describes which file an event should be in, either the + * generation or a birth round of the event depending on the {@link AncientMode}. * @return true if it is legal for this event to be in the file described by this object */ - public boolean canContain(final long generation) { - return generation >= minimumGeneration && generation <= maximumGeneration; + public boolean canContain(final long eventSequenceNumber) { + return eventSequenceNumber >= lowerBound && eventSequenceNumber <= upperBound; } /** diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFileIterator.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFileIterator.java index 3eb061153a3e..d89eb00f6040 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFileIterator.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFileIterator.java @@ -20,19 +20,23 @@ import com.swirlds.common.io.extendable.ExtendableInputStream; import com.swirlds.common.io.extendable.extensions.CountingStreamExtension; import com.swirlds.common.io.streams.SerializableDataInputStream; +import com.swirlds.platform.event.AncientMode; import com.swirlds.platform.event.GossipEvent; +import edu.umd.cs.findbugs.annotations.NonNull; import java.io.BufferedInputStream; import java.io.EOFException; import java.io.FileInputStream; import java.io.IOException; import java.util.NoSuchElementException; +import java.util.Objects; /** * Iterates over the events in a single preconsensus event file. */ public class PcesFileIterator implements IOIterator { - private final long minimumGeneration; + private final long lowerBound; + private final AncientMode fileType; private final SerializableDataInputStream stream; private boolean hasPartialEvent = false; private final CountingStreamExtension counter; @@ -42,15 +46,17 @@ public class PcesFileIterator implements IOIterator { /** * Create a new iterator that walks over events in a preconsensus event file. * - * @param fileDescriptor - * describes a preconsensus event file - * @param minimumGeneration - * the minimum generation to return, any events in the file with a smaller - * generation are ignored and not returned + * @param fileDescriptor describes a preconsensus event file + * @param lowerBound the lower bound for all events to be returned, corresponds to either generation or birth + * round depending on the {@link PcesFile} type + * @param fileType the type of file to read */ - public PcesFileIterator(final PcesFile fileDescriptor, final long minimumGeneration) throws IOException { + public PcesFileIterator( + @NonNull final PcesFile fileDescriptor, final long lowerBound, @NonNull final AncientMode fileType) + throws IOException { - this.minimumGeneration = minimumGeneration; + this.lowerBound = lowerBound; + this.fileType = Objects.requireNonNull(fileType); counter = new CountingStreamExtension(); stream = new SerializableDataInputStream(new ExtendableInputStream( new BufferedInputStream( @@ -79,7 +85,7 @@ private void findNext() throws IOException { try { final GossipEvent candidate = stream.readSerializable(false, GossipEvent::new); - if (candidate.getGeneration() >= minimumGeneration) { + if (candidate.getAncientIndicator(fileType) >= lowerBound) { next = candidate; } } catch (final EOFException e) { @@ -95,8 +101,8 @@ private void findNext() throws IOException { } /** - * If true then this file contained a partial event. If false then the last event in the file was fully written - * when the file was closed. + * If true then this file contained a partial event. If false then the last event in the file was fully written when + * the file was closed. */ public boolean hasPartialEvent() { return hasPartialEvent; @@ -115,6 +121,7 @@ public boolean hasNext() throws IOException { * {@inheritDoc} */ @Override + @NonNull public GossipEvent next() throws IOException { if (!hasNext()) { throw new NoSuchElementException("no files remain in this iterator"); diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFileManager.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFileManager.java index d991e77aa57e..e0a28d27dcf8 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFileManager.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFileManager.java @@ -17,12 +17,16 @@ package com.swirlds.platform.event.preconsensus; import static com.swirlds.logging.legacy.LogMarker.STARTUP; +import static com.swirlds.platform.event.AncientMode.BIRTH_ROUND_THRESHOLD; +import static com.swirlds.platform.event.AncientMode.GENERATION_THRESHOLD; import static com.swirlds.platform.event.preconsensus.PcesUtilities.getDatabaseDirectory; import com.swirlds.base.time.Time; import com.swirlds.base.units.UnitConstants; import com.swirlds.common.context.PlatformContext; import com.swirlds.common.platform.NodeId; +import com.swirlds.platform.event.AncientMode; +import com.swirlds.platform.eventhandling.EventConfig; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.nio.file.Files; @@ -47,9 +51,9 @@ public class PcesFileManager { private static final Logger logger = LogManager.getLogger(PcesFileManager.class); /** - * This constant can be used when the caller wants all events, regardless of generation. + * This constant can be used when the caller wants all events, regardless of the lower bound. */ - public static final long NO_MINIMUM_GENERATION = -1; + public static final long NO_LOWER_BOUND = -1; /** * Provides the wall clock time. @@ -80,11 +84,15 @@ public class PcesFileManager { private final PcesFileTracker files; + /** + * The PCES file type for new files. + */ + private final AncientMode newFileType; + /** * Constructor * * @param platformContext the platform context for this node - * @param time provides wall clock time * @param files the files to track * @param selfId the ID of this node * @param startingRound the round number of the initial state of the system @@ -92,7 +100,6 @@ public class PcesFileManager { */ public PcesFileManager( @NonNull final PlatformContext platformContext, - @NonNull final Time time, @NonNull final PcesFileTracker files, @NonNull final NodeId selfId, final long startingRound) @@ -108,7 +115,7 @@ public PcesFileManager( final PcesConfig preconsensusEventStreamConfig = platformContext.getConfiguration().getConfigData(PcesConfig.class); - this.time = Objects.requireNonNull(time); + this.time = platformContext.getTime(); this.files = Objects.requireNonNull(files); this.metrics = new PcesMetrics(platformContext.getMetrics()); this.minimumRetentionPeriod = preconsensusEventStreamConfig.minimumRetentionPeriod(); @@ -116,6 +123,13 @@ public PcesFileManager( this.currentOrigin = PcesUtilities.getInitialOrigin(files, startingRound); + this.newFileType = platformContext + .getConfiguration() + .getConfigData(EventConfig.class) + .useBirthRoundAncientThreshold() + ? BIRTH_ROUND_THRESHOLD + : GENERATION_THRESHOLD; + initializeMetrics(); } @@ -126,15 +140,15 @@ private void initializeMetrics() throws IOException { totalFileByteCount = files.getTotalFileByteCount(); if (files.getFileCount() > 0) { - metrics.getPreconsensusEventFileOldestGeneration() - .set(files.getFirstFile().getMinimumGeneration()); - metrics.getPreconsensusEventFileYoungestGeneration() - .set(files.getLastFile().getMaximumGeneration()); + metrics.getPreconsensusEventFileOldestIdentifier() + .set(files.getFirstFile().getLowerBound()); + metrics.getPreconsensusEventFileYoungestIdentifier() + .set(files.getLastFile().getUpperBound()); final Duration age = Duration.between(files.getFirstFile().getTimestamp(), time.now()); metrics.getPreconsensusEventFileOldestSeconds().set(age.toSeconds()); } else { - metrics.getPreconsensusEventFileOldestGeneration().set(NO_MINIMUM_GENERATION); - metrics.getPreconsensusEventFileYoungestGeneration().set(NO_MINIMUM_GENERATION); + metrics.getPreconsensusEventFileOldestIdentifier().set(NO_LOWER_BOUND); + metrics.getPreconsensusEventFileYoungestIdentifier().set(NO_LOWER_BOUND); metrics.getPreconsensusEventFileOldestSeconds().set(0); } updateFileSizeMetrics(); @@ -180,36 +194,35 @@ public void registerDiscontinuity(final long newOriginRound) { * Create a new event file descriptor for the next event file, and start tracking it. (Note, this method doesn't * actually open the file, it just permits the file to be opened by the caller.) * - * @param minimumGeneration the minimum generation that can be stored in the file - * @param maximumGeneration the maximum generation that can be stored in the file + * @param lowerBound the lower bound that can be stored in the file + * @param upperBound the upper bound that can be stored in the file * @return a new event file descriptor */ - public @NonNull PcesFile getNextFileDescriptor(final long minimumGeneration, final long maximumGeneration) { + public @NonNull PcesFile getNextFileDescriptor(final long lowerBound, final long upperBound) { - if (minimumGeneration > maximumGeneration) { - throw new IllegalArgumentException("minimum generation must be less than or equal to maximum generation"); + if (lowerBound > upperBound) { + throw new IllegalArgumentException("lower bound must be less than or equal to the upper bound"); } - final long minimumGenerationForFile; - final long maximumGenerationForFile; + final long lowerBoundForFile; + final long upperBoundForFile; if (files.getFileCount() == 0) { // This is the first file - minimumGenerationForFile = minimumGeneration; - maximumGenerationForFile = maximumGeneration; + lowerBoundForFile = lowerBound; + upperBoundForFile = upperBound; } else { // This is not the first file, min/max values are constrained to only increase - minimumGenerationForFile = - Math.max(minimumGeneration, files.getLastFile().getMinimumGeneration()); - maximumGenerationForFile = - Math.max(maximumGeneration, files.getLastFile().getMaximumGeneration()); + lowerBoundForFile = Math.max(lowerBound, files.getLastFile().getLowerBound()); + upperBoundForFile = Math.max(upperBound, files.getLastFile().getUpperBound()); } final PcesFile descriptor = PcesFile.of( + newFileType, time.now(), getNextSequenceNumber(), - minimumGenerationForFile, - maximumGenerationForFile, + lowerBoundForFile, + upperBoundForFile, currentOrigin, databaseDirectory); @@ -220,15 +233,15 @@ public void registerDiscontinuity(final long newOriginRound) { PcesUtilities.fileSanityChecks( false, previousFile.getSequenceNumber(), - previousFile.getMinimumGeneration(), - previousFile.getMaximumGeneration(), + previousFile.getLowerBound(), + previousFile.getUpperBound(), currentOrigin, previousFile.getTimestamp(), descriptor); } files.addFile(descriptor); - metrics.getPreconsensusEventFileYoungestGeneration().set(descriptor.getMaximumGeneration()); + metrics.getPreconsensusEventFileYoungestIdentifier().set(descriptor.getUpperBound()); return descriptor; } @@ -239,40 +252,40 @@ public void registerDiscontinuity(final long newOriginRound) { * @param file the file that has been completely written */ public void finishedWritingFile(@NonNull final PcesMutableFile file) { - final long previousFileHighestGeneration; + final long previousFileUpperBound; if (files.getFileCount() == 1) { - previousFileHighestGeneration = 0; + previousFileUpperBound = 0; } else { - previousFileHighestGeneration = - files.getFile(files.getFileCount() - 2).getMaximumGeneration(); + previousFileUpperBound = files.getFile(files.getFileCount() - 2).getUpperBound(); } - // Compress the generational span of the file. Reduces overlap between files. - final PcesFile compressedDescriptor = file.compressGenerationalSpan(previousFileHighestGeneration); + // Compress the span of the file. Reduces overlap between files. + final PcesFile compressedDescriptor = file.compressSpan(previousFileUpperBound); files.setFile(files.getFileCount() - 1, compressedDescriptor); // Update metrics totalFileByteCount += file.fileSize(); metrics.getPreconsensusEventFileRate().cycle(); - metrics.getPreconsensusEventAverageFileSpan().update(file.getGenerationalSpan()); - metrics.getPreconsensusEventAverageUnUtilizedFileSpan().update(file.getUnUtilizedGenerationalSpan()); + metrics.getPreconsensusEventAverageFileSpan().update(file.getSpan()); + metrics.getPreconsensusEventAverageUnUtilizedFileSpan().update(file.getUnUtilizedSpan()); updateFileSizeMetrics(); } /** * Prune old event files. Files are pruned if they are too old AND if they do not contain events with high enough - * generations. + * ancient indicators. * - * @param minimumGeneration the minimum generation that we need to keep in the database. It's possible that this - * operation won't delete all files with events older than this value, but this operation - * is guaranteed not to delete any files that may contain events with a higher generation. + * @param lowerBoundToKeep the minimum ancient indicator that we need to keep in this store. It's possible that + * this operation won't delete all files with events older than this value, but this + * operation is guaranteed not to delete any files that may contain events with a higher + * ancient indicator. * @throws IOException if there is an error deleting files */ - public void pruneOldFiles(final long minimumGeneration) throws IOException { + public void pruneOldFiles(final long lowerBoundToKeep) throws IOException { final Instant minimumTimestamp = time.now().minus(minimumRetentionPeriod); while (files.getFileCount() > 0 - && files.getFirstFile().getMaximumGeneration() < minimumGeneration + && files.getFirstFile().getUpperBound() < lowerBoundToKeep && files.getFirstFile().getTimestamp().isBefore(minimumTimestamp)) { final PcesFile file = files.removeFirstFile(); @@ -281,8 +294,8 @@ public void pruneOldFiles(final long minimumGeneration) throws IOException { } if (files.getFileCount() > 0) { - metrics.getPreconsensusEventFileOldestGeneration() - .set(files.getFirstFile().getMinimumGeneration()); + metrics.getPreconsensusEventFileOldestIdentifier() + .set(files.getFirstFile().getLowerBound()); final Duration age = Duration.between(files.getFirstFile().getTimestamp(), time.now()); metrics.getPreconsensusEventFileOldestSeconds().set(age.toSeconds()); } diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFileReader.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFileReader.java index 13f16c6bbcf3..14f2583cc4a5 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFileReader.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFileReader.java @@ -23,6 +23,7 @@ import com.swirlds.common.context.PlatformContext; import com.swirlds.common.io.utility.RecycleBin; import com.swirlds.common.utility.ValueReference; +import com.swirlds.platform.event.AncientMode; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.nio.file.Files; @@ -53,6 +54,7 @@ private PcesFileReader() {} * @param databaseDirectory the directory to scan for files * @param startingRound the round to start reading from * @param permitGaps if gaps are permitted in sequence number + * @param typeToRead the type of file to read, files of other types will be ignored * @return the files read from disk * @throws IOException if there is an error reading the files */ @@ -61,29 +63,31 @@ public static PcesFileTracker readFilesFromDisk( @NonNull final RecycleBin recycleBin, @NonNull final Path databaseDirectory, final long startingRound, - final boolean permitGaps) + final boolean permitGaps, + final AncientMode typeToRead) throws IOException { Objects.requireNonNull(platformContext); Objects.requireNonNull(databaseDirectory); - final PcesFileTracker files = new PcesFileTracker(); + final PcesFileTracker files = new PcesFileTracker(typeToRead); try (final Stream fileStream = Files.walk(databaseDirectory)) { fileStream .filter(f -> !Files.isDirectory(f)) .map(PcesUtilities::parseFile) .filter(Objects::nonNull) + .filter(f -> f.getFileType() == typeToRead) .sorted() .forEachOrdered(buildFileHandler(files, permitGaps)); } final PcesConfig preconsensusEventStreamConfig = platformContext.getConfiguration().getConfigData(PcesConfig.class); - final boolean doInitialGenerationalCompaction = preconsensusEventStreamConfig.compactLastFileOnStartup(); + final boolean doInitialSpanCompaction = preconsensusEventStreamConfig.compactLastFileOnStartup(); - if (files.getFileCount() != 0 && doInitialGenerationalCompaction) { - compactGenerationalSpanOfLastFile(files); + if (files.getFileCount() != 0 && doInitialSpanCompaction) { + compactSpanOfLastFile(files); } resolveDiscontinuities(databaseDirectory, recycleBin, files, startingRound); @@ -93,22 +97,22 @@ public static PcesFileTracker readFilesFromDisk( /** * It's possible (if not probable) that the node was shut down prior to the last file being closed and having its - * generational span compaction. This method performs that compaction if necessary. + * span compaction completed. This method performs that compaction if necessary. */ - private static void compactGenerationalSpanOfLastFile(@NonNull final PcesFileTracker files) { + private static void compactSpanOfLastFile(@NonNull final PcesFileTracker files) { Objects.requireNonNull(files); final PcesFile lastFile = files.getFile(files.getFileCount() - 1); - final long previousMaximumGeneration; + final long previousMaximumBound; if (files.getFileCount() > 1) { final PcesFile secondToLastFile = files.getFile(files.getFileCount() - 2); - previousMaximumGeneration = secondToLastFile.getMaximumGeneration(); + previousMaximumBound = secondToLastFile.getUpperBound(); } else { - previousMaximumGeneration = 0; + previousMaximumBound = 0; } - final PcesFile compactedFile = compactPreconsensusEventFile(lastFile, previousMaximumGeneration); + final PcesFile compactedFile = compactPreconsensusEventFile(lastFile, previousMaximumBound); files.setFile(files.getFileCount() - 1, compactedFile); } @@ -122,8 +126,8 @@ private static void compactGenerationalSpanOfLastFile(@NonNull final PcesFileTra @NonNull private static Consumer buildFileHandler(@NonNull final PcesFileTracker files, final boolean permitGaps) { final ValueReference previousSequenceNumber = new ValueReference<>(-1L); - final ValueReference previousMinimumGeneration = new ValueReference<>(-1L); - final ValueReference previousMaximumGeneration = new ValueReference<>(-1L); + final ValueReference previousMinimumBound = new ValueReference<>(-1L); + final ValueReference previousMaximumBound = new ValueReference<>(-1L); final ValueReference previousOrigin = new ValueReference<>(-1L); final ValueReference previousTimestamp = new ValueReference<>(); @@ -132,16 +136,16 @@ private static Consumer buildFileHandler(@NonNull final PcesFileTracke fileSanityChecks( permitGaps, previousSequenceNumber.getValue(), - previousMinimumGeneration.getValue(), - previousMaximumGeneration.getValue(), + previousMinimumBound.getValue(), + previousMaximumBound.getValue(), previousOrigin.getValue(), previousTimestamp.getValue(), descriptor); } previousSequenceNumber.setValue(descriptor.getSequenceNumber()); - previousMinimumGeneration.setValue(descriptor.getMinimumGeneration()); - previousMaximumGeneration.setValue(descriptor.getMaximumGeneration()); + previousMinimumBound.setValue(descriptor.getLowerBound()); + previousMaximumBound.setValue(descriptor.getUpperBound()); previousTimestamp.setValue(descriptor.getTimestamp()); // If the sequence number is good then add it to the collection of tracked files diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFileTracker.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFileTracker.java index 4941126f44b9..7812db42a27f 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFileTracker.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesFileTracker.java @@ -17,10 +17,11 @@ package com.swirlds.platform.event.preconsensus; import static com.swirlds.logging.legacy.LogMarker.STARTUP; -import static com.swirlds.platform.event.preconsensus.PcesFileManager.NO_MINIMUM_GENERATION; +import static com.swirlds.platform.event.preconsensus.PcesFileManager.NO_LOWER_BOUND; import com.swirlds.common.utility.RandomAccessDeque; import com.swirlds.common.utility.UnmodifiableIterator; +import com.swirlds.platform.event.AncientMode; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.nio.file.Files; @@ -46,6 +47,17 @@ public class PcesFileTracker { */ private final RandomAccessDeque files = new RandomAccessDeque<>(INITIAL_RING_BUFFER_SIZE); + private final AncientMode fileType; + + /** + * Constructor. + * + * @param fileType the type of file to track + */ + public PcesFileTracker(@NonNull final AncientMode fileType) { + this.fileType = Objects.requireNonNull(fileType); + } + /** * Get the first file in the file list. * @@ -124,6 +136,7 @@ public void addFile(@NonNull final PcesFile file) { * @param index the index of the file to get * @return the file at the specified index */ + @NonNull public PcesFile getFile(final int index) { return files.get(index); } @@ -140,20 +153,22 @@ public void setFile(final int index, @NonNull final PcesFile file) { } /** - * Get an iterator that walks over all events starting with a specified generation. + * Get an iterator that walks over all events starting with a specified lower bound. *

* Note: this method only works at system startup time, using this iterator after startup has undefined behavior. A * future task will be to enable event iteration after startup. * - * @param minimumGeneration the desired minimum generation, iterator is guaranteed to return all available events - * with a generation greater or equal to this value. No events with a smaller generation - * will be returned. A value of {@link PcesFileManager ::NO_MINIMUM_GENERATION} - * will cause the returned iterator to walk over all available events. - * @param startingRound the round to start iterating from + * @param lowerBound the desired lower bound, iterator is guaranteed to return all available events with an + * ancient indicator (i.e. a generation or a birth round depending on the + * {@link AncientMode}) greater or equal to this value. No events with a smaller ancient + * identifier will be returned. A value of {@link PcesFileManager#NO_LOWER_BOUND} will cause + * the returned iterator to walk over all available events. + * @param startingRound the round to start iterating from * @return an iterator that walks over events */ - public @NonNull PcesMultiFileIterator getEventIterator(final long minimumGeneration, final long startingRound) { - return new PcesMultiFileIterator(minimumGeneration, getFileIterator(minimumGeneration, startingRound)); + @NonNull + public PcesMultiFileIterator getEventIterator(final long lowerBound, final long startingRound) { + return new PcesMultiFileIterator(lowerBound, getFileIterator(lowerBound, startingRound), fileType); } /** @@ -162,18 +177,21 @@ public void setFile(final int index, @NonNull final PcesFile file) { * Note: this method only works at system startup time, using this iterator after startup has undefined behavior. A * future task will be to enable event iteration after startup. * - * @param minimumGeneration the desired minimum generation, iterator is guaranteed to walk over all files that may - * contain events with a generation greater or equal to this value. A value of - * {@link PcesFileManager#NO_MINIMUM_GENERATION} will cause the returned - * iterator to walk over all available event files. - * @param startingRound the round to start iterating from + * @param lowerBound the desired lower bound, iterator is guaranteed to walk over all files that may contain events + * with an ancient indicator (i.e. a generation or birth round depending on the + * {@link AncientMode}) greater or equal to this value. A value of + * {@link PcesFileManager#NO_LOWER_BOUND} will cause the returned iterator to walk over all + * available event files. + * @param originRound the origin round to start iterating from. The origin of a PCES segment is used to + * differentiate segments of PCES files separated by discontinuities. * @return an unmodifiable iterator that walks over event files in order */ - public @NonNull Iterator getFileIterator(final long minimumGeneration, final long startingRound) { - final int firstFileIndex = getFirstRelevantFileIndex(startingRound); + @NonNull + public Iterator getFileIterator(final long lowerBound, final long originRound) { + final int firstFileIndex = getFirstRelevantFileIndex(originRound); - // Edge case: we want all events regardless of generation - if (minimumGeneration == NO_MINIMUM_GENERATION) { + // Edge case: we want all events regardless of lower bound + if (lowerBound == NO_LOWER_BOUND) { return new UnmodifiableIterator<>(files.iterator(firstFileIndex)); } @@ -183,29 +201,29 @@ public void setFile(final int index, @NonNull final PcesFile file) { return Collections.emptyIterator(); } - // Edge case: our first file comes after the requested starting generation - if (files.get(firstFileIndex).getMinimumGeneration() >= minimumGeneration) { - // Unless we observe at least one file with a minimum generation less than the requested minimum, - // then we can't know for certain that we have all data for the requested minimum generation. + // Edge case: our first file comes after the requested lower bound + if (files.get(firstFileIndex).getLowerBound() >= lowerBound) { + // Unless we observe at least one file with a lower bound less than the requested minimum, + // then we can't know for certain that we have all data for the requested lower bound. logger.warn( STARTUP.getMarker(), "The preconsensus event stream has insufficient data to guarantee that all events with the " - + "requested generation of {} are present, the first file has a minimum generation of {}", - minimumGeneration, - files.getFirst().getMinimumGeneration()); + + "requested lower bound of {} are present, the first file has a lower bound of {}", + lowerBound, + files.getFirst().getLowerBound()); return new UnmodifiableIterator<>(files.iterator(firstFileIndex)); } - // Edge case: all of our data comes before the requested starting generation - if (files.getLast().getMaximumGeneration() < minimumGeneration) { + // Edge case: all of our data comes before the requested lower bound + if (files.getLast().getUpperBound() < lowerBound) { logger.warn( STARTUP.getMarker(), "The preconsensus event stream has insufficient data to guarantee that " - + "all events with the requested minimum generation of {} are present, " - + "the last file has a maximum generation of {}", - minimumGeneration, - files.getLast().getMaximumGeneration()); + + "all events with the requested lower bound of {} are present, " + + "the last file has a lower bound of {}", + lowerBound, + files.getLast().getUpperBound()); return Collections.emptyIterator(); } @@ -213,14 +231,14 @@ public void setFile(final int index, @NonNull final PcesFile file) { final int fileCount = files.size(); for (int index = firstFileIndex; index < fileCount; index++) { final PcesFile file = files.get(index); - if (file.getMaximumGeneration() >= minimumGeneration) { - // We have found the first file that may contain events at the requested generation. + if (file.getUpperBound() >= lowerBound) { + // We have found the first file that may contain events at the requested lower bound. return new UnmodifiableIterator<>(files.iterator(index)); } } // It should not be possible to reach this point. - throw new IllegalStateException("Failed to find a file that may contain events at the requested generation"); + throw new IllegalStateException("Failed to find a file that may contain events at the requested lower bound"); } /** @@ -230,10 +248,10 @@ public void setFile(final int index, @NonNull final PcesFile file) { * If no file is compatible with the starting round, return -1. If there are no compatible files, that means there * are either no files, or all files have an origin that exceeds the starting round. * - * @param startingRound the round to start streaming from + * @param originRound the origin round to start streaming from * @return the index of the first file to consider for a given starting round */ - public int getFirstRelevantFileIndex(final long startingRound) { + public int getFirstRelevantFileIndex(final long originRound) { // When streaming from the preconsensus event stream, we need to start at // the file with the largest origin that does not exceed the starting round. @@ -243,7 +261,7 @@ public int getFirstRelevantFileIndex(final long startingRound) { for (int index = 0; index < files.size(); index++) { final long fileOrigin = files.get(index).getOrigin(); - if (fileOrigin > startingRound) { + if (fileOrigin > originRound) { // Once we find the first file with an origin that exceeds the starting round, we can stop searching. // File origins only increase, so we know that all files after this one will also exceed the round. return candidateIndex; diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesMetrics.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesMetrics.java index 7ee5499c3232..0e4fe2db98a4 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesMetrics.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesMetrics.java @@ -55,30 +55,27 @@ public class PcesMetrics { private static final RunningAverageMetric.Config PRECONSENSUS_EVENT_AVERAGE_FILE_SPAN_CONFIG = new RunningAverageMetric.Config(CATEGORY, "preconsensusEventAverageFileSpan") - .withUnit("generations") - .withDescription("The average generational span of preconsensus event files. Only reflects" + .withDescription("The average span of preconsensus event files. Only reflects" + "files written since the last restart."); private final RunningAverageMetric preconsensusEventAverageFileSpan; private static final RunningAverageMetric.Config PRECONSENSUS_EVENT_AVERAGE_UN_UTILIZED_FILE_SPAN_CONFIG = new RunningAverageMetric.Config(CATEGORY, "preconsensusEventAverageUnutilizedFileSpan") - .withUnit("generations") .withDescription( - "The average unutilized generational span of preconsensus event files prior " + "The average unutilized span of preconsensus event files prior " + "to span compaction. Only reflects files written since the last restart. Smaller is better."); private final RunningAverageMetric preconsensusEventAverageUnUtilizedFileSpan; - private static final LongGauge.Config PRECONSENSUS_EVENT_FILE_OLDEST_GENERATION_CONFIG = new LongGauge.Config( - CATEGORY, "preconsensusEventFileOldestGeneration") - .withUnit("generation") - .withDescription("The oldest possible generation that is being " + "stored in preconsensus event files."); - private final LongGauge preconsensusEventFileOldestGeneration; + private static final LongGauge.Config PRECONSENSUS_EVENT_FILE_OLDEST_IDENTIFIER_CONFIG = new LongGauge.Config( + CATEGORY, "preconsensusEventFileOldestIdentifier") + .withDescription("The oldest possible ancient indicator that is being stored in preconsensus event files."); + private final LongGauge preconsensusEventFileOldestIdentifier; - private static final LongGauge.Config PRECONSENSUS_EVENT_FILE_YOUNGEST_GENERATION_CONFIG = new LongGauge.Config( - CATEGORY, "preconsensusEventFileYoungestGeneration") - .withUnit("generation") - .withDescription("The youngest possible generation that is being " + "stored in preconsensus event files."); - private final LongGauge preconsensusEventFileYoungestGeneration; + private static final LongGauge.Config PRECONSENSUS_EVENT_FILE_YOUNGEST_IDENTIFIER_CONFIG = new LongGauge.Config( + CATEGORY, "preconsensusEventFileYoungestIdentifier") + .withDescription( + "The youngest possible ancient indicator that is being stored in preconsensus event files."); + private final LongGauge preconsensusEventFileYoungestIdentifier; private static final LongGauge.Config PRECONSENSUS_EVENT_FILE_OLDEST_SECONDS_CONFIG = new LongGauge.Config( CATEGORY, "preconsensusEventFileOldestSeconds") @@ -99,9 +96,9 @@ public PcesMetrics(final Metrics metrics) { preconsensusEventAverageFileSpan = metrics.getOrCreate(PRECONSENSUS_EVENT_AVERAGE_FILE_SPAN_CONFIG); preconsensusEventAverageUnUtilizedFileSpan = metrics.getOrCreate(PRECONSENSUS_EVENT_AVERAGE_UN_UTILIZED_FILE_SPAN_CONFIG); - preconsensusEventFileOldestGeneration = metrics.getOrCreate(PRECONSENSUS_EVENT_FILE_OLDEST_GENERATION_CONFIG); - preconsensusEventFileYoungestGeneration = - metrics.getOrCreate(PRECONSENSUS_EVENT_FILE_YOUNGEST_GENERATION_CONFIG); + preconsensusEventFileOldestIdentifier = metrics.getOrCreate(PRECONSENSUS_EVENT_FILE_OLDEST_IDENTIFIER_CONFIG); + preconsensusEventFileYoungestIdentifier = + metrics.getOrCreate(PRECONSENSUS_EVENT_FILE_YOUNGEST_IDENTIFIER_CONFIG); preconsensusEventFileOldestSeconds = metrics.getOrCreate(PRECONSENSUS_EVENT_FILE_OLDEST_SECONDS_CONFIG); } @@ -134,31 +131,31 @@ public SpeedometerMetric getPreconsensusEventFileRate() { } /** - * Get the metric tracking the average generational file span. + * Get the metric tracking the average file span. */ public RunningAverageMetric getPreconsensusEventAverageFileSpan() { return preconsensusEventAverageFileSpan; } /** - * Get the metric tracking the average un-utilized generational file span. + * Get the metric tracking the average un-utilized file span. */ public RunningAverageMetric getPreconsensusEventAverageUnUtilizedFileSpan() { return preconsensusEventAverageUnUtilizedFileSpan; } /** - * Get the metric tracking the oldest possible generation that is being stored in preconsensus event files. + * Get the metric tracking the oldest possible ancient indicator that is being stored in preconsensus event files. */ - public LongGauge getPreconsensusEventFileOldestGeneration() { - return preconsensusEventFileOldestGeneration; + public LongGauge getPreconsensusEventFileOldestIdentifier() { + return preconsensusEventFileOldestIdentifier; } /** - * Get the metric tracking the youngest possible generation that is being stored in preconsensus event files. + * Get the metric tracking the youngest possible ancient indicator that is being stored in preconsensus event files. */ - public LongGauge getPreconsensusEventFileYoungestGeneration() { - return preconsensusEventFileYoungestGeneration; + public LongGauge getPreconsensusEventFileYoungestIdentifier() { + return preconsensusEventFileYoungestIdentifier; } /** diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesMultiFileIterator.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesMultiFileIterator.java index 37ad408d8611..3b4b019a77fa 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesMultiFileIterator.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesMultiFileIterator.java @@ -17,6 +17,7 @@ package com.swirlds.platform.event.preconsensus; import com.swirlds.common.io.IOIterator; +import com.swirlds.platform.event.AncientMode; import com.swirlds.platform.event.GossipEvent; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; @@ -30,24 +31,28 @@ public class PcesMultiFileIterator implements IOIterator { private final Iterator fileIterator; + private final AncientMode fileType; private PcesFileIterator currentIterator; - private final long minimumGeneration; + private final long lowerBound; private GossipEvent next; private int truncatedFileCount = 0; /** * Create an iterator that walks over events in a series of event files. * - * @param minimumGeneration - * the minimum generation of events to return, events with lower - * generations are not returned - * @param fileIterator - * an iterator that walks over event files + * @param lowerBound the minimum ancient indicator of events to return, events with lower ancient indicators are + * not returned + * @param fileIterator an iterator that walks over event files + * @param fileType the type of file to read */ - public PcesMultiFileIterator(final long minimumGeneration, @NonNull final Iterator fileIterator) { + public PcesMultiFileIterator( + final long lowerBound, + @NonNull final Iterator fileIterator, + @NonNull final AncientMode fileType) { this.fileIterator = Objects.requireNonNull(fileIterator); - this.minimumGeneration = minimumGeneration; + this.lowerBound = lowerBound; + this.fileType = Objects.requireNonNull(fileType); } /** @@ -64,7 +69,7 @@ private void findNext() throws IOException { break; } - currentIterator = new PcesFileIterator(fileIterator.next(), minimumGeneration); + currentIterator = new PcesFileIterator(fileIterator.next(), lowerBound, fileType); } else { next = currentIterator.next(); } @@ -84,6 +89,7 @@ public boolean hasNext() throws IOException { * {@inheritDoc} */ @Override + @NonNull public GossipEvent next() throws IOException { if (!hasNext()) { throw new NoSuchElementException("iterator is empty, can not get next element"); @@ -96,8 +102,8 @@ public GossipEvent next() throws IOException { } /** - * Get the number of files that had partial event data at the end. This can happen if JVM is shut down - * abruptly while and event is being written to disk. + * Get the number of files that had partial event data at the end. This can happen if JVM is shut down abruptly + * while and event is being written to disk. * * @return the number of files that had partial event data at the end that have been encountered so far */ diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesMutableFile.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesMutableFile.java index 91381981147a..0a498380097b 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesMutableFile.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesMutableFile.java @@ -46,9 +46,9 @@ public class PcesMutableFile { private final CountingStreamExtension counter; /** - * The highest generation of all events written to the file. + * The highest ancient indicator of all events written to the file. */ - private long highestGenerationInFile; + private long highestAncientIdentifierInFile; /** * The output stream to write to. @@ -74,17 +74,17 @@ public class PcesMutableFile { new FileOutputStream(descriptor.getPath().toFile())), counter)); out.writeInt(FILE_VERSION); - highestGenerationInFile = descriptor.getMinimumGeneration(); + highestAncientIdentifierInFile = descriptor.getLowerBound(); } /** - * Check if this file is eligible to contain an event based on generational bounds. + * Check if this file is eligible to contain an event based on bounds. * - * @param generation the generation of the event in question + * @param ancientIdentifier the ancient indicator of the event in question * @return true if this file is eligible to contain the event */ - public boolean canContain(final long generation) { - return descriptor.canContain(generation); + public boolean canContain(final long ancientIdentifier) { + return descriptor.canContain(ancientIdentifier); } /** @@ -93,31 +93,32 @@ public boolean canContain(final long generation) { * @param event the event to write */ public void writeEvent(final GossipEvent event) throws IOException { - if (!descriptor.canContain(event.getGeneration())) { + if (!descriptor.canContain(event.getAncientIndicator(descriptor.getFileType()))) { throw new IllegalStateException( - "Cannot write event " + event.getHashedData().getHash() + " with generation " - + event.getGeneration() + " to file " + descriptor); + "Cannot write event " + event.getHashedData().getHash() + " with ancient indicator " + + event.getAncientIndicator(descriptor.getFileType()) + " to file " + descriptor); } out.writeSerializable(event, false); - highestGenerationInFile = Math.max(highestGenerationInFile, event.getGeneration()); + highestAncientIdentifierInFile = + Math.max(highestAncientIdentifierInFile, event.getAncientIndicator(descriptor.getFileType())); } /** * Atomically rename this file so that its un-utilized span is 0. * - * @param highestGenerationInPreviousFile the previous file's highest generation. Even if we are not utilizing the - * entire span of this file, we cannot reduce the highest generation so that - * it is smaller than the previous file's highest generation. + * @param upperBoundInPreviousFile the previous file's upper bound. Even if we are not utilizing the + * entire span of this file, we cannot reduce the upper bound so that + * it is smaller than the previous file's highest upper bound. * @return the new span compressed file */ - public PcesFile compressGenerationalSpan(final long highestGenerationInPreviousFile) { - if (highestGenerationInFile == descriptor.getMaximumGeneration()) { + public PcesFile compressSpan(final long upperBoundInPreviousFile) { + if (highestAncientIdentifierInFile == descriptor.getUpperBound()) { // No need to compress, we used the entire span. return descriptor; } final PcesFile newDescriptor = descriptor.buildFileWithCompressedSpan( - Math.max(highestGenerationInFile, highestGenerationInPreviousFile)); + Math.max(highestAncientIdentifierInFile, upperBoundInPreviousFile)); try { Files.move(descriptor.getPath(), newDescriptor.getPath(), StandardCopyOption.ATOMIC_MOVE); @@ -152,26 +153,26 @@ public long fileSize() { } /** - * Get the difference between the highest generation written to the file and the lowest legal generation for this - * file. Higher values mean that the maximum generation was chosen well. + * Get the difference between the highest ancient indicator written to the file and the lowest legal ancient indicator for this + * file. Higher values mean that the upper bound was chosen well. */ - public long getUtilizedGenerationalSpan() { - return highestGenerationInFile - descriptor.getMinimumGeneration(); + public long getUtilizedSpan() { + return highestAncientIdentifierInFile - descriptor.getLowerBound(); } /** - * Get the generational span that is unused in this file. Low values mean that the maximum generation was chosen + * Get the span that is unused in this file. Low values mean that the upperBound was chosen * well, resulting in less overlap between files. A value of 0 represents a "perfect" choice. */ - public long getUnUtilizedGenerationalSpan() { - return descriptor.getMaximumGeneration() - highestGenerationInFile; + public long getUnUtilizedSpan() { + return descriptor.getUpperBound() - highestAncientIdentifierInFile; } /** - * Get the span of generations that this file can legally contain. + * Get the span of ancient indicators that this file can legally contain. */ - public long getGenerationalSpan() { - return descriptor.getMaximumGeneration() - descriptor.getMinimumGeneration(); + public long getSpan() { + return descriptor.getUpperBound() - descriptor.getLowerBound(); } /** diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesUtilities.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesUtilities.java index 906b4cb4abfa..ecf700b8766c 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesUtilities.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesUtilities.java @@ -23,6 +23,7 @@ import com.swirlds.common.context.PlatformContext; import com.swirlds.common.io.IOIterator; import com.swirlds.common.platform.NodeId; +import com.swirlds.platform.event.AncientMode; import com.swirlds.platform.event.GossipEvent; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; @@ -48,24 +49,26 @@ public final class PcesUtilities { private PcesUtilities() {} /** - * Compact the generational span of a PCES file. + * Compact the span of a PCES file. * - * @param originalFile the file to compact - * @param previousMaximumGeneration the maximum generation of the previous PCES file, used to prevent using a - * smaller maximum generation than the previous file. + * @param originalFile the file to compact + * @param previousUpperBound the upper bound of the previous PCES file, used to prevent using a smaller upper bound + * than the previous file. * @return the new compacted PCES file. */ @NonNull public static PcesFile compactPreconsensusEventFile( - @NonNull final PcesFile originalFile, final long previousMaximumGeneration) { + @NonNull final PcesFile originalFile, final long previousUpperBound) { - // Find the maximum generation in the file. - long maxGeneration = originalFile.getMinimumGeneration(); - try (final IOIterator iterator = new PcesFileIterator(originalFile, 0)) { + final AncientMode fileType = originalFile.getFileType(); + + // Find the true upper bound in the file. + long newUpperBound = originalFile.getLowerBound(); + try (final IOIterator iterator = new PcesFileIterator(originalFile, 0, fileType)) { while (iterator.hasNext()) { final GossipEvent next = iterator.next(); - maxGeneration = Math.max(maxGeneration, next.getGeneration()); + newUpperBound = Math.max(newUpperBound, next.getAncientIndicator(fileType)); } } catch (final IOException e) { @@ -73,17 +76,17 @@ public static PcesFile compactPreconsensusEventFile( return originalFile; } - // Important: do not decrease the maximum generation below the value of the previous file's maximum generation. - maxGeneration = Math.max(maxGeneration, previousMaximumGeneration); + // Important: do not decrease the upper bound below the value of the previous file's upper bound. + newUpperBound = Math.max(newUpperBound, previousUpperBound); - if (maxGeneration == originalFile.getMaximumGeneration()) { + if (newUpperBound == originalFile.getUpperBound()) { // The file cannot have its span compacted any further. logger.info(STARTUP.getMarker(), "No span compaction necessary for {}", originalFile.getPath()); return originalFile; } - // Now, compact the generational span of the file using the newly discovered maximum generation. - final PcesFile newFile = originalFile.buildFileWithCompressedSpan(maxGeneration); + // Now, compact the span of the file using the newly discovered upper bound. + final PcesFile newFile = originalFile.buildFileWithCompressedSpan(newUpperBound); try { Files.move(originalFile.getPath(), newFile.getPath(), StandardCopyOption.ATOMIC_MOVE); } catch (final IOException e) { @@ -93,9 +96,9 @@ public static PcesFile compactPreconsensusEventFile( logger.info( STARTUP.getMarker(), - "Span compaction completed for {}, new maximum generation is {}", + "Span compaction completed for {}, new upper bound is {}", originalFile.getPath(), - maxGeneration); + newUpperBound); return newFile; } @@ -136,10 +139,10 @@ public static void compactPreconsensusEventFiles(@NonNull final Path rootPath) { logger.error(EXCEPTION.getMarker(), "Failed to walk directory tree {}", rootPath, e); } - long previousMaximumGeneration = 0; + long previousUpperBound = 0; for (final PcesFile file : files) { - final PcesFile compactedFile = compactPreconsensusEventFile(file, previousMaximumGeneration); - previousMaximumGeneration = compactedFile.getMaximumGeneration(); + final PcesFile compactedFile = compactPreconsensusEventFile(file, previousUpperBound); + previousUpperBound = compactedFile.getUpperBound(); } } @@ -147,20 +150,20 @@ public static void compactPreconsensusEventFiles(@NonNull final Path rootPath) { * Perform sanity checks on the properties of the next file in the sequence, to ensure that we maintain various * invariants. * - * @param permitGaps if gaps are permitted in sequence number - * @param previousSequenceNumber the sequence number of the previous file - * @param previousMinimumGeneration the minimum generation of the previous file - * @param previousMaximumGeneration the maximum generation of the previous file - * @param previousOrigin the origin round of the previous file - * @param previousTimestamp the timestamp of the previous file - * @param descriptor the descriptor of the next file + * @param permitGaps if gaps are permitted in sequence number + * @param previousSequenceNumber the sequence number of the previous file + * @param previousLowerBound the upper bound of the previous file + * @param previousUpperBound the lower bound of the previous file + * @param previousOrigin the origin round of the previous file + * @param previousTimestamp the timestamp of the previous file + * @param descriptor the descriptor of the next file * @throws IllegalStateException if any of the required invariants are violated by the next file */ public static void fileSanityChecks( final boolean permitGaps, final long previousSequenceNumber, - final long previousMinimumGeneration, - final long previousMaximumGeneration, + final long previousLowerBound, + final long previousUpperBound, final long previousOrigin, @NonNull final Instant previousTimestamp, @NonNull final PcesFile descriptor) { @@ -172,18 +175,18 @@ public static void fileSanityChecks( + descriptor.getSequenceNumber()); } - // Minimum generation may never decrease - if (descriptor.getMinimumGeneration() < previousMinimumGeneration) { - throw new IllegalStateException("Minimum generation must never decrease, file " + descriptor.getPath() - + " has a minimum generation that is less than the previous minimum generation of " - + previousMinimumGeneration); + // Lower bound may never decrease + if (descriptor.getLowerBound() < previousLowerBound) { + throw new IllegalStateException("Lower bound must never decrease, file " + descriptor.getPath() + + " has a lower bound that is less than the previous lower bound of " + + previousLowerBound); } - // Maximum generation may never decrease - if (descriptor.getMaximumGeneration() < previousMaximumGeneration) { - throw new IllegalStateException("Maximum generation must never decrease, file " + descriptor.getPath() - + " has a maximum generation that is less than the previous maximum generation of " - + previousMaximumGeneration); + // Upper bound may never decrease + if (descriptor.getUpperBound() < previousUpperBound) { + throw new IllegalStateException("Upper bound must never decrease, file " + descriptor.getPath() + + " has an upper bound that is less than the previous upper bound of " + + previousUpperBound); } // Timestamp must never decrease diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesWriter.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesWriter.java index ef8ab8d6f02d..8ad6b20c1084 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesWriter.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/event/preconsensus/PcesWriter.java @@ -22,7 +22,10 @@ import com.swirlds.common.context.PlatformContext; import com.swirlds.common.utility.LongRunningAverage; +import com.swirlds.platform.consensus.NonAncientEventWindow; +import com.swirlds.platform.event.AncientMode; import com.swirlds.platform.event.GossipEvent; +import com.swirlds.platform.eventhandling.EventConfig; import com.swirlds.platform.wiring.DoneStreamingPcesTrigger; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; @@ -50,10 +53,10 @@ public class PcesWriter { private PcesMutableFile currentMutableFile; /** - * The current minimum generation required to be considered non-ancient. Only read and written on the handle - * thread. + * The current minimum ancient indicator required to be considered non-ancient. Only read and written on the handle + * thread. Either a round or generation depending on the {@link AncientMode}. */ - private long minimumGenerationNonAncient = 0; + private long nonAncientBoundary = 0; /** * The desired file size, in megabytes. Is not a hard limit, it's possible that we may exceed this value by a small @@ -63,52 +66,52 @@ public class PcesWriter { private final int preferredFileSizeMegabytes; /** - * When creating a new file, make sure that it has at least this much generational capacity for events after the - * first event written to the file. + * When creating a new file, make sure that it has at least this much capacity between the upper bound and lower + * bound for events after the first event written to the file. */ - private final int minimumGenerationalCapacity; + private final int minimumSpan; /** - * The minimum generation that we are required to keep around. + * The minimum ancient indicator that we are required to keep around. Will be either a birth round or a generation, + * depending on the {@link AncientMode}. */ - private long minimumGenerationToStore; + private long minimumAncientIdentifierToStore; /** - * A running average of the generational span utilization in each file. Generational span utilization is defined as - * the difference between the highest generation of all events in the file and the minimum legal generation for that - * file. Higher generational utilization is always better, as it means that we have a lower un-utilized generational - * span. Un-utilized generational span is defined as the difference between the highest legal generation in a file - * and the highest actual generation of all events in the file. The reason why we want to minimize un-utilized - * generational span is to reduce the generational overlap between files, which in turn makes it faster to search - * for events with particular generations. The purpose of this running average is to intelligently choose the - * maximum generation for each new file to minimize un-utilized generational span while still meeting file size - * requirements. + * A running average of the span utilization in each file. Span utilization is defined as the difference between the + * highest ancient indicator of all events in the file and the minimum legal ancient indicator for that file. + * Higher utilization is always better, as it means that we have a lower un-utilized span. Un-utilized span is + * defined as the difference between the highest legal ancient indicator in a file and the highest actual ancient + * identifier of all events in the file. The reason why we want to minimize un-utilized span is to reduce the + * overlap between files, which in turn makes it faster to search for events with particular ancient indicator. The + * purpose of this running average is to intelligently choose upper bound for each new file to minimize un-utilized + * span while still meeting file size requirements. */ - private final LongRunningAverage averageGenerationalSpanUtilization; + private final LongRunningAverage averageSpanUtilization; /** - * The previous generational span. Set to a constant at bootstrap time. + * The previous span. Set to a constant at bootstrap time. */ - private long previousGenerationalSpan; + private long previousSpan; /** - * If true then use {@link #bootstrapGenerationalSpanOverlapFactor} to compute the maximum generation for new files. - * If false then use {@link #generationalSpanOverlapFactor} to compute the maximum generation for new files. - * Bootstrap mode is used until we create the first file that exceeds the preferred file size. + * If true then use {@link #bootstrapSpanOverlapFactor} to compute the upper bound new files. If false then use + * {@link #spanOverlapFactor} to compute the upper bound for new files. Bootstrap mode is used until we create the + * first file that exceeds the preferred file size. */ private boolean bootstrapMode = true; /** - * During bootstrap mode, multiply this value by the running average when deciding the generation span for a new - * file (i.e. the difference between the maximum and the minimum legal generation). + * During bootstrap mode, multiply this value by the running average when deciding the upper bound for a new file + * (i.e. the difference between the maximum and the minimum legal ancient indicator). */ - private final double bootstrapGenerationalSpanOverlapFactor; + private final double bootstrapSpanOverlapFactor; /** - * When not in boostrap mode, multiply this value by the running average when deciding the generation span for a new - * file (i.e. the difference between the maximum and the minimum legal generation). + * When not in boostrap mode, multiply this value by the running average when deciding the span for a new file (i.e. + * the difference between the maximum and the minimum legal ancient indicator). */ - private final double generationalSpanOverlapFactor; + private final double spanOverlapFactor; /** * The highest event sequence number that has been written to the stream (but possibly not yet flushed). @@ -116,11 +119,18 @@ public class PcesWriter { private long lastWrittenEvent = -1; /** - * If true then all added events are new and need to be written to the stream. If false then all added events - * are already durable and do not need to be written to the stream. + * If true then all added events are new and need to be written to the stream. If false then all added events are + * already durable and do not need to be written to the stream. */ private boolean streamingNewEvents = false; + /** + * The type of the PCES file. There are currently two types: one bound by generations and one bound by birth rounds. + * The original type of files are bound by generations. The new type of files are bound by birth rounds. Once + * migration has been completed to birth round bound files, support for the generation bound files will be removed. + */ + private final AncientMode fileType; + /** * Constructor * @@ -136,14 +146,20 @@ public PcesWriter(@NonNull final PlatformContext platformContext, @NonNull final preferredFileSizeMegabytes = config.preferredFileSizeMegabytes(); - averageGenerationalSpanUtilization = - new LongRunningAverage(config.generationalUtilizationSpanRunningAverageLength()); - previousGenerationalSpan = config.bootstrapGenerationalSpan(); - bootstrapGenerationalSpanOverlapFactor = config.bootstrapGenerationalSpanOverlapFactor(); - generationalSpanOverlapFactor = config.generationalSpanOverlapFactor(); - minimumGenerationalCapacity = config.minimumGenerationalCapacity(); + averageSpanUtilization = new LongRunningAverage(config.spanUtilizationRunningAverageLength()); + previousSpan = config.bootstrapSpan(); + bootstrapSpanOverlapFactor = config.bootstrapSpanOverlapFactor(); + spanOverlapFactor = config.spanOverlapFactor(); + minimumSpan = config.minimumSpan(); this.fileManager = fileManager; + + fileType = platformContext + .getConfiguration() + .getConfigData(EventConfig.class) + .useBirthRoundAncientThreshold() + ? AncientMode.BIRTH_ROUND_THRESHOLD + : AncientMode.GENERATION_THRESHOLD; } /** @@ -176,7 +192,7 @@ public Long writeEvent(@NonNull final GossipEvent event) { return lastWrittenEvent; } - if (event.getGeneration() < minimumGenerationNonAncient) { + if (event.getAncientIndicator(fileType) < nonAncientBoundary) { event.setStreamSequenceNumber(GossipEvent.STALE_EVENT_STREAM_SEQUENCE_NUMBER); return null; } @@ -229,21 +245,21 @@ private static void validateSequenceNumber(@NonNull final GossipEvent event) { } /** - * Let the event writer know the minimum generation for non-ancient events. Ancient events will be ignored if added - * to the event writer. + * Let the event writer know the current non-ancient event boundary. Ancient events will be ignored if added to the + * event writer. * - * @param minimumGenerationNonAncient the minimum generation of a non-ancient event + * @param nonAncientBoundary describes the boundary between ancient and non-ancient events * @return the sequence number of the last event durably written to the stream if this method call resulted in any * additional events being durably written to the stream, otherwise null */ @Nullable - public Long setMinimumGenerationNonAncient(final long minimumGenerationNonAncient) { - if (minimumGenerationNonAncient < this.minimumGenerationNonAncient) { - throw new IllegalArgumentException("Minimum generation non-ancient cannot be decreased. Current = " - + this.minimumGenerationNonAncient + ", requested = " + minimumGenerationNonAncient); + public Long updateNonAncientEventBoundary(@NonNull final NonAncientEventWindow nonAncientBoundary) { + if (nonAncientBoundary.getAncientThreshold() < this.nonAncientBoundary) { + throw new IllegalArgumentException("Non-ancient boundary cannot be decreased. Current = " + + this.nonAncientBoundary + ", requested = " + nonAncientBoundary); } - this.minimumGenerationNonAncient = minimumGenerationNonAncient; + this.nonAncientBoundary = nonAncientBoundary.getAncientThreshold(); if (!streamingNewEvents || currentMutableFile == null) { return null; @@ -258,12 +274,12 @@ public Long setMinimumGenerationNonAncient(final long minimumGenerationNonAncien } /** - * Set the minimum generation needed to be kept on disk. + * Set the minimum ancient indicator needed to be kept on disk. * - * @param minimumGenerationToStore the minimum generation required to be stored on disk + * @param minimumAncientIdentifierToStore the minimum ancient indicator required to be stored on disk */ - public void setMinimumGenerationToStore(final long minimumGenerationToStore) { - this.minimumGenerationToStore = minimumGenerationToStore; + public void setMinimumAncientIdentifierToStore(final long minimumAncientIdentifierToStore) { + this.minimumAncientIdentifierToStore = minimumAncientIdentifierToStore; pruneOldFiles(); } @@ -279,7 +295,7 @@ private void pruneOldFiles() { } try { - fileManager.pruneOldFiles(minimumGenerationToStore); + fileManager.pruneOldFiles(minimumAncientIdentifierToStore); } catch (final IOException e) { throw new UncheckedIOException("unable to prune old files", e); } @@ -292,9 +308,9 @@ private void pruneOldFiles() { */ private void closeFile() { try { - previousGenerationalSpan = currentMutableFile.getUtilizedGenerationalSpan(); + previousSpan = currentMutableFile.getUtilizedSpan(); if (!bootstrapMode) { - averageGenerationalSpanUtilization.add(previousGenerationalSpan); + averageSpanUtilization.add(previousSpan); } currentMutableFile.close(); @@ -310,20 +326,22 @@ private void closeFile() { } /** - * Calculate the generation span for a new file that is about to be created. + * Calculate the span for a new file that is about to be created. + * + * @param minimumLowerBound the minimum lower bound that is legal to use for the new file + * @param nextAncientIdentifierToWrite the ancient indicator of the next event that will be written */ - private long computeNewFileSpan(final long minimumFileGeneration, final long nextGenerationToWrite) { + private long computeNewFileSpan(final long minimumLowerBound, final long nextAncientIdentifierToWrite) { - final long basisSpan = (bootstrapMode || averageGenerationalSpanUtilization.isEmpty()) - ? previousGenerationalSpan - : averageGenerationalSpanUtilization.getAverage(); + final long basisSpan = (bootstrapMode || averageSpanUtilization.isEmpty()) + ? previousSpan + : averageSpanUtilization.getAverage(); - final double overlapFactor = - bootstrapMode ? bootstrapGenerationalSpanOverlapFactor : generationalSpanOverlapFactor; + final double overlapFactor = bootstrapMode ? bootstrapSpanOverlapFactor : spanOverlapFactor; final long desiredSpan = (long) (basisSpan * overlapFactor); - final long minimumSpan = (nextGenerationToWrite + minimumGenerationalCapacity) - minimumFileGeneration; + final long minimumSpan = (nextAncientIdentifierToWrite + this.minimumSpan) - minimumLowerBound; return Math.max(desiredSpan, minimumSpan); } @@ -338,7 +356,8 @@ private long computeNewFileSpan(final long minimumFileGeneration, final long nex private Long prepareOutputStream(@NonNull final GossipEvent eventToWrite) throws IOException { Long latestDurableSequenceNumberUpdate = null; if (currentMutableFile != null) { - final boolean fileCanContainEvent = currentMutableFile.canContain(eventToWrite.getGeneration()); + final boolean fileCanContainEvent = + currentMutableFile.canContain(eventToWrite.getAncientIndicator(fileType)); final boolean fileIsFull = UNIT_BYTES.convertTo(currentMutableFile.fileSize(), UNIT_MEGABYTES) >= preferredFileSizeMegabytes; @@ -353,11 +372,11 @@ private Long prepareOutputStream(@NonNull final GossipEvent eventToWrite) throws } if (currentMutableFile == null) { - final long maximumGeneration = minimumGenerationNonAncient - + computeNewFileSpan(minimumGenerationNonAncient, eventToWrite.getGeneration()); + final long upperBound = nonAncientBoundary + + computeNewFileSpan(nonAncientBoundary, eventToWrite.getAncientIndicator(fileType)); currentMutableFile = fileManager - .getNextFileDescriptor(minimumGenerationNonAncient, maximumGeneration) + .getNextFileDescriptor(nonAncientBoundary, upperBound) .getMutableFile(); } diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/recovery/EventRecoveryWorkflow.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/recovery/EventRecoveryWorkflow.java index 51ffb24bad36..a1e36e5ce491 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/recovery/EventRecoveryWorkflow.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/recovery/EventRecoveryWorkflow.java @@ -44,6 +44,7 @@ import com.swirlds.platform.event.GossipEvent; import com.swirlds.platform.event.preconsensus.PcesFile; import com.swirlds.platform.event.preconsensus.PcesMutableFile; +import com.swirlds.platform.eventhandling.EventConfig; import com.swirlds.platform.internal.EventImpl; import com.swirlds.platform.recovery.emergencyfile.EmergencyRecoveryFile; import com.swirlds.platform.recovery.internal.EventStreamRoundIterator; @@ -188,6 +189,10 @@ public static void recoverState( logger.info(STARTUP.getMarker(), "Signed state written to disk"); final PcesFile preconsensusEventFile = PcesFile.of( + platformContext + .getConfiguration() + .getConfigData(EventConfig.class) + .getAncientMode(), Instant.now(), 0, recoveredState.judge().getGeneration(), diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/wiring/LinkedEventIntakeWiring.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/wiring/LinkedEventIntakeWiring.java index 2a2327d2f9b1..9c3aa0d3c14b 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/wiring/LinkedEventIntakeWiring.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/wiring/LinkedEventIntakeWiring.java @@ -31,21 +31,18 @@ /** * Wiring for the {@link LinkedEventIntake}. * - * @param eventInput the input wire for events to be added to the hashgraph - * @param pauseInput the input wire for pausing the linked event intake - * @param consensusRoundOutput the output wire for consensus rounds - * @param nonAncientEventWindowOutput the output wire for the {@link NonAncientEventWindow}. This output is - * transformed from the consensus round output - * @param minimumGenerationNonAncientOutput the output wire for the minimum generation non-ancient. This output is - * transformed from the consensus round output - * @param flushRunnable the runnable to flush the intake + * @param eventInput the input wire for events to be added to the hashgraph + * @param pauseInput the input wire for pausing the linked event intake + * @param consensusRoundOutput the output wire for consensus rounds + * @param nonAncientEventWindowOutput the output wire for the {@link NonAncientEventWindow}. This output is transformed + * from the consensus round output + * @param flushRunnable the runnable to flush the intake */ public record LinkedEventIntakeWiring( @NonNull InputWire eventInput, @NonNull InputWire pauseInput, @NonNull OutputWire consensusRoundOutput, @NonNull OutputWire nonAncientEventWindowOutput, - @NonNull OutputWire minimumGenerationNonAncientOutput, @NonNull Runnable flushRunnable) { /** @@ -66,10 +63,6 @@ public static LinkedEventIntakeWiring create(@NonNull final TaskScheduler consensusRound.getNonAncientEventWindow()), - consensusRoundOutput.buildTransformer( - "getMinimumGenerationNonAncient", - "rounds", - consensusRound -> consensusRound.getGenerations().getMinGenerationNonAncient()), taskScheduler::flush); } diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/wiring/PlatformWiring.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/wiring/PlatformWiring.java index 09e4ac07da40..e023b97cdd3f 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/wiring/PlatformWiring.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/wiring/PlatformWiring.java @@ -159,6 +159,7 @@ private void solderNonAncientEventWindow() { nonAncientEventWindowOutputWire.solderTo(orphanBufferWiring.nonAncientEventWindowInput(), INJECT); nonAncientEventWindowOutputWire.solderTo(inOrderLinkerWiring.nonAncientEventWindowInput(), INJECT); nonAncientEventWindowOutputWire.solderTo(eventCreationManagerWiring.nonAncientEventWindowInput(), INJECT); + nonAncientEventWindowOutputWire.solderTo(pcesWriterWiring.nonAncientEventWindowInput(), INJECT); } /** @@ -181,9 +182,6 @@ private void wire() { orphanBufferWiring.eventOutput().solderTo(stateSignatureCollectorWiring.preconsensusEventInput()); solderNonAncientEventWindow(); - linkedEventIntakeWiring - .minimumGenerationNonAncientOutput() - .solderTo(pcesWriterWiring.minimumGenerationNonAncientInput(), INJECT); pcesReplayerWiring.doneStreamingPcesOutputWire().solderTo(pcesWriterWiring.doneStreamingPcesInputWire()); pcesReplayerWiring.eventOutput().solderTo(eventHasherWiring.eventInput()); @@ -193,7 +191,7 @@ private void wire() { signedStateFileManagerWiring .oldestMinimumGenerationOnDiskOutputWire() - .solderTo(pcesWriterWiring.minimumGenerationToStoreInputWire()); + .solderTo(pcesWriterWiring.minimumAncientIdentifierToStoreInputWire()); } /** @@ -377,7 +375,7 @@ public StandardOutputWire getPcesReplayerEventOutput() { * @return the input wire for the PCES writer minimum generation to store */ public InputWire getPcesMinimumGenerationToStoreInput() { - return pcesWriterWiring.minimumGenerationToStoreInputWire(); + return pcesWriterWiring.minimumAncientIdentifierToStoreInputWire(); } /** @@ -405,15 +403,6 @@ public void updateNonAncientEventWindow(@NonNull final NonAncientEventWindow non eventCreationManagerWiring.nonAncientEventWindowInput().inject(nonAncientEventWindow); } - /** - * Inject a new minimum generation non-ancient on all components that need it. - * - * @param minimumGenerationNonAncient the new minimum generation non-ancient - */ - public void updateMinimumGenerationNonAncient(final long minimumGenerationNonAncient) { - pcesWriterWiring.minimumGenerationNonAncientInput().inject(minimumGenerationNonAncient); - } - /** * Flush the intake pipeline. */ diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/wiring/components/PcesWriterWiring.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/wiring/components/PcesWriterWiring.java index 016bc5b06792..8b9b8f4d42ef 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/wiring/components/PcesWriterWiring.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/wiring/components/PcesWriterWiring.java @@ -20,6 +20,7 @@ import com.swirlds.common.wiring.wires.input.BindableInputWire; import com.swirlds.common.wiring.wires.input.InputWire; import com.swirlds.common.wiring.wires.output.OutputWire; +import com.swirlds.platform.consensus.NonAncientEventWindow; import com.swirlds.platform.event.GossipEvent; import com.swirlds.platform.event.preconsensus.PcesWriter; import com.swirlds.platform.wiring.DoneStreamingPcesTrigger; @@ -28,19 +29,20 @@ /** * Wiring for the {@link PcesWriter}. * - * @param doneStreamingPcesInputWire the input wire for the trigger to indicate that PCES streaming is complete - * @param eventInputWire the input wire for events to be written - * @param discontinuityInputWire the input wire for PCES discontinuities - * @param minimumGenerationNonAncientInput the input wire for the minimum generation of non-ancient events - * @param minimumGenerationToStoreInputWire the input wire for the minimum generation of events to store - * @param latestDurableSequenceNumberOutput the output wire for the latest durable sequence number + * @param doneStreamingPcesInputWire the input wire for the trigger to indicate that PCES streaming is + * complete + * @param eventInputWire the input wire for events to be written + * @param discontinuityInputWire the input wire for PCES discontinuities + * @param nonAncientEventWindowInput the input wire for non ancient event windows + * @param minimumAncientIdentifierToStoreInputWire the input wire for the minimum ancient identifier of events to store + * @param latestDurableSequenceNumberOutput the output wire for the latest durable sequence number */ public record PcesWriterWiring( @NonNull InputWire doneStreamingPcesInputWire, @NonNull InputWire eventInputWire, @NonNull InputWire discontinuityInputWire, - @NonNull InputWire minimumGenerationNonAncientInput, - @NonNull InputWire minimumGenerationToStoreInputWire, + @NonNull InputWire nonAncientEventWindowInput, + @NonNull InputWire minimumAncientIdentifierToStoreInputWire, @NonNull OutputWire latestDurableSequenceNumberOutput) { /** @@ -55,8 +57,8 @@ public static PcesWriterWiring create(@NonNull final TaskScheduler taskSch taskScheduler.buildInputWire("done streaming pces"), taskScheduler.buildInputWire("events to write"), taskScheduler.buildInputWire("discontinuity"), - taskScheduler.buildInputWire("minimum generation non ancient"), - taskScheduler.buildInputWire("minimum generation to store"), + taskScheduler.buildInputWire("non-ancient event window"), + taskScheduler.buildInputWire("minimum identifier to store"), taskScheduler.getOutputWire()); } @@ -70,9 +72,9 @@ public void bind(@NonNull final PcesWriter pcesWriter) { .bind(pcesWriter::beginStreamingNewEvents); ((BindableInputWire) eventInputWire).bind(pcesWriter::writeEvent); ((BindableInputWire) discontinuityInputWire).bind(pcesWriter::registerDiscontinuity); - ((BindableInputWire) minimumGenerationNonAncientInput) - .bind(pcesWriter::setMinimumGenerationNonAncient); - ((BindableInputWire) minimumGenerationToStoreInputWire) - .bind(pcesWriter::setMinimumGenerationToStore); + ((BindableInputWire) nonAncientEventWindowInput) + .bind(pcesWriter::updateNonAncientEventBoundary); + ((BindableInputWire) minimumAncientIdentifierToStoreInputWire) + .bind(pcesWriter::setMinimumAncientIdentifierToStore); } } diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/wiring/diagram-commands.txt b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/wiring/diagram-commands.txt index daa330d0ec85..726fcc16a347 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/wiring/diagram-commands.txt +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/wiring/diagram-commands.txt @@ -6,7 +6,6 @@ Groups high level grouping of components and component groups. Substitutes known pcli diagram \ -l 'applicationTransactionPrehandler:futures:linkedEventIntake' \ -s 'getNonAncientEventWindow:non-ancient event window:ʘ' \ - -s 'getMinimumGenerationNonAncient:minimum generation non ancient:ẞ' \ -s 'heartbeat:heartbeat:♡' \ -s 'eventCreationManager:non-validated events:†' \ -s 'applicationTransactionPrehandler:futures:★' \ @@ -14,11 +13,11 @@ pcli diagram \ -l 'eventDurabilityNexus:countUpLatch:linkedEventIntake' \ -g 'Event Validation:internalEventValidator,eventDeduplicator,eventSignatureValidator' \ -g 'Orphan Buffer:orphanBuffer,orphanBufferSplitter' \ - -g 'Linked Event Intake:linkedEventIntake,linkedEventIntakeSplitter,getNonAncientEventWindow,getMinimumGenerationNonAncient' \ + -g 'Linked Event Intake:linkedEventIntake,linkedEventIntakeSplitter,getNonAncientEventWindow' \ -g 'State File Management:signedStateFileManager,extractOldestMinimumGenerationOnDisk,toStateWrittenToDiskAction' \ -g 'State Signature Collection:systemTransactionsFilter,systemTransactionsSplitter,stateSignatureTransactionsFilter,stateSignatureCollector' \ -g 'Intake Pipeline:Event Validation,Orphan Buffer,eventHasher' \ - -g 'PCES:pcesSequencer,pcesWriter,eventDurabilityNexus' \ + -g 'Preconsensus Event Stream:pcesSequencer,pcesWriter,eventDurabilityNexus' \ -g 'Consensus Pipeline:inOrderLinker,Linked Event Intake' ######################################################################################################################## @@ -29,7 +28,6 @@ Same as 'Uncollapsed' but with low level things collapsed. Attempts to hide thin pcli diagram \ -l 'applicationTransactionPrehandler:futures:linkedEventIntake' \ -s 'getNonAncientEventWindow:non-ancient event window:ʘ' \ - -s 'getMinimumGenerationNonAncient:minimum generation non ancient:ẞ' \ -s 'heartbeat:heartbeat:♡' \ -s 'eventCreationManager:non-validated events:†' \ -s 'applicationTransactionPrehandler:futures:★' \ @@ -37,13 +35,14 @@ pcli diagram \ -l 'eventDurabilityNexus:countUpLatch:linkedEventIntake' \ -g 'Event Validation:internalEventValidator,eventDeduplicator,eventSignatureValidator' \ -g 'Orphan Buffer:orphanBuffer,orphanBufferSplitter' \ - -g 'Linked Event Intake:linkedEventIntake,linkedEventIntakeSplitter,getNonAncientEventWindow,getMinimumGenerationNonAncient' \ + -g 'Linked Event Intake:linkedEventIntake,linkedEventIntakeSplitter,getNonAncientEventWindow' \ -g 'State File Management:signedStateFileManager,extractOldestMinimumGenerationOnDisk,toStateWrittenToDiskAction' \ -g 'State Signature Collection:systemTransactionsFilter,systemTransactionsSplitter,stateSignatureTransactionsFilter,stateSignatureCollector' \ -g 'Intake Pipeline:Event Validation,Orphan Buffer,eventHasher' \ - -g 'PCES:pcesSequencer,pcesWriter,eventDurabilityNexus' \ + -g 'Preconsensus Event Stream:pcesSequencer,pcesWriter,eventDurabilityNexus' \ -g 'Consensus Pipeline:inOrderLinker,Linked Event Intake' \ -c 'Orphan Buffer' \ -c 'Linked Event Intake' \ -c 'State File Management' \ - -c 'State Signature Collection' + -c 'State Signature Collection' \ + -c 'Preconsensus Event Stream' diff --git a/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/RandomEventUtils.java b/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/RandomEventUtils.java index 26efe7f79bc5..32bb719d9171 100644 --- a/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/RandomEventUtils.java +++ b/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/RandomEventUtils.java @@ -159,13 +159,14 @@ public static IndexedEvent randomEventWithTimestamp( final Random random, final NodeId creatorId, final Instant timestamp, + final long birthRound, final ConsensusTransactionImpl[] transactions, final EventImpl selfParent, final EventImpl otherParent, final boolean fakeHash) { final BaseEventHashedData hashedData = randomEventHashedDataWithTimestamp( - random, creatorId, timestamp, transactions, selfParent, otherParent, fakeHash); + random, creatorId, timestamp, birthRound, transactions, selfParent, otherParent, fakeHash); final byte[] sig = new byte[SignatureType.RSA.signatureLength()]; random.nextBytes(sig); @@ -249,6 +250,7 @@ public static BaseEventHashedData randomEventHashedDataWithTimestamp( @NonNull final Random random, @NonNull final NodeId creatorId, @NonNull final Instant timestamp, + final long birthRound, @Nullable final ConsensusTransactionImpl[] transactions, @Nullable final EventImpl selfParent, @Nullable final EventImpl otherParent, @@ -260,21 +262,21 @@ public static BaseEventHashedData randomEventHashedDataWithTimestamp( selfParent.getBaseHash(), selfParent.getCreatorId(), selfParent.getGeneration(), - EventConstants.BIRTH_ROUND_UNDEFINED); + selfParent.getBaseEvent().getHashedData().getBirthRound()); final EventDescriptor otherDescriptor = (otherParent == null || otherParent.getBaseHash() == null) ? null : new EventDescriptor( otherParent.getBaseHash(), otherParent.getCreatorId(), otherParent.getGeneration(), - EventConstants.BIRTH_ROUND_UNDEFINED); + otherParent.getBaseEvent().getHashedData().getBirthRound()); final BaseEventHashedData hashedData = new BaseEventHashedData( new BasicSoftwareVersion(1), creatorId, selfDescriptor, otherDescriptor == null ? Collections.emptyList() : Collections.singletonList(otherDescriptor), - EventConstants.BIRTH_ROUND_UNDEFINED, + birthRound, timestamp, transactions); diff --git a/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/generator/AbstractGraphGenerator.java b/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/generator/AbstractGraphGenerator.java index e3f1210c51f3..27ff9957933e 100644 --- a/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/generator/AbstractGraphGenerator.java +++ b/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/generator/AbstractGraphGenerator.java @@ -93,19 +93,6 @@ public final IndexedEvent generateEvent() { return next; } - /** - * Generate an event, but don't build the descriptor. - * - * @return the generated event with unbuilt descriptor - */ - public final IndexedEvent generateEventWithoutDescriptor() { - final IndexedEvent next = buildNextEvent(numEventsGenerated); - next.getBaseEvent().signalPrehandleCompletion(); - numEventsGenerated++; - updateMaxGeneration(next); - return next; - } - /** * {@inheritDoc} */ diff --git a/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/source/AbstractEventSource.java b/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/source/AbstractEventSource.java index 46b060d41975..7d28ef6430f4 100644 --- a/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/source/AbstractEventSource.java +++ b/platform-sdk/swirlds-platform-core/src/testFixtures/java/com/swirlds/platform/test/fixtures/event/source/AbstractEventSource.java @@ -84,14 +84,14 @@ public abstract class AbstractEventSource> impl private long eventCount; /** - * A dynamic value generator that is used to determine the age of the other parent - * (for when we ask another node for the other parent) + * A dynamic value generator that is used to determine the age of the other parent (for when we ask another node for + * the other parent) */ private DynamicValueGenerator otherParentRequestIndex; /** - * A dynamic value generator that is used to determine the age of the other parent - * (for when another node is requesting the other parent from us) + * A dynamic value generator that is used to determine the age of the other parent (for when another node is + * requesting the other parent from us) */ private DynamicValueGenerator otherParentProviderIndex; @@ -108,12 +108,9 @@ public abstract class AbstractEventSource> impl /** * Creates a new instance with the supplied transaction generator and weight. * - * @param useFakeHashes - * indicates if fake hashes should be used instead of real ones - * @param transactionGenerator - * a transaction generator to use when creating events - * @param weight - * the weight allocated to this event source + * @param useFakeHashes indicates if fake hashes should be used instead of real ones + * @param transactionGenerator a transaction generator to use when creating events + * @param weight the weight allocated to this event source */ protected AbstractEventSource( final boolean useFakeHashes, final TransactionGenerator transactionGenerator, final long weight) { @@ -148,8 +145,8 @@ protected AbstractEventSource(final AbstractEventSource that) { } /** - * Maintain the maximum size of a list of events by removing the last element (if needed). Utility method that - * is useful for child classes. + * Maintain the maximum size of a list of events by removing the last element (if needed). Utility method that is + * useful for child classes. */ protected void pruneEventList(final LinkedList events) { if (events.size() > getRecentEventRetentionSize()) { @@ -226,12 +223,24 @@ public IndexedEvent generateEvent( final IndexedEvent otherParentEvent = otherParent.getRecentEvent(random, otherParentIndex); + // Temporary hack: use the generation as the birth round. This should be sufficient for many use cases + // that don't use the birth round for anything other than deciding if the event is ancient. This won't + // work for consensus, and we will have to upgrade our simulation framework prior to converting + // consensus to use the birth round. + final IndexedEvent latestSelfEvent = getLatestEvent(random); + final long birthRound = Math.max( + otherParentEvent == null ? 0 : otherParentEvent.getGeneration(), + latestSelfEvent == null ? 0 : latestSelfEvent.getGeneration()) + + 1; + ; + event = RandomEventUtils.randomEventWithTimestamp( random, nodeId, timestamp, + birthRound, transactionGenerator.generate(random), - getLatestEvent(random), + latestSelfEvent, otherParentEvent, useFakeHashes); diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesFileManagerTests.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesFileManagerTests.java index 91887be86cd9..3bbb226ac1e1 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesFileManagerTests.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesFileManagerTests.java @@ -18,7 +18,9 @@ import static com.swirlds.common.test.fixtures.AssertionUtils.assertIteratorEquality; import static com.swirlds.common.test.fixtures.RandomUtils.getRandomPrintSeed; -import static com.swirlds.platform.event.preconsensus.PcesFileManager.NO_MINIMUM_GENERATION; +import static com.swirlds.platform.event.AncientMode.BIRTH_ROUND_THRESHOLD; +import static com.swirlds.platform.event.AncientMode.GENERATION_THRESHOLD; +import static com.swirlds.platform.event.preconsensus.PcesFileManager.NO_LOWER_BOUND; import static com.swirlds.platform.test.event.preconsensus.PcesFileReaderTests.createDummyFile; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -35,12 +37,15 @@ import com.swirlds.common.test.fixtures.TestRecycleBin; import com.swirlds.common.utility.CompareTo; import com.swirlds.config.api.Configuration; +import com.swirlds.platform.event.AncientMode; import com.swirlds.platform.event.preconsensus.PcesConfig_; import com.swirlds.platform.event.preconsensus.PcesFile; import com.swirlds.platform.event.preconsensus.PcesFileManager; import com.swirlds.platform.event.preconsensus.PcesFileReader; import com.swirlds.platform.event.preconsensus.PcesFileTracker; +import com.swirlds.platform.eventhandling.EventConfig_; import com.swirlds.test.framework.config.TestConfigBuilder; +import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.nio.file.Path; import java.time.Duration; @@ -48,10 +53,13 @@ import java.util.ArrayList; import java.util.List; import java.util.Random; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; /** * Tests for {@link PcesFileManager} @@ -80,66 +88,74 @@ void beforeEach() throws IOException { files = new ArrayList<>(); } - private PlatformContext buildContext() { + protected static Stream buildArguments() { + return Stream.of(Arguments.of(GENERATION_THRESHOLD), Arguments.of(BIRTH_ROUND_THRESHOLD)); + } + + private PlatformContext buildContext(@NonNull final AncientMode ancientMode, @NonNull final Time time) { final Configuration configuration = new TestConfigBuilder() .withValue(PcesConfig_.DATABASE_DIRECTORY, testDirectory.resolve("data")) .withValue( "event.preconsensus.recycleBinDirectory", testDirectory.resolve("recycle")) // FUTURE: No property defined for value .withValue(PcesConfig_.PREFERRED_FILE_SIZE_MEGABYTES, 5) + .withValue(EventConfig_.USE_BIRTH_ROUND_ANCIENT_THRESHOLD, ancientMode == BIRTH_ROUND_THRESHOLD) .withValue(PcesConfig_.PERMIT_GAPS, false) + .withValue(PcesConfig_.COMPACT_LAST_FILE_ON_STARTUP, false) .getOrCreateConfig(); final Metrics metrics = new NoOpMetrics(); - return new DefaultPlatformContext(configuration, metrics, CryptographyHolder.get(), Time.getCurrent()); + return new DefaultPlatformContext(configuration, metrics, CryptographyHolder.get(), time); } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Generate Descriptors With Manager Test") - void generateDescriptorsWithManagerTest() throws IOException { - final PlatformContext platformContext = buildContext(); + void generateDescriptorsWithManagerTest(@NonNull final AncientMode ancientMode) throws IOException { + final PlatformContext platformContext = buildContext(ancientMode, Time.getCurrent()); final long maxDelta = random.nextLong(10, 20); - long minimumGeneration = random.nextLong(0, 1000); - long maximumGeneration = random.nextLong(minimumGeneration, minimumGeneration + maxDelta); + long lowerBound = random.nextLong(0, 1000); + long upperBound = random.nextLong(lowerBound, lowerBound + maxDelta); Instant timestamp = Instant.now(); final PcesFileManager generatingManager = - new PcesFileManager(platformContext, Time.getCurrent(), new PcesFileTracker(), new NodeId(0), 0); + new PcesFileManager(platformContext, new PcesFileTracker(ancientMode), new NodeId(0), 0); for (int i = 0; i < fileCount; i++) { - final PcesFile file = generatingManager.getNextFileDescriptor(minimumGeneration, maximumGeneration); + final PcesFile file = generatingManager.getNextFileDescriptor(lowerBound, upperBound); - assertTrue(file.getMinimumGeneration() >= minimumGeneration); - assertTrue(file.getMaximumGeneration() >= maximumGeneration); + assertTrue(file.getLowerBound() >= lowerBound); + assertTrue(file.getUpperBound() >= upperBound); - // Intentionally allow generations to be chosen that may not be legal (i.e. a generation decreases) - minimumGeneration = random.nextLong(minimumGeneration - 1, maximumGeneration + 1); - maximumGeneration = random.nextLong(minimumGeneration, minimumGeneration + maxDelta); + // Intentionally allow bounds to be chosen that may not be legal (i.e. ancient threshold decreases) + lowerBound = random.nextLong(lowerBound - 1, upperBound + 1); + upperBound = random.nextLong(lowerBound, lowerBound + maxDelta); timestamp = timestamp.plusMillis(random.nextInt(1, 100_000)); files.add(file); createDummyFile(file); } - final PcesFileTracker fileTracker = - PcesFileReader.readFilesFromDisk(buildContext(), TestRecycleBin.getInstance(), fileDirectory, 0, false); + final PcesFileTracker fileTracker = PcesFileReader.readFilesFromDisk( + platformContext, TestRecycleBin.getInstance(), fileDirectory, 0, false, ancientMode); - assertIteratorEquality(files.iterator(), fileTracker.getFileIterator(NO_MINIMUM_GENERATION, 0)); + assertIteratorEquality(files.iterator(), fileTracker.getFileIterator(NO_LOWER_BOUND, 0)); } - @Test - @DisplayName("Incremental Pruning By Generation Test") - void incrementalPruningByGenerationTest() throws IOException { + @ParameterizedTest + @MethodSource("buildArguments") + @DisplayName("Incremental Pruning By Ancient Boundary Test") + void incrementalPruningByAncientBoundaryTest(@NonNull final AncientMode ancientMode) throws IOException { // Intentionally pick values close to wrapping around the 3 digit to 4 digit sequence number. // This will cause the files not to line up alphabetically, and this is a scenario that the // code should be able to handle. final long firstSequenceNumber = random.nextLong(950, 1000); final long maxDelta = random.nextLong(10, 20); - long minimumGeneration = random.nextLong(0, 1000); - long maximumGeneration = random.nextLong(minimumGeneration, minimumGeneration + maxDelta); + long lowerBound = random.nextLong(0, 1000); + long upperBound = random.nextLong(lowerBound, lowerBound + maxDelta); Instant timestamp = Instant.now(); for (long sequenceNumber = firstSequenceNumber; @@ -147,19 +163,16 @@ void incrementalPruningByGenerationTest() throws IOException { sequenceNumber++) { final PcesFile file = - PcesFile.of(timestamp, sequenceNumber, minimumGeneration, maximumGeneration, 0, fileDirectory); + PcesFile.of(ancientMode, timestamp, sequenceNumber, lowerBound, upperBound, 0, fileDirectory); - minimumGeneration = random.nextLong(minimumGeneration, maximumGeneration + 1); - maximumGeneration = - Math.max(maximumGeneration + 1, random.nextLong(minimumGeneration, minimumGeneration + maxDelta)); + lowerBound = random.nextLong(lowerBound, upperBound + 1); + upperBound = Math.max(upperBound + 1, random.nextLong(lowerBound, lowerBound + maxDelta)); timestamp = timestamp.plusMillis(random.nextInt(1, 100_000)); files.add(file); createDummyFile(file); } - final PlatformContext platformContext = buildContext(); - // Choose a file in the middle. The goal is to incrementally purge all files before this file, but not // to purge this file or any of the ones after it. final int middleFileIndex = fileCount / 2; @@ -170,25 +183,26 @@ void incrementalPruningByGenerationTest() throws IOException { // Set the far in the future, we want all files to be GC eligible by temporal reckoning. final FakeTime time = new FakeTime(lastFile.getTimestamp().plus(Duration.ofHours(1)), Duration.ZERO); + final PlatformContext platformContext = buildContext(ancientMode, time); - final PcesFileTracker fileTracker = - PcesFileReader.readFilesFromDisk(buildContext(), TestRecycleBin.getInstance(), fileDirectory, 0, false); - final PcesFileManager manager = new PcesFileManager(platformContext, time, fileTracker, new NodeId(0), 0); + final PcesFileTracker fileTracker = PcesFileReader.readFilesFromDisk( + platformContext, TestRecycleBin.getInstance(), fileDirectory, 0, false, ancientMode); + final PcesFileManager manager = new PcesFileManager(platformContext, fileTracker, new NodeId(0), 0); - assertIteratorEquality(files.iterator(), fileTracker.getFileIterator(NO_MINIMUM_GENERATION, 0)); + assertIteratorEquality(files.iterator(), fileTracker.getFileIterator(NO_LOWER_BOUND, 0)); - // Increase the pruned generation a little at a time, + // Increase the pruned ancient threshold a little at a time, // until the middle file is almost GC eligible but not quite. - for (long generation = firstFile.getMaximumGeneration() - 100; - generation <= middleFile.getMaximumGeneration(); - generation++) { + for (long ancientThreshold = firstFile.getUpperBound() - 100; + ancientThreshold <= middleFile.getUpperBound(); + ancientThreshold++) { - manager.pruneOldFiles(generation); + manager.pruneOldFiles(ancientThreshold); // Parse files afresh to make sure we aren't "cheating" by just // removing the in-memory descriptor without also removing the file on disk final PcesFileTracker freshFileTracker = PcesFileReader.readFilesFromDisk( - buildContext(), TestRecycleBin.getInstance(), fileDirectory, 0, false); + platformContext, TestRecycleBin.getInstance(), fileDirectory, 0, false, ancientMode); final PcesFile firstUnPrunedFile = freshFileTracker.getFirstFile(); @@ -203,13 +217,13 @@ void incrementalPruningByGenerationTest() throws IOException { // Check the first file that wasn't pruned assertTrue(firstUnPrunedIndex <= middleFileIndex); if (firstUnPrunedIndex < middleFileIndex) { - assertTrue(firstUnPrunedFile.getMaximumGeneration() >= generation); + assertTrue(firstUnPrunedFile.getUpperBound() >= ancientThreshold); } // Check the file right before the first un-pruned file. if (firstUnPrunedIndex > 0) { final PcesFile lastPrunedFile = files.get(firstUnPrunedIndex - 1); - assertTrue(lastPrunedFile.getMaximumGeneration() < generation); + assertTrue(lastPrunedFile.getUpperBound() < ancientThreshold); } // Check all remaining files to make sure we didn't accidentally delete something from the end @@ -219,17 +233,16 @@ void incrementalPruningByGenerationTest() throws IOException { } // iterate over all files in the fresh file tracker to make sure they match expected - assertIteratorEquality( - expectedFiles.iterator(), freshFileTracker.getFileIterator(NO_MINIMUM_GENERATION, 0)); + assertIteratorEquality(expectedFiles.iterator(), freshFileTracker.getFileIterator(NO_LOWER_BOUND, 0)); } // Now, prune files so that the middle file is no longer needed. - manager.pruneOldFiles(middleFile.getMaximumGeneration() + 1); + manager.pruneOldFiles(middleFile.getUpperBound() + 1); // Parse files afresh to make sure we aren't "cheating" by just // removing the in-memory descriptor without also removing the file on disk - final PcesFileTracker freshFileTracker = - PcesFileReader.readFilesFromDisk(buildContext(), TestRecycleBin.getInstance(), fileDirectory, 0, false); + final PcesFileTracker freshFileTracker = PcesFileReader.readFilesFromDisk( + platformContext, TestRecycleBin.getInstance(), fileDirectory, 0, false, ancientMode); final PcesFile firstUnPrunedFile = freshFileTracker.getFirstFile(); @@ -244,17 +257,18 @@ void incrementalPruningByGenerationTest() throws IOException { assertEquals(middleFileIndex + 1, firstUnPrunedIndex); } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Incremental Pruning By Timestamp Test") - void incrementalPruningByTimestampTest() throws IOException { + void incrementalPruningByTimestampTest(@NonNull final AncientMode ancientMode) throws IOException { // Intentionally pick values close to wrapping around the 3 digit to 4 digit sequence number. // This will cause the files not to line up alphabetically, and this is a scenario that the // code should be able to handle. final long firstSequenceNumber = random.nextLong(950, 1000); final long maxDelta = random.nextLong(10, 20); - long minimumGeneration = random.nextLong(0, 1000); - long maximumGeneration = random.nextLong(minimumGeneration, minimumGeneration + maxDelta); + long lowerBound = random.nextLong(0, 1000); + long upperBound = random.nextLong(lowerBound, lowerBound + maxDelta); Instant timestamp = Instant.now(); for (long sequenceNumber = firstSequenceNumber; @@ -262,19 +276,16 @@ void incrementalPruningByTimestampTest() throws IOException { sequenceNumber++) { final PcesFile file = - PcesFile.of(timestamp, sequenceNumber, minimumGeneration, maximumGeneration, 0, fileDirectory); + PcesFile.of(ancientMode, timestamp, sequenceNumber, lowerBound, upperBound, 0, fileDirectory); - minimumGeneration = random.nextLong(minimumGeneration, maximumGeneration + 1); - maximumGeneration = - Math.max(maximumGeneration, random.nextLong(minimumGeneration, minimumGeneration + maxDelta)); + lowerBound = random.nextLong(lowerBound, upperBound + 1); + upperBound = Math.max(upperBound, random.nextLong(lowerBound, lowerBound + maxDelta)); timestamp = timestamp.plusMillis(random.nextInt(1, 100_000)); files.add(file); createDummyFile(file); } - final PlatformContext platformContext = buildContext(); - // Choose a file in the middle. The goal is to incrementally purge all files before this file, but not // to purge this file or any of the ones after it. final int middleFileIndex = fileCount / 2; @@ -285,12 +296,13 @@ void incrementalPruningByTimestampTest() throws IOException { // Set the clock before the first file is not garbage collection eligible final FakeTime time = new FakeTime(firstFile.getTimestamp().plus(Duration.ofMinutes(59)), Duration.ZERO); + final PlatformContext platformContext = buildContext(ancientMode, time); - final PcesFileTracker fileTracker = - PcesFileReader.readFilesFromDisk(buildContext(), TestRecycleBin.getInstance(), fileDirectory, 0, false); - final PcesFileManager manager = new PcesFileManager(platformContext, time, fileTracker, new NodeId(0), 0); + final PcesFileTracker fileTracker = PcesFileReader.readFilesFromDisk( + platformContext, TestRecycleBin.getInstance(), fileDirectory, 0, false, ancientMode); + final PcesFileManager manager = new PcesFileManager(platformContext, fileTracker, new NodeId(0), 0); - assertIteratorEquality(files.iterator(), fileTracker.getFileIterator(NO_MINIMUM_GENERATION, 0)); + assertIteratorEquality(files.iterator(), fileTracker.getFileIterator(NO_LOWER_BOUND, 0)); // Increase the timestamp a little at a time. We should gradually delete files up until // all files before the middle file have been deleted. @@ -300,12 +312,12 @@ void incrementalPruningByTimestampTest() throws IOException { Duration nextTimeIncrease = Duration.ofSeconds(random.nextInt(1, 20)); while (time.now().plus(nextTimeIncrease).isBefore(endingTime)) { time.tick(nextTimeIncrease); - manager.pruneOldFiles(lastFile.getMaximumGeneration() + 1); + manager.pruneOldFiles(lastFile.getUpperBound() + 1); // Parse files afresh to make sure we aren't "cheating" by just // removing the in-memory descriptor without also removing the file on disk final PcesFileTracker freshFileTracker = PcesFileReader.readFilesFromDisk( - buildContext(), TestRecycleBin.getInstance(), fileDirectory, 0, false); + platformContext, TestRecycleBin.getInstance(), fileDirectory, 0, false, ancientMode); final PcesFile firstUnPrunedFile = freshFileTracker.getFirstFile(); @@ -320,7 +332,7 @@ void incrementalPruningByTimestampTest() throws IOException { // Check the first file that wasn't pruned assertTrue(firstUnPrunedIndex <= middleFileIndex); if (firstUnPrunedIndex < middleFileIndex - && files.get(firstUnPrunedIndex).getMaximumGeneration() < middleFile.getMaximumGeneration()) { + && files.get(firstUnPrunedIndex).getUpperBound() < middleFile.getUpperBound()) { assertTrue(CompareTo.isGreaterThanOrEqualTo( firstUnPrunedFile.getTimestamp().plus(Duration.ofHours(1)), time.now())); } @@ -336,20 +348,19 @@ void incrementalPruningByTimestampTest() throws IOException { for (int index = firstUnPrunedIndex; index < fileCount; index++) { expectedFiles.add(files.get(index)); } - assertIteratorEquality( - expectedFiles.iterator(), freshFileTracker.getFileIterator(NO_MINIMUM_GENERATION, 0)); + assertIteratorEquality(expectedFiles.iterator(), freshFileTracker.getFileIterator(NO_LOWER_BOUND, 0)); nextTimeIncrease = Duration.ofSeconds(random.nextInt(1, 20)); } // tick time to 1 millisecond after the time of the middle file, so that it's now eligible for deletion time.tick(Duration.between(time.now(), endingTime).plus(Duration.ofMillis(1))); - manager.pruneOldFiles(lastFile.getMaximumGeneration() + 1); + manager.pruneOldFiles(lastFile.getUpperBound() + 1); // Parse files afresh to make sure we aren't "cheating" by just // removing the in-memory descriptor without also removing the file on disk - final PcesFileTracker freshFileTracker = - PcesFileReader.readFilesFromDisk(buildContext(), TestRecycleBin.getInstance(), fileDirectory, 0, false); + final PcesFileTracker freshFileTracker = PcesFileReader.readFilesFromDisk( + platformContext, TestRecycleBin.getInstance(), fileDirectory, 0, false, ancientMode); final PcesFile firstUnPrunedFile = freshFileTracker.getFirstFile(); diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesFileReaderTests.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesFileReaderTests.java index 7967ebfb01b7..4ff70fb357be 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesFileReaderTests.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesFileReaderTests.java @@ -19,7 +19,9 @@ import static com.swirlds.common.test.fixtures.AssertionUtils.assertIteratorEquality; import static com.swirlds.common.test.fixtures.RandomUtils.getRandomPrintSeed; import static com.swirlds.common.threading.manager.AdHocThreadManager.getStaticThreadManager; -import static com.swirlds.platform.event.preconsensus.PcesFileManager.NO_MINIMUM_GENERATION; +import static com.swirlds.platform.event.AncientMode.BIRTH_ROUND_THRESHOLD; +import static com.swirlds.platform.event.AncientMode.GENERATION_THRESHOLD; +import static com.swirlds.platform.event.preconsensus.PcesFileManager.NO_LOWER_BOUND; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -38,11 +40,13 @@ import com.swirlds.common.platform.NodeId; import com.swirlds.common.test.fixtures.TestRecycleBin; import com.swirlds.config.api.Configuration; +import com.swirlds.platform.event.AncientMode; import com.swirlds.platform.event.preconsensus.PcesConfig_; import com.swirlds.platform.event.preconsensus.PcesFile; import com.swirlds.platform.event.preconsensus.PcesFileReader; import com.swirlds.platform.event.preconsensus.PcesFileTracker; import com.swirlds.platform.event.preconsensus.PcesMutableFile; +import com.swirlds.platform.eventhandling.EventConfig_; import com.swirlds.test.framework.config.TestConfigBuilder; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.FileOutputStream; @@ -65,7 +69,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; @DisplayName("PcesFileReader Tests") class PcesFileReaderTests { @@ -82,14 +87,11 @@ class PcesFileReaderTests { private final int fileCount = 100; - private List files; - @BeforeEach void beforeEach() throws IOException { FileUtils.deleteDirectory(testDirectory); fileDirectory = testDirectory.resolve("data").resolve("0"); random = getRandomPrintSeed(); - files = new ArrayList<>(); } @AfterEach @@ -97,11 +99,11 @@ void afterEach() throws IOException { FileUtils.deleteDirectory(testDirectory); } - private PlatformContext buildContext() { - return buildContext(false); + private PlatformContext buildContext(@NonNull final AncientMode ancientMode) { + return buildContext(false, ancientMode); } - private PlatformContext buildContext(final boolean permitGaps) { + private PlatformContext buildContext(final boolean permitGaps, @NonNull final AncientMode ancientMode) { final Configuration configuration = new TestConfigBuilder() .withValue(PcesConfig_.DATABASE_DIRECTORY, testDirectory.resolve("data")) .withValue( @@ -109,6 +111,8 @@ private PlatformContext buildContext(final boolean permitGaps) { testDirectory.resolve("recycle")) // FUTURE: No property defined for value .withValue(PcesConfig_.PREFERRED_FILE_SIZE_MEGABYTES, 5) .withValue(PcesConfig_.PERMIT_GAPS, permitGaps) + .withValue(PcesConfig_.COMPACT_LAST_FILE_ON_STARTUP, false) + .withValue(EventConfig_.USE_BIRTH_ROUND_ANCIENT_THRESHOLD, ancientMode == BIRTH_ROUND_THRESHOLD) .getOrCreateConfig(); final Metrics metrics = new NoOpMetrics(); @@ -116,6 +120,10 @@ private PlatformContext buildContext(final boolean permitGaps) { return new DefaultPlatformContext(configuration, metrics, CryptographyHolder.get(), Time.getCurrent()); } + protected static Stream buildArguments() { + return Stream.of(Arguments.of(GENERATION_THRESHOLD), Arguments.of(BIRTH_ROUND_THRESHOLD)); + } + /** * Create a dummy file. * @@ -133,18 +141,21 @@ public static void createDummyFile(final PcesFile descriptor) throws IOException out.close(); } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Read Files In Order Test") - void readFilesInOrderTest() throws IOException { + void readFilesInOrderTest(@NonNull final AncientMode ancientMode) throws IOException { // Intentionally pick values close to wrapping around the 3 digit to 4 digit sequence number. // This will cause the files not to line up alphabetically, and this is a scenario that the // code should be able to handle. final long firstSequenceNumber = random.nextLong(950, 1000); + final List files = new ArrayList<>(); + final long maxDelta = random.nextLong(10, 20); - long minimumGeneration = random.nextLong(0, 1000); - final long nonExistentGeneration = minimumGeneration - 1; - long maximumGeneration = random.nextLong(minimumGeneration, minimumGeneration + maxDelta); + long lowerBound = random.nextLong(0, 1000); + final long nonExistentValue = lowerBound - 1; + long upperBound = random.nextLong(lowerBound, lowerBound + maxDelta); Instant timestamp = Instant.now(); for (long sequenceNumber = firstSequenceNumber; @@ -152,94 +163,104 @@ void readFilesInOrderTest() throws IOException { sequenceNumber++) { final PcesFile file = - PcesFile.of(timestamp, sequenceNumber, minimumGeneration, maximumGeneration, 0, fileDirectory); + PcesFile.of(ancientMode, timestamp, sequenceNumber, lowerBound, upperBound, 0, fileDirectory); - minimumGeneration = random.nextLong(minimumGeneration, maximumGeneration + 1); - maximumGeneration = - Math.max(maximumGeneration, random.nextLong(minimumGeneration, minimumGeneration + maxDelta)); + lowerBound = random.nextLong(lowerBound, upperBound + 1); + upperBound = Math.max(upperBound, random.nextLong(lowerBound, lowerBound + maxDelta)); timestamp = timestamp.plusMillis(random.nextInt(1, 100_000)); files.add(file); createDummyFile(file); } - final PcesFileTracker fileTracker = - PcesFileReader.readFilesFromDisk(buildContext(), TestRecycleBin.getInstance(), fileDirectory, 0, false); + final PcesFileTracker fileTracker = PcesFileReader.readFilesFromDisk( + buildContext(ancientMode), TestRecycleBin.getInstance(), fileDirectory, 0, false, ancientMode); - assertIteratorEquality(files.iterator(), fileTracker.getFileIterator(NO_MINIMUM_GENERATION, 0)); + assertIteratorEquality(files.iterator(), fileTracker.getFileIterator(NO_LOWER_BOUND, 0)); assertIteratorEquality( - files.iterator(), fileTracker.getFileIterator(files.getFirst().getMaximumGeneration(), 0)); + files.iterator(), fileTracker.getFileIterator(files.getFirst().getUpperBound(), 0)); - // attempt to start a non-existent generation - assertIteratorEquality(files.iterator(), fileTracker.getFileIterator(nonExistentGeneration, 0)); + // attempt to start a non-existent ancient indicator + assertIteratorEquality(files.iterator(), fileTracker.getFileIterator(nonExistentValue, 0)); } @ParameterizedTest - @ValueSource(booleans = {true, false}) + @MethodSource("buildArguments") @DisplayName("Read Files In Order Gap Test") - void readFilesInOrderGapTest(final boolean permitGaps) throws IOException { - // Intentionally pick values close to wrapping around the 3 digit to 4 digit sequence number. - // This will cause the files not to line up alphabetically, and this is a scenario that the - // code should be able to handle. - final long firstSequenceNumber = random.nextLong(950, 1000); + void readFilesInOrderGapTest(@NonNull final AncientMode ancientMode) throws IOException { + for (final boolean permitGaps : List.of(true, false)) { + // Intentionally pick values close to wrapping around the 3 digit to 4 digit sequence number. + // This will cause the files not to line up alphabetically, and this is a scenario that the + // code should be able to handle. + final long firstSequenceNumber = random.nextLong(950, 1000); - final long maxDelta = random.nextLong(10, 20); - long minimumGeneration = random.nextLong(0, 1000); - long maximumGeneration = random.nextLong(minimumGeneration, minimumGeneration + maxDelta); - Instant timestamp = Instant.now(); + final List files = new ArrayList<>(); - // Skip the file right in the middle of the sequence - final long sequenceNumberToSkip = (2 * firstSequenceNumber + fileCount) / 2; + final long maxDelta = random.nextLong(10, 20); + long lowerBound = random.nextLong(0, 1000); + long upperBound = random.nextLong(lowerBound, lowerBound + maxDelta); + Instant timestamp = Instant.now(); - for (long sequenceNumber = firstSequenceNumber; - sequenceNumber < firstSequenceNumber + fileCount; - sequenceNumber++) { + // Skip the file right in the middle of the sequence + final long sequenceNumberToSkip = (2 * firstSequenceNumber + fileCount) / 2; - final PcesFile file = - PcesFile.of(timestamp, sequenceNumber, minimumGeneration, maximumGeneration, 0, fileDirectory); + for (long sequenceNumber = firstSequenceNumber; + sequenceNumber < firstSequenceNumber + fileCount; + sequenceNumber++) { - minimumGeneration = random.nextLong(minimumGeneration, maximumGeneration + 1); - maximumGeneration = - Math.max(maximumGeneration, random.nextLong(minimumGeneration, minimumGeneration + maxDelta)); - timestamp = timestamp.plusMillis(random.nextInt(1, 100_000)); + final PcesFile file = + PcesFile.of(ancientMode, timestamp, sequenceNumber, lowerBound, upperBound, 0, fileDirectory); + + lowerBound = random.nextLong(lowerBound, upperBound + 1); + upperBound = Math.max(upperBound, random.nextLong(lowerBound, lowerBound + maxDelta)); + timestamp = timestamp.plusMillis(random.nextInt(1, 100_000)); - if (sequenceNumber == sequenceNumberToSkip) { - // Intentionally don't write a file - continue; + if (sequenceNumber == sequenceNumberToSkip) { + // Intentionally don't write a file + continue; + } + + files.add(file); + createDummyFile(file); } - files.add(file); - createDummyFile(file); - } + final PlatformContext platformContext = buildContext(permitGaps, ancientMode); - final PlatformContext platformContext = buildContext(permitGaps); - - if (permitGaps) { - final PcesFileTracker fileTracker = PcesFileReader.readFilesFromDisk( - platformContext, TestRecycleBin.getInstance(), fileDirectory, 0, true); - // Gaps are allowed. We should see all files except for the one that was skipped. - assertIteratorEquality(files.iterator(), fileTracker.getFileIterator(NO_MINIMUM_GENERATION, 0)); - } else { - // Gaps are not allowed. - assertThrows( - IllegalStateException.class, - () -> PcesFileReader.readFilesFromDisk( - platformContext, TestRecycleBin.getInstance(), fileDirectory, 0, permitGaps)); + if (permitGaps) { + final PcesFileTracker fileTracker = PcesFileReader.readFilesFromDisk( + platformContext, TestRecycleBin.getInstance(), fileDirectory, 0, true, ancientMode); + // Gaps are allowed. We should see all files except for the one that was skipped. + assertIteratorEquality(files.iterator(), fileTracker.getFileIterator(NO_LOWER_BOUND, 0)); + } else { + // Gaps are not allowed. + assertThrows( + IllegalStateException.class, + () -> PcesFileReader.readFilesFromDisk( + platformContext, + TestRecycleBin.getInstance(), + fileDirectory, + 0, + permitGaps, + ancientMode)); + } } } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Read Files From Middle Test") - void readFilesFromMiddleTest() throws IOException { + void readFilesFromMiddleTest(@NonNull final AncientMode ancientMode) throws IOException { // Intentionally pick values close to wrapping around the 3 digit to 4 digit sequence number. // This will cause the files not to line up alphabetically, and this is a scenario that the // code should be able to handle. final long firstSequenceNumber = random.nextLong(950, 1000); + final List files = new ArrayList<>(); + final long maxDelta = random.nextLong(10, 20); - long minimumGeneration = random.nextLong(0, 1000); - long maximumGeneration = random.nextLong(minimumGeneration, minimumGeneration + maxDelta); + long lowerBound = random.nextLong(0, 1000); + long upperBound = random.nextLong(lowerBound, lowerBound + maxDelta); Instant timestamp = Instant.now(); for (long sequenceNumber = firstSequenceNumber; @@ -247,29 +268,27 @@ void readFilesFromMiddleTest() throws IOException { sequenceNumber++) { final PcesFile file = - PcesFile.of(timestamp, sequenceNumber, minimumGeneration, maximumGeneration, 0, fileDirectory); + PcesFile.of(ancientMode, timestamp, sequenceNumber, lowerBound, upperBound, 0, fileDirectory); - minimumGeneration = random.nextLong(minimumGeneration, maximumGeneration + 1); - maximumGeneration = - Math.max(maximumGeneration, random.nextLong(minimumGeneration, minimumGeneration + maxDelta)); + lowerBound = random.nextLong(lowerBound, upperBound + 1); + upperBound = Math.max(upperBound, random.nextLong(lowerBound, lowerBound + maxDelta)); timestamp = timestamp.plusMillis(random.nextInt(1, 100_000)); files.add(file); createDummyFile(file); } - final PcesFileTracker fileTracker = - PcesFileReader.readFilesFromDisk(buildContext(), TestRecycleBin.getInstance(), fileDirectory, 0, false); + final PcesFileTracker fileTracker = PcesFileReader.readFilesFromDisk( + buildContext(ancientMode), TestRecycleBin.getInstance(), fileDirectory, 0, false, ancientMode); // For this test, we want to iterate over files so that we are guaranteed to observe every event - // with a generation greater than or equal to the target generation. Choose a generation that falls - // roughly in the middle of the sequence of files. - final long targetGeneration = (files.getFirst().getMaximumGeneration() - + files.get(fileCount - 1).getMaximumGeneration()) - / 2; + // with an ancient indicator greater than or equal to the target threshold. Choose an ancient indicator + // that falls roughly in the middle of the sequence of files. + final long targetAncientIdentifier = + (files.getFirst().getUpperBound() + files.get(fileCount - 1).getUpperBound()) / 2; final List iteratedFiles = new ArrayList<>(); - fileTracker.getFileIterator(targetGeneration, 0).forEachRemaining(iteratedFiles::add); + fileTracker.getFileIterator(targetAncientIdentifier, 0).forEachRemaining(iteratedFiles::add); // Find the index in the file list that was returned first by the iterator int indexOfFirstFile = 0; @@ -280,11 +299,11 @@ void readFilesFromMiddleTest() throws IOException { } // The file immediately before the returned file should not contain any targeted events - assertTrue(files.get(indexOfFirstFile - 1).getMaximumGeneration() < targetGeneration); + assertTrue(files.get(indexOfFirstFile - 1).getUpperBound() < targetAncientIdentifier); // The first file returned from the iterator should - // have a maximum generation greater than or equal to the target generation. - assertTrue(iteratedFiles.getFirst().getMaximumGeneration() >= targetGeneration); + // have an upper bound greater than or equal to the target ancient indicator. + assertTrue(iteratedFiles.getFirst().getUpperBound() >= targetAncientIdentifier); // Make sure that the iterator returns files in the correct order. final List expectedFiles = new ArrayList<>(iteratedFiles.size()); @@ -296,22 +315,23 @@ void readFilesFromMiddleTest() throws IOException { /** * Similar to the other test that starts iteration in the middle, except that files will have the same - * generational - * bounds with high probability. Not a scenario we are likely to encounter in production, but it's a tricky - * edge + * bounds with high probability. Not a scenario we are likely to encounter in production, but it's a tricky edge * case we need to handle elegantly. */ - @Test - @DisplayName("Read Files From Middle Repeating Generations Test") - void readFilesFromMiddleRepeatingGenerationsTest() throws IOException { + @ParameterizedTest + @MethodSource("buildArguments") + @DisplayName("Read Files From Middle Repeating Boundaries Test") + void readFilesFromMiddleRepeatingBoundariesTest(@NonNull final AncientMode ancientMode) throws IOException { // Intentionally pick values close to wrapping around the 3 digit to 4 digit sequence number. // This will cause the files not to line up alphabetically, and this is a scenario that the // code should be able to handle. final long firstSequenceNumber = random.nextLong(950, 1000); + final List files = new ArrayList<>(); + final long maxDelta = random.nextLong(10, 20); - long minimumGeneration = random.nextLong(0, 1000); - long maximumGeneration = random.nextLong(minimumGeneration, minimumGeneration + maxDelta); + long lowerBound = random.nextLong(0, 1000); + long upperBound = random.nextLong(lowerBound, lowerBound + maxDelta); Instant timestamp = Instant.now(); for (long sequenceNumber = firstSequenceNumber; @@ -319,13 +339,12 @@ void readFilesFromMiddleRepeatingGenerationsTest() throws IOException { sequenceNumber++) { final PcesFile file = - PcesFile.of(timestamp, sequenceNumber, minimumGeneration, maximumGeneration, 0, fileDirectory); + PcesFile.of(ancientMode, timestamp, sequenceNumber, lowerBound, upperBound, 0, fileDirectory); - // Advance the generation bounds only 10% of the time + // Advance the bounds only 10% of the time if (random.nextLong() < 0.1) { - minimumGeneration = random.nextLong(minimumGeneration, maximumGeneration + 1); - maximumGeneration = - Math.max(maximumGeneration, random.nextLong(minimumGeneration, minimumGeneration + maxDelta)); + lowerBound = random.nextLong(lowerBound, upperBound + 1); + upperBound = Math.max(upperBound, random.nextLong(lowerBound, lowerBound + maxDelta)); } timestamp = timestamp.plusMillis(random.nextInt(1, 100_000)); @@ -333,18 +352,17 @@ void readFilesFromMiddleRepeatingGenerationsTest() throws IOException { createDummyFile(file); } - final PcesFileTracker fileTracker = - PcesFileReader.readFilesFromDisk(buildContext(), TestRecycleBin.getInstance(), fileDirectory, 0, false); + final PcesFileTracker fileTracker = PcesFileReader.readFilesFromDisk( + buildContext(ancientMode), TestRecycleBin.getInstance(), fileDirectory, 0, false, ancientMode); // For this test, we want to iterate over files so that we are guaranteed to observe every event - // with a generation greater than or equal to the target generation. Choose a generation that falls + // with an ancient indicator greater than or equal to the target. Choose an ancient indicator that falls // roughly in the middle of the sequence of files. - final long targetGeneration = (files.getFirst().getMaximumGeneration() - + files.get(fileCount - 1).getMaximumGeneration()) - / 2; + final long targetAncientIdentifier = + (files.getFirst().getUpperBound() + files.get(fileCount - 1).getUpperBound()) / 2; final List iteratedFiles = new ArrayList<>(); - fileTracker.getFileIterator(targetGeneration, 0).forEachRemaining(iteratedFiles::add); + fileTracker.getFileIterator(targetAncientIdentifier, 0).forEachRemaining(iteratedFiles::add); // Find the index in the file list that was returned first by the iterator int indexOfFirstFile = 0; @@ -355,11 +373,11 @@ void readFilesFromMiddleRepeatingGenerationsTest() throws IOException { } // The file immediately before the returned file should not contain any targeted events - assertTrue(files.get(indexOfFirstFile - 1).getMaximumGeneration() < targetGeneration); + assertTrue(files.get(indexOfFirstFile - 1).getUpperBound() < targetAncientIdentifier); // The first file returned from the iterator should - // have a maximum generation greater than or equal to the target generation. - assertTrue(iteratedFiles.getFirst().getMaximumGeneration() >= targetGeneration); + // have an upper bound greater than or equal to the target ancient indicator. + assertTrue(iteratedFiles.getFirst().getUpperBound() >= targetAncientIdentifier); // Make sure that the iterator returns files in the correct order. final List expectedFiles = new ArrayList<>(iteratedFiles.size()); @@ -369,17 +387,20 @@ void readFilesFromMiddleRepeatingGenerationsTest() throws IOException { assertIteratorEquality(expectedFiles.iterator(), iteratedFiles.iterator()); } - @Test - @DisplayName("Read Files From High Generation Test") - void readFilesFromHighGenerationTest() throws IOException { + @ParameterizedTest + @MethodSource("buildArguments") + @DisplayName("Read Files From High ancient indicator Test") + void readFilesFromHighAncientIdentifierTest(@NonNull final AncientMode ancientMode) throws IOException { // Intentionally pick values close to wrapping around the 3 digit to 4 digit sequence number. // This will cause the files not to line up alphabetically, and this is a scenario that the // code should be able to handle. final long firstSequenceNumber = random.nextLong(950, 1000); + final List files = new ArrayList<>(); + final long maxDelta = random.nextLong(10, 20); - long minimumGeneration = random.nextLong(0, 1000); - long maximumGeneration = random.nextLong(minimumGeneration, minimumGeneration + maxDelta); + long lowerBound = random.nextLong(0, 1000); + long upperBound = random.nextLong(lowerBound, lowerBound + maxDelta); Instant timestamp = Instant.now(); for (long sequenceNumber = firstSequenceNumber; @@ -387,39 +408,38 @@ void readFilesFromHighGenerationTest() throws IOException { sequenceNumber++) { final PcesFile file = - PcesFile.of(timestamp, sequenceNumber, minimumGeneration, maximumGeneration, 0, fileDirectory); + PcesFile.of(ancientMode, timestamp, sequenceNumber, lowerBound, upperBound, 0, fileDirectory); - minimumGeneration = random.nextLong(minimumGeneration, maximumGeneration + 1); - maximumGeneration = - Math.max(maximumGeneration, random.nextLong(minimumGeneration, minimumGeneration + maxDelta)); + lowerBound = random.nextLong(lowerBound, upperBound + 1); + upperBound = Math.max(upperBound, random.nextLong(lowerBound, lowerBound + maxDelta)); timestamp = timestamp.plusMillis(random.nextInt(1, 100_000)); files.add(file); createDummyFile(file); } - final PcesFileTracker fileTracker = - PcesFileReader.readFilesFromDisk(buildContext(), TestRecycleBin.getInstance(), fileDirectory, 0, false); + final PcesFileTracker fileTracker = PcesFileReader.readFilesFromDisk( + buildContext(ancientMode), TestRecycleBin.getInstance(), fileDirectory, 0, false, ancientMode); - // Request a generation higher than all files in the data store - final long targetGeneration = files.get(fileCount - 1).getMaximumGeneration() + 1; + // Request an ancient indicator higher than all files in the data store + final long targetAncientIdentifier = files.get(fileCount - 1).getUpperBound() + 1; - final Iterator iterator = fileTracker.getFileIterator(targetGeneration, 0); + final Iterator iterator = fileTracker.getFileIterator(targetAncientIdentifier, 0); assertFalse(iterator.hasNext()); } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Read Files From Empty Stream Test") - void readFilesFromEmptyStreamTest() { + void readFilesFromEmptyStreamTest(@NonNull final AncientMode ancientMode) { assertThrows( NoSuchFileException.class, () -> PcesFileReader.readFilesFromDisk( - buildContext(), TestRecycleBin.getInstance(), fileDirectory, 0, false)); + buildContext(ancientMode), TestRecycleBin.getInstance(), fileDirectory, 0, false, ancientMode)); } /** - * When fixing a discontinuity, invalid files are moved to a "recycle bin" directory. This method validates - * that + * When fixing a discontinuity, invalid files are moved to a "recycle bin" directory. This method validates that * behavior. */ private void validateRecycledFiles( @@ -451,24 +471,26 @@ private void validateRecycledFiles( } /** - * In this test, a discontinuity is placed in the middle of the stream. We begin iterating at the first file - * in the + * In this test, a discontinuity is placed in the middle of the stream. We begin iterating at the first file in the * stream. */ - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Start And First File Discontinuity In Middle Test") - void startAtFirstFileDiscontinuityInMiddleTest() throws IOException { + void startAtFirstFileDiscontinuityInMiddleTest(@NonNull final AncientMode ancientMode) throws IOException { final List filesBeforeDiscontinuity = new ArrayList<>(); final List filesAfterDiscontinuity = new ArrayList<>(); + final List files = new ArrayList<>(); + // Intentionally pick values close to wrapping around the 3 digit to 4 digit sequence number. // This will cause the files not to line up alphabetically, and this is a scenario that the // code should be able to handle. final long firstSequenceNumber = random.nextLong(950, 1000); final long maxDelta = random.nextLong(10, 20); - long minimumGeneration = random.nextLong(0, 1000); - long maximumGeneration = random.nextLong(minimumGeneration, minimumGeneration + maxDelta); + long lowerBound = random.nextLong(0, 1000); + long upperBound = random.nextLong(lowerBound, lowerBound + maxDelta); Instant timestamp = Instant.now(); final long discontinuitySequenceNumber = @@ -486,11 +508,10 @@ void startAtFirstFileDiscontinuityInMiddleTest() throws IOException { } final PcesFile file = - PcesFile.of(timestamp, sequenceNumber, minimumGeneration, maximumGeneration, origin, fileDirectory); + PcesFile.of(ancientMode, timestamp, sequenceNumber, lowerBound, upperBound, origin, fileDirectory); - minimumGeneration = random.nextLong(minimumGeneration, maximumGeneration + 1); - maximumGeneration = - Math.max(maximumGeneration, random.nextLong(minimumGeneration, minimumGeneration + maxDelta)); + lowerBound = random.nextLong(lowerBound, upperBound + 1); + upperBound = Math.max(upperBound, random.nextLong(lowerBound, lowerBound + maxDelta)); timestamp = timestamp.plusMillis(random.nextInt(1, 100_000)); files.add(file); @@ -502,7 +523,7 @@ void startAtFirstFileDiscontinuityInMiddleTest() throws IOException { createDummyFile(file); } - final PlatformContext platformContext = buildContext(); + final PlatformContext platformContext = buildContext(ancientMode); final RecycleBinImpl recycleBin = new RecycleBinImpl( platformContext.getConfiguration(), new NoOpMetrics(), @@ -513,63 +534,62 @@ void startAtFirstFileDiscontinuityInMiddleTest() throws IOException { // Scenario 1: choose an origin that lands on the discontinuity exactly. final long startingRound1 = origin; - final PcesFileTracker fileTracker1 = - PcesFileReader.readFilesFromDisk(platformContext, recycleBin, fileDirectory, startingRound1, false); + final PcesFileTracker fileTracker1 = PcesFileReader.readFilesFromDisk( + platformContext, recycleBin, fileDirectory, startingRound1, false, ancientMode); assertIteratorEquality( - filesAfterDiscontinuity.iterator(), - fileTracker1.getFileIterator(NO_MINIMUM_GENERATION, startingRound1)); + filesAfterDiscontinuity.iterator(), fileTracker1.getFileIterator(NO_LOWER_BOUND, startingRound1)); // Scenario 2: choose an origin that lands after the discontinuity. final long startingRound2 = random.nextLong(origin + 1, origin + 1000); - final PcesFileTracker fileTracker2 = - PcesFileReader.readFilesFromDisk(platformContext, recycleBin, fileDirectory, startingRound2, false); + final PcesFileTracker fileTracker2 = PcesFileReader.readFilesFromDisk( + platformContext, recycleBin, fileDirectory, startingRound2, false, ancientMode); assertIteratorEquality( - filesAfterDiscontinuity.iterator(), - fileTracker2.getFileIterator(NO_MINIMUM_GENERATION, startingRound2)); + filesAfterDiscontinuity.iterator(), fileTracker2.getFileIterator(NO_LOWER_BOUND, startingRound2)); // Scenario 3: choose an origin that comes before the discontinuity. This will cause the files // after the discontinuity to be deleted. final long startingRound3 = random.nextLong(startingOrigin, origin - 1); - final PcesFileTracker fileTracker3 = - PcesFileReader.readFilesFromDisk(platformContext, recycleBin, fileDirectory, startingRound3, false); + final PcesFileTracker fileTracker3 = PcesFileReader.readFilesFromDisk( + platformContext, recycleBin, fileDirectory, startingRound3, false, ancientMode); assertIteratorEquality( - filesBeforeDiscontinuity.iterator(), - fileTracker3.getFileIterator(NO_MINIMUM_GENERATION, startingRound3)); + filesBeforeDiscontinuity.iterator(), fileTracker3.getFileIterator(NO_LOWER_BOUND, startingRound3)); validateRecycledFiles(filesBeforeDiscontinuity, files, platformContext); // Scenario 4: choose an origin that is incompatible with all state files. This will cause all remaining // files to be deleted. final long startingRound4 = 0; - final PcesFileTracker fileTracker4 = - PcesFileReader.readFilesFromDisk(platformContext, recycleBin, fileDirectory, startingRound4, false); + final PcesFileTracker fileTracker4 = PcesFileReader.readFilesFromDisk( + platformContext, recycleBin, fileDirectory, startingRound4, false, ancientMode); assertIteratorEquality( - Collections.emptyIterator(), fileTracker4.getFileIterator(NO_MINIMUM_GENERATION, startingRound4)); + Collections.emptyIterator(), fileTracker4.getFileIterator(NO_LOWER_BOUND, startingRound4)); validateRecycledFiles(List.of(), files, platformContext); } /** - * In this test, a discontinuity is placed in the middle of the stream. We begin iterating at a file that - * comes + * In this test, a discontinuity is placed in the middle of the stream. We begin iterating at a file that comes * before the discontinuity, but it isn't the first file in the stream. */ - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Start At Middle File Discontinuity In Middle Test") - void startAtMiddleFileDiscontinuityInMiddleTest() throws IOException { + void startAtMiddleFileDiscontinuityInMiddleTest(@NonNull final AncientMode ancientMode) throws IOException { final List filesBeforeDiscontinuity = new ArrayList<>(); final List filesAfterDiscontinuity = new ArrayList<>(); + final List files = new ArrayList<>(); + // Intentionally pick values close to wrapping around the 3 digit to 4 digit sequence number. // This will cause the files not to line up alphabetically, and this is a scenario that the // code should be able to handle. final int firstSequenceNumber = random.nextInt(950, 1000); final long maxDelta = random.nextLong(10, 20); - long minimumGeneration = random.nextLong(0, 1000); - long maximumGeneration = random.nextLong(minimumGeneration + 1, minimumGeneration + maxDelta); + long lowerBound = random.nextLong(0, 1000); + long upperBound = random.nextLong(lowerBound + 1, lowerBound + maxDelta); Instant timestamp = Instant.now(); final int discontinuitySequenceNumber = @@ -589,10 +609,10 @@ void startAtMiddleFileDiscontinuityInMiddleTest() throws IOException { } final PcesFile file = - PcesFile.of(timestamp, sequenceNumber, minimumGeneration, maximumGeneration, origin, fileDirectory); + PcesFile.of(ancientMode, timestamp, sequenceNumber, lowerBound, upperBound, origin, fileDirectory); - minimumGeneration = random.nextLong(minimumGeneration + 1, maximumGeneration + 1); - maximumGeneration = random.nextLong(maximumGeneration + 1, maximumGeneration + maxDelta); + lowerBound = random.nextLong(lowerBound + 1, upperBound + 1); + upperBound = random.nextLong(upperBound + 1, upperBound + maxDelta); timestamp = timestamp.plusMillis(random.nextInt(1, 100_000)); if (sequenceNumber >= startSequenceNumber) { @@ -608,9 +628,9 @@ void startAtMiddleFileDiscontinuityInMiddleTest() throws IOException { // Note that the file at index 0 is not the first file in the stream, // but it is the first file we want to iterate - final long startGeneration = files.getFirst().getMaximumGeneration(); + final long startAncientIdentifier = files.getFirst().getUpperBound(); - final PlatformContext platformContext = buildContext(); + final PlatformContext platformContext = buildContext(ancientMode); final RecycleBinImpl recycleBin = new RecycleBinImpl( platformContext.getConfiguration(), @@ -622,49 +642,54 @@ void startAtMiddleFileDiscontinuityInMiddleTest() throws IOException { // Scenario 1: choose an origin that lands on the discontinuity exactly. final long startingRound1 = origin; - final PcesFileTracker fileTracker1 = - PcesFileReader.readFilesFromDisk(platformContext, recycleBin, fileDirectory, startingRound1, false); + final PcesFileTracker fileTracker1 = PcesFileReader.readFilesFromDisk( + platformContext, recycleBin, fileDirectory, startingRound1, false, ancientMode); assertIteratorEquality( - filesAfterDiscontinuity.iterator(), fileTracker1.getFileIterator(startGeneration, startingRound1)); + filesAfterDiscontinuity.iterator(), + fileTracker1.getFileIterator(startAncientIdentifier, startingRound1)); // Scenario 2: choose an origin that lands after the discontinuity. final long startingRound2 = random.nextLong(origin + 1, origin + 1000); - final PcesFileTracker fileTracker2 = - PcesFileReader.readFilesFromDisk(platformContext, recycleBin, fileDirectory, startingRound2, false); + final PcesFileTracker fileTracker2 = PcesFileReader.readFilesFromDisk( + platformContext, recycleBin, fileDirectory, startingRound2, false, ancientMode); assertIteratorEquality( - filesAfterDiscontinuity.iterator(), fileTracker2.getFileIterator(startGeneration, startingRound2)); + filesAfterDiscontinuity.iterator(), + fileTracker2.getFileIterator(startAncientIdentifier, startingRound2)); // Scenario 3: choose an origin that comes before the discontinuity. This will cause the files // after the discontinuity to be deleted. final long startingRound3 = random.nextLong(startingOrigin, origin - 1); - final PcesFileTracker fileTracker3 = - PcesFileReader.readFilesFromDisk(platformContext, recycleBin, fileDirectory, startingRound3, false); + final PcesFileTracker fileTracker3 = PcesFileReader.readFilesFromDisk( + platformContext, recycleBin, fileDirectory, startingRound3, false, ancientMode); assertIteratorEquality( - filesBeforeDiscontinuity.iterator(), fileTracker3.getFileIterator(startGeneration, startingRound3)); + filesBeforeDiscontinuity.iterator(), + fileTracker3.getFileIterator(startAncientIdentifier, startingRound3)); validateRecycledFiles(filesBeforeDiscontinuity, files, platformContext); // Scenario 4: choose an origin that is incompatible with all state files. This will cause all remaining // files to be deleted. final long startingRound4 = 0; - final PcesFileTracker fileTracker4 = - PcesFileReader.readFilesFromDisk(platformContext, recycleBin, fileDirectory, startingRound4, false); + final PcesFileTracker fileTracker4 = PcesFileReader.readFilesFromDisk( + platformContext, recycleBin, fileDirectory, startingRound4, false, ancientMode); assertIteratorEquality( - Collections.emptyIterator(), fileTracker4.getFileIterator(startGeneration, startingRound4)); + Collections.emptyIterator(), fileTracker4.getFileIterator(startAncientIdentifier, startingRound4)); validateRecycledFiles(List.of(), files, platformContext); } /** - * In this test, a discontinuity is placed in the middle of the stream, and we begin iterating on that exact - * file. + * In this test, a discontinuity is placed in the middle of the stream, and we begin iterating on that exact file. */ - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Start At Middle File Discontinuity In Middle Test") - void startAtDiscontinuityInMiddleTest() throws IOException { + void startAtDiscontinuityInMiddleTest(@NonNull final AncientMode ancientMode) throws IOException { final List filesBeforeDiscontinuity = new ArrayList<>(); final List filesAfterDiscontinuity = new ArrayList<>(); + final List files = new ArrayList<>(); + // Intentionally pick values close to wrapping around the 3 digit to 4 digit sequence number. // This will cause the files not to line up alphabetically, and this is a scenario that the // code should be able to handle. @@ -674,8 +699,8 @@ void startAtDiscontinuityInMiddleTest() throws IOException { // increases by at least 1 from file to file. The purpose for this is to make validation logic simpler. final long maxDelta = random.nextLong(10, 20); - long minimumGeneration = random.nextLong(0, 1000); - long maximumGeneration = random.nextLong(minimumGeneration + 1, minimumGeneration + maxDelta); + long lowerBound = random.nextLong(0, 1000); + long upperBound = random.nextLong(lowerBound + 1, lowerBound + maxDelta); Instant timestamp = Instant.now(); final int discontinuitySequenceNumber = @@ -693,10 +718,10 @@ void startAtDiscontinuityInMiddleTest() throws IOException { } final PcesFile file = - PcesFile.of(timestamp, sequenceNumber, minimumGeneration, maximumGeneration, origin, fileDirectory); + PcesFile.of(ancientMode, timestamp, sequenceNumber, lowerBound, upperBound, origin, fileDirectory); - minimumGeneration = random.nextLong(minimumGeneration + 1, maximumGeneration + 1); - maximumGeneration = random.nextLong(maximumGeneration + 1, maximumGeneration + maxDelta); + lowerBound = random.nextLong(lowerBound + 1, upperBound + 1); + upperBound = random.nextLong(upperBound + 1, upperBound + maxDelta); timestamp = timestamp.plusMillis(random.nextInt(1, 100_000)); files.add(file); @@ -710,9 +735,9 @@ void startAtDiscontinuityInMiddleTest() throws IOException { // Note that the file at index 0 is not the first file in the stream, // but it is the first file we want to iterate - final long startGeneration = filesAfterDiscontinuity.getFirst().getMaximumGeneration(); + final long startAncientIdentifier = filesAfterDiscontinuity.getFirst().getUpperBound(); - final PlatformContext platformContext = buildContext(); + final PlatformContext platformContext = buildContext(ancientMode); final RecycleBinImpl recycleBin = new RecycleBinImpl( platformContext.getConfiguration(), @@ -724,50 +749,54 @@ void startAtDiscontinuityInMiddleTest() throws IOException { // Scenario 1: choose an origin that lands on the discontinuity exactly. final long startingRound1 = origin; - final PcesFileTracker fileTracker1 = - PcesFileReader.readFilesFromDisk(platformContext, recycleBin, fileDirectory, startingRound1, false); + final PcesFileTracker fileTracker1 = PcesFileReader.readFilesFromDisk( + platformContext, recycleBin, fileDirectory, startingRound1, false, ancientMode); assertIteratorEquality( - filesAfterDiscontinuity.iterator(), fileTracker1.getFileIterator(startGeneration, startingRound1)); + filesAfterDiscontinuity.iterator(), + fileTracker1.getFileIterator(startAncientIdentifier, startingRound1)); // Scenario 2: choose an origin that lands after the discontinuity. final long startingRound2 = random.nextLong(origin + 1, origin + 1000); - final PcesFileTracker fileTracker2 = - PcesFileReader.readFilesFromDisk(platformContext, recycleBin, fileDirectory, startingRound2, false); + final PcesFileTracker fileTracker2 = PcesFileReader.readFilesFromDisk( + platformContext, recycleBin, fileDirectory, startingRound2, false, ancientMode); assertIteratorEquality( - filesAfterDiscontinuity.iterator(), fileTracker2.getFileIterator(startGeneration, startingRound2)); + filesAfterDiscontinuity.iterator(), + fileTracker2.getFileIterator(startAncientIdentifier, startingRound2)); // Scenario 3: choose an origin that comes before the discontinuity. This will cause the files // after the discontinuity to be deleted. final long startingRound3 = random.nextLong(startingOrigin, origin - 1); - final PcesFileTracker fileTracker3 = - PcesFileReader.readFilesFromDisk(platformContext, recycleBin, fileDirectory, startingRound3, false); - // There is no files with a compatible origin and events with generations in the span we want. + final PcesFileTracker fileTracker3 = PcesFileReader.readFilesFromDisk( + platformContext, recycleBin, fileDirectory, startingRound3, false, ancientMode); + // There is no files with a compatible origin and events with ancient indicators in the span we want. assertIteratorEquality( - Collections.emptyIterator(), fileTracker3.getFileIterator(startGeneration, startingRound3)); + Collections.emptyIterator(), fileTracker3.getFileIterator(startAncientIdentifier, startingRound3)); validateRecycledFiles(filesBeforeDiscontinuity, files, platformContext); // Scenario 4: choose an origin that is incompatible with all state files. This will cause all remaining // files to be deleted. final long startingRound4 = 0; - final PcesFileTracker fileTracker4 = - PcesFileReader.readFilesFromDisk(platformContext, recycleBin, fileDirectory, startingRound4, false); + final PcesFileTracker fileTracker4 = PcesFileReader.readFilesFromDisk( + platformContext, recycleBin, fileDirectory, startingRound4, false, ancientMode); assertIteratorEquality( - Collections.emptyIterator(), fileTracker4.getFileIterator(startGeneration, startingRound4)); + Collections.emptyIterator(), fileTracker4.getFileIterator(startAncientIdentifier, startingRound4)); validateRecycledFiles(List.of(), files, platformContext); } /** - * In this test, a discontinuity is placed in the middle of the stream, and we begin iterating after that - * file. + * In this test, a discontinuity is placed in the middle of the stream, and we begin iterating after that file. */ - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Start After Discontinuity In Middle Test") - void startAfterDiscontinuityInMiddleTest() throws IOException { + void startAfterDiscontinuityInMiddleTest(@NonNull final AncientMode ancientMode) throws IOException { final List filesBeforeDiscontinuity = new ArrayList<>(); final List filesAfterDiscontinuity = new ArrayList<>(); + final List files = new ArrayList<>(); + // Intentionally pick values close to wrapping around the 3 digit to 4 digit sequence number. // This will cause the files not to line up alphabetically, and this is a scenario that the // code should be able to handle. @@ -777,8 +806,8 @@ void startAfterDiscontinuityInMiddleTest() throws IOException { // increases by at least 1 from file to file. The purpose for this is to make validation logic simpler. final long maxDelta = random.nextLong(10, 20); - long minimumGeneration = random.nextLong(0, 1000); - long maximumGeneration = random.nextLong(minimumGeneration + 1, minimumGeneration + maxDelta); + long lowerBound = random.nextLong(0, 1000); + long upperBound = random.nextLong(lowerBound + 1, lowerBound + maxDelta); Instant timestamp = Instant.now(); final int discontinuitySequenceNumber = @@ -796,10 +825,10 @@ void startAfterDiscontinuityInMiddleTest() throws IOException { } final PcesFile file = - PcesFile.of(timestamp, sequenceNumber, minimumGeneration, maximumGeneration, origin, fileDirectory); + PcesFile.of(ancientMode, timestamp, sequenceNumber, lowerBound, upperBound, origin, fileDirectory); - minimumGeneration = random.nextLong(minimumGeneration + 1, maximumGeneration + 1); - maximumGeneration = random.nextLong(maximumGeneration + 1, maximumGeneration + maxDelta); + lowerBound = random.nextLong(lowerBound + 1, upperBound + 1); + upperBound = random.nextLong(upperBound + 1, upperBound + maxDelta); timestamp = timestamp.plusMillis(random.nextInt(1, 100_000)); files.add(file); @@ -813,9 +842,9 @@ void startAfterDiscontinuityInMiddleTest() throws IOException { // Note that the file at index 0 is not the first file in the stream, // but it is the first file we want to iterate - final long startGeneration = filesAfterDiscontinuity.getFirst().getMaximumGeneration(); + final long startAncientBoundary = filesAfterDiscontinuity.getFirst().getUpperBound(); - final PlatformContext platformContext = buildContext(); + final PlatformContext platformContext = buildContext(ancientMode); final RecycleBinImpl recycleBin = new RecycleBinImpl( platformContext.getConfiguration(), new NoOpMetrics(), @@ -826,36 +855,107 @@ void startAfterDiscontinuityInMiddleTest() throws IOException { // Scenario 1: choose an origin that lands on the discontinuity exactly. final long startingRound1 = origin; - final PcesFileTracker fileTracker1 = - PcesFileReader.readFilesFromDisk(platformContext, recycleBin, fileDirectory, startingRound1, false); + final PcesFileTracker fileTracker1 = PcesFileReader.readFilesFromDisk( + platformContext, recycleBin, fileDirectory, startingRound1, false, ancientMode); assertIteratorEquality( - filesAfterDiscontinuity.iterator(), fileTracker1.getFileIterator(startGeneration, startingRound1)); + filesAfterDiscontinuity.iterator(), fileTracker1.getFileIterator(startAncientBoundary, startingRound1)); // Scenario 2: choose an origin that lands after the discontinuity. final long startingRound2 = random.nextLong(origin + 1, origin + 1000); - final PcesFileTracker fileTracker2 = - PcesFileReader.readFilesFromDisk(platformContext, recycleBin, fileDirectory, startingRound2, false); + final PcesFileTracker fileTracker2 = PcesFileReader.readFilesFromDisk( + platformContext, recycleBin, fileDirectory, startingRound2, false, ancientMode); assertIteratorEquality( - filesAfterDiscontinuity.iterator(), fileTracker2.getFileIterator(startGeneration, startingRound2)); + filesAfterDiscontinuity.iterator(), fileTracker2.getFileIterator(startAncientBoundary, startingRound2)); // Scenario 3: choose an origin that comes before the discontinuity. This will cause the files // after the discontinuity to be deleted. final long startingRound3 = random.nextLong(startingOrigin, origin - 1); - final PcesFileTracker fileTracker3 = - PcesFileReader.readFilesFromDisk(platformContext, recycleBin, fileDirectory, startingRound3, false); + final PcesFileTracker fileTracker3 = PcesFileReader.readFilesFromDisk( + platformContext, recycleBin, fileDirectory, startingRound3, false, ancientMode); assertIteratorEquality( - Collections.emptyIterator(), fileTracker3.getFileIterator(startGeneration, startingRound3)); + Collections.emptyIterator(), fileTracker3.getFileIterator(startAncientBoundary, startingRound3)); validateRecycledFiles(filesBeforeDiscontinuity, files, platformContext); // Scenario 4: choose an origin that is incompatible with all state files. This will cause all remaining // files to be deleted. final long startingRound4 = 0; - final PcesFileTracker fileTracker4 = - PcesFileReader.readFilesFromDisk(platformContext, recycleBin, fileDirectory, startingRound4, false); + final PcesFileTracker fileTracker4 = PcesFileReader.readFilesFromDisk( + platformContext, recycleBin, fileDirectory, startingRound4, false, ancientMode); assertIteratorEquality( - Collections.emptyIterator(), fileTracker4.getFileIterator(startGeneration, startingRound4)); + Collections.emptyIterator(), fileTracker4.getFileIterator(startAncientBoundary, startingRound4)); validateRecycledFiles(List.of(), files, platformContext); } + + @Test + void readFilesOfBothTypesTest() throws IOException { + // Intentionally pick values close to wrapping around the 3 digit to 4 digit sequence number. + // This will cause the files not to line up alphabetically, and this is a scenario that the + // code should be able to handle. + final long firstSequenceNumber = random.nextLong(950, 1000); + + final long maxDelta = random.nextLong(10, 20); + long lowerBound = random.nextLong(0, 1000); + long upperBound = random.nextLong(lowerBound, lowerBound + maxDelta); + Instant timestamp = Instant.now(); + + // Phase 1: write files using generations + + final List generationFiles = new ArrayList<>(); + + for (long sequenceNumber = firstSequenceNumber; + sequenceNumber < firstSequenceNumber + fileCount; + sequenceNumber++) { + + final PcesFile file = PcesFile.of( + GENERATION_THRESHOLD, timestamp, sequenceNumber, lowerBound, upperBound, 0, fileDirectory); + + lowerBound = random.nextLong(lowerBound, upperBound + 1); + upperBound = Math.max(upperBound, random.nextLong(lowerBound, lowerBound + maxDelta)); + timestamp = timestamp.plusMillis(random.nextInt(1, 100_000)); + + generationFiles.add(file); + createDummyFile(file); + } + + // Phase 2: write files using birth rounds + final List birthRoundFiles = new ArrayList<>(); + + for (long sequenceNumber = firstSequenceNumber; + sequenceNumber < firstSequenceNumber + fileCount; + sequenceNumber++) { + + final PcesFile file = PcesFile.of( + BIRTH_ROUND_THRESHOLD, timestamp, sequenceNumber, lowerBound, upperBound, 0, fileDirectory); + + lowerBound = random.nextLong(lowerBound, upperBound + 1); + upperBound = Math.max(upperBound, random.nextLong(lowerBound, lowerBound + maxDelta)); + timestamp = timestamp.plusMillis(random.nextInt(1, 100_000)); + + birthRoundFiles.add(file); + createDummyFile(file); + } + + // Phase 3: read files of both types + + final PcesFileTracker generationFileTracker = PcesFileReader.readFilesFromDisk( + buildContext(GENERATION_THRESHOLD), + TestRecycleBin.getInstance(), + fileDirectory, + 0, + false, + GENERATION_THRESHOLD); + + final PcesFileTracker birthRoundFileTracker = PcesFileReader.readFilesFromDisk( + buildContext(BIRTH_ROUND_THRESHOLD), + TestRecycleBin.getInstance(), + fileDirectory, + 0, + false, + BIRTH_ROUND_THRESHOLD); + + assertIteratorEquality(generationFiles.iterator(), generationFileTracker.getFileIterator(NO_LOWER_BOUND, 0)); + assertIteratorEquality(birthRoundFiles.iterator(), birthRoundFileTracker.getFileIterator(NO_LOWER_BOUND, 0)); + } } diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesFileTests.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesFileTests.java index 3270991f140e..bfd1a53a52df 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesFileTests.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesFileTests.java @@ -20,8 +20,12 @@ import static com.swirlds.common.test.fixtures.RandomUtils.randomInstant; import static com.swirlds.common.test.fixtures.io.FileManipulation.writeRandomBytes; import static com.swirlds.common.threading.manager.AdHocThreadManager.getStaticThreadManager; +import static com.swirlds.platform.event.AncientMode.BIRTH_ROUND_THRESHOLD; +import static com.swirlds.platform.event.AncientMode.GENERATION_THRESHOLD; import static com.swirlds.platform.event.preconsensus.PcesFile.EVENT_FILE_SEPARATOR; +import static com.swirlds.platform.event.preconsensus.PcesFile.MAXIMUM_BIRTH_ROUND_PREFIX; import static com.swirlds.platform.event.preconsensus.PcesFile.MAXIMUM_GENERATION_PREFIX; +import static com.swirlds.platform.event.preconsensus.PcesFile.MINIMUM_BIRTH_ROUND_PREFIX; import static com.swirlds.platform.event.preconsensus.PcesFile.MINIMUM_GENERATION_PREFIX; import static com.swirlds.platform.event.preconsensus.PcesFile.ORIGIN_PREFIX; import static com.swirlds.platform.event.preconsensus.PcesFile.SEQUENCE_NUMBER_PREFIX; @@ -39,8 +43,10 @@ import com.swirlds.common.platform.NodeId; import com.swirlds.common.test.fixtures.RandomUtils; import com.swirlds.config.api.Configuration; +import com.swirlds.platform.event.AncientMode; import com.swirlds.platform.event.preconsensus.PcesFile; import com.swirlds.test.framework.config.TestConfigBuilder; +import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -54,11 +60,15 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; @DisplayName("PcesFile Tests") class PcesFileTests { @@ -80,64 +90,88 @@ void afterEach() throws IOException { FileUtils.deleteDirectory(testDirectory); } - @Test + protected static Stream buildArguments() { + return Stream.of(Arguments.of(GENERATION_THRESHOLD), Arguments.of(BIRTH_ROUND_THRESHOLD)); + } + + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Invalid Parameters Test") - void invalidParametersTest() { - assertThrows(IllegalArgumentException.class, () -> PcesFile.of(Instant.now(), -1, 1, 2, 0, Path.of("foo"))); + void invalidParametersTest(@NonNull final AncientMode ancientMode) { + assertThrows( + IllegalArgumentException.class, + () -> PcesFile.of(ancientMode, Instant.now(), -1, 1, 2, 0, Path.of("foo"))); - assertThrows(IllegalArgumentException.class, () -> PcesFile.of(Instant.now(), 1, -1, 2, 0, Path.of("foo"))); + assertThrows( + IllegalArgumentException.class, + () -> PcesFile.of(ancientMode, Instant.now(), 1, -1, 2, 0, Path.of("foo"))); - assertThrows(IllegalArgumentException.class, () -> PcesFile.of(Instant.now(), 1, -2, -1, 0, Path.of("foo"))); + assertThrows( + IllegalArgumentException.class, + () -> PcesFile.of(ancientMode, Instant.now(), 1, -2, -1, 0, Path.of("foo"))); - assertThrows(IllegalArgumentException.class, () -> PcesFile.of(Instant.now(), 1, 1, -1, 0, Path.of("foo"))); + assertThrows( + IllegalArgumentException.class, + () -> PcesFile.of(ancientMode, Instant.now(), 1, 1, -1, 0, Path.of("foo"))); - assertThrows(IllegalArgumentException.class, () -> PcesFile.of(Instant.now(), 1, 2, 1, 0, Path.of("foo"))); + assertThrows( + IllegalArgumentException.class, + () -> PcesFile.of(ancientMode, Instant.now(), 1, 2, 1, 0, Path.of("foo"))); - assertThrows(NullPointerException.class, () -> PcesFile.of(null, 1, 1, 2, 0, Path.of("foo"))); + assertThrows(NullPointerException.class, () -> PcesFile.of(ancientMode, null, 1, 1, 2, 0, Path.of("foo"))); - assertThrows(IllegalArgumentException.class, () -> PcesFile.of(Instant.now(), 1, 1, 2, -1, Path.of("foo"))); + assertThrows( + IllegalArgumentException.class, + () -> PcesFile.of(ancientMode, Instant.now(), 1, 1, 2, -1, Path.of("foo"))); - assertThrows(NullPointerException.class, () -> PcesFile.of(Instant.now(), 1, 1, 2, 0, null)); + assertThrows(NullPointerException.class, () -> PcesFile.of(ancientMode, Instant.now(), 1, 1, 2, 0, null)); } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("File Name Test") - void fileNameTest() { + void fileNameTest(@NonNull final AncientMode ancientMode) { final Random random = getRandomPrintSeed(); int count = 100; while (count-- > 0) { final long sequenceNumber = random.nextLong(1000); - final long minimumGeneration = random.nextLong(1000); - final long maximumGeneration = random.nextLong(minimumGeneration, minimumGeneration + 1000); + final long lowerBound = random.nextLong(1000); + final long upperBound = random.nextLong(lowerBound, lowerBound + 1000); final long origin = random.nextLong(1000); final Instant timestamp = RandomUtils.randomInstant(random); + final String lowerBoundPrefix = + ancientMode == GENERATION_THRESHOLD ? MINIMUM_GENERATION_PREFIX : MINIMUM_BIRTH_ROUND_PREFIX; + final String upperBoundPrefix = + ancientMode == GENERATION_THRESHOLD ? MAXIMUM_GENERATION_PREFIX : MAXIMUM_BIRTH_ROUND_PREFIX; + final String expectedName = timestamp.toString().replace(":", "+") + EVENT_FILE_SEPARATOR + SEQUENCE_NUMBER_PREFIX - + sequenceNumber + EVENT_FILE_SEPARATOR + MINIMUM_GENERATION_PREFIX - + minimumGeneration + EVENT_FILE_SEPARATOR + MAXIMUM_GENERATION_PREFIX - + maximumGeneration + EVENT_FILE_SEPARATOR + ORIGIN_PREFIX + origin + ".pces"; + + sequenceNumber + EVENT_FILE_SEPARATOR + lowerBoundPrefix + + lowerBound + EVENT_FILE_SEPARATOR + upperBoundPrefix + + upperBound + EVENT_FILE_SEPARATOR + ORIGIN_PREFIX + origin + ".pces"; final PcesFile file = PcesFile.of( - timestamp, sequenceNumber, minimumGeneration, maximumGeneration, origin, Path.of("foo/bar")); + ancientMode, timestamp, sequenceNumber, lowerBound, upperBound, origin, Path.of("foo/bar")); assertEquals(expectedName, file.getFileName()); assertEquals(expectedName, file.toString()); } } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("File Path Test") - void filePathTest() { + void filePathTest(@NonNull final AncientMode ancientMode) { final Random random = getRandomPrintSeed(); int count = 100; while (count-- > 0) { final long sequenceNumber = random.nextLong(1000); - final long minimumGeneration = random.nextLong(1000); - final long maximumGeneration = random.nextLong(minimumGeneration, minimumGeneration + 1000); + final long lowerBound = random.nextLong(1000); + final long upperBound = random.nextLong(lowerBound, lowerBound + 1000); final long origin = random.nextLong(1000); final Instant timestamp = RandomUtils.randomInstant(random); @@ -152,10 +186,11 @@ void filePathTest() { assertEquals( expectedPath, PcesFile.of( + ancientMode, timestamp, sequenceNumber, - minimumGeneration, - maximumGeneration, + lowerBound, + upperBound, origin, Path.of("foo/bar")) .getPath() @@ -163,32 +198,34 @@ void filePathTest() { } } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Parsing Test") - void parsingTest() throws IOException { + void parsingTest(@NonNull final AncientMode ancientMode) throws IOException { final Random random = getRandomPrintSeed(); int count = 100; while (count-- > 0) { final long sequenceNumber = random.nextLong(1000); - final long minimumGeneration = random.nextLong(1000); - final long maximumGeneration = random.nextLong(minimumGeneration, minimumGeneration + 1000); + final long lowerBound = random.nextLong(1000); + final long upperBound = random.nextLong(lowerBound, lowerBound + 1000); final long origin = random.nextLong(1000); final Instant timestamp = RandomUtils.randomInstant(random); final Path directory = Path.of("foo/bar/baz"); final PcesFile expected = - PcesFile.of(timestamp, sequenceNumber, minimumGeneration, maximumGeneration, origin, directory); + PcesFile.of(ancientMode, timestamp, sequenceNumber, lowerBound, upperBound, origin, directory); final PcesFile parsed = PcesFile.of(expected.getPath()); assertEquals(expected, parsed); assertEquals(sequenceNumber, parsed.getSequenceNumber()); - assertEquals(minimumGeneration, parsed.getMinimumGeneration()); - assertEquals(maximumGeneration, parsed.getMaximumGeneration()); + assertEquals(lowerBound, parsed.getLowerBound()); + assertEquals(upperBound, parsed.getUpperBound()); assertEquals(origin, parsed.getOrigin()); assertEquals(timestamp, parsed.getTimestamp()); + assertEquals(ancientMode, parsed.getFileType()); } } @@ -216,9 +253,10 @@ void invalidFileParsingTest() { } @SuppressWarnings("resource") - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Deletion Test") - void deletionTest() throws IOException { + void deletionTest(@NonNull final AncientMode ancientMode) throws IOException { final Random random = getRandomPrintSeed(); final Instant now = Instant.now(); @@ -240,8 +278,8 @@ void deletionTest() throws IOException { final List files = new ArrayList<>(); for (int index = 0; index < times.size(); index++) { final Instant timestamp = times.get(index); - // We don't care about generations for this test - final PcesFile file = PcesFile.of(timestamp, index, 0, 0, 0, testDirectory); + // We don't care about ancient indicators for this test + final PcesFile file = PcesFile.of(ancientMode, timestamp, index, 0, 0, 0, testDirectory); writeRandomBytes(random, file.getPath(), 100); files.add(file); @@ -275,9 +313,10 @@ void deletionTest() throws IOException { } @SuppressWarnings("resource") - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Recycle Test") - void recycleTest() throws IOException { + void recycleTest(@NonNull final AncientMode ancientMode) throws IOException { final Random random = getRandomPrintSeed(); final Instant now = Instant.now(); @@ -314,8 +353,8 @@ void recycleTest() throws IOException { final List files = new ArrayList<>(); for (int index = 0; index < times.size(); index++) { final Instant timestamp = times.get(index); - // We don't care about generations for this test - final PcesFile file = PcesFile.of(timestamp, index, 0, 0, 0, streamDirectory); + // We don't care about ancient indicators for this test + final PcesFile file = PcesFile.of(ancientMode, timestamp, index, 0, 0, 0, streamDirectory); writeRandomBytes(random, file.getPath(), 100); files.add(file); @@ -354,9 +393,10 @@ void recycleTest() throws IOException { } } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("compareTo() Test") - void compareToTest() { + void compareToTest(@NonNull final AncientMode ancientMode) { final Random random = getRandomPrintSeed(); final Path directory = Path.of("foo/bar/baz"); @@ -365,24 +405,26 @@ void compareToTest() { final long sequenceA = random.nextLong(100); final long sequenceB = random.nextLong(100); - final long minimumGenerationA = random.nextLong(100); - final long minimumGenerationB = random.nextLong(100); + final long lowerBoundA = random.nextLong(100); + final long lowerBoundB = random.nextLong(100); - final long maximumGenerationA = random.nextLong(minimumGenerationA, minimumGenerationA + 100); - final long maximumGenerationB = random.nextLong(minimumGenerationB, minimumGenerationB + 100); + final long upperBoundA = random.nextLong(lowerBoundA, lowerBoundA + 100); + final long upperBoundB = random.nextLong(lowerBoundB, lowerBoundB + 100); final PcesFile a = PcesFile.of( + ancientMode, randomInstant(random), sequenceA, - minimumGenerationA, - maximumGenerationA, + lowerBoundA, + upperBoundA, random.nextLong(1000), directory); final PcesFile b = PcesFile.of( + ancientMode, randomInstant(random), sequenceB, - minimumGenerationB, - maximumGenerationB, + lowerBoundB, + upperBoundB, random.nextLong(1000), directory); @@ -390,65 +432,67 @@ void compareToTest() { } } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("canContain() Test") - void canContainTest() { + void canContainTest(@NonNull final AncientMode ancientMode) { final Random random = getRandomPrintSeed(); final Path directory = Path.of("foo/bar/baz"); for (int i = 0; i < 1000; i++) { final long sequenceNumber = random.nextLong(1000); - final long minimumGeneration = random.nextLong(1000); - final long maximumGeneration = random.nextLong(minimumGeneration + 1, minimumGeneration + 1000); + final long lowerBound = random.nextLong(1000); + final long upperBound = random.nextLong(lowerBound + 1, lowerBound + 1000); final Instant timestamp = RandomUtils.randomInstant(random); final PcesFile file = - PcesFile.of(timestamp, sequenceNumber, minimumGeneration, maximumGeneration, 0, directory); + PcesFile.of(ancientMode, timestamp, sequenceNumber, lowerBound, upperBound, 0, directory); // An event with a sequence number that is too small - assertFalse(file.canContain(minimumGeneration - random.nextLong(1, 100))); + assertFalse(file.canContain(lowerBound - random.nextLong(1, 100))); // An event with a sequence number matching the minimum exactly - assertTrue(file.canContain(minimumGeneration)); + assertTrue(file.canContain(lowerBound)); // An event with a sequence somewhere between the minimum and maximum - assertTrue(file.canContain(maximumGeneration)); + assertTrue(file.canContain(upperBound)); // An event with a sequence somewhere exactly matching the maximum - assertTrue(file.canContain(maximumGeneration)); + assertTrue(file.canContain(upperBound)); // An event with a sequence number that is too big - assertFalse(file.canContain(maximumGeneration + random.nextLong(1, 100))); + assertFalse(file.canContain(upperBound + random.nextLong(1, 100))); } } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Span Compression Test") - void spanCompressionTest() { + void spanCompressionTest(@NonNull final AncientMode ancientMode) { final Random random = getRandomPrintSeed(); final Path directory = Path.of("foo/bar/baz"); final long sequenceNumber = random.nextLong(1000); - final long minimumGeneration = random.nextLong(1000); - final long maximumGeneration = random.nextLong(minimumGeneration + 5, minimumGeneration + 1000); + final long lowerBound = random.nextLong(1000); + final long upperBound = random.nextLong(lowerBound + 5, lowerBound + 1000); final long origin = random.nextLong(1000); final Instant timestamp = randomInstant(random); final PcesFile file = - PcesFile.of(timestamp, sequenceNumber, minimumGeneration, maximumGeneration, origin, directory); + PcesFile.of(ancientMode, timestamp, sequenceNumber, lowerBound, upperBound, origin, directory); - assertThrows(IllegalArgumentException.class, () -> file.buildFileWithCompressedSpan(minimumGeneration - 1)); - assertThrows(IllegalArgumentException.class, () -> file.buildFileWithCompressedSpan(maximumGeneration + 1)); + assertThrows(IllegalArgumentException.class, () -> file.buildFileWithCompressedSpan(lowerBound - 1)); + assertThrows(IllegalArgumentException.class, () -> file.buildFileWithCompressedSpan(upperBound + 1)); - final long newMaximumGeneration = random.nextLong(minimumGeneration, maximumGeneration); + final long newMaximumUpperBound = random.nextLong(lowerBound, upperBound); - final PcesFile compressedFile = file.buildFileWithCompressedSpan(newMaximumGeneration); + final PcesFile compressedFile = file.buildFileWithCompressedSpan(newMaximumUpperBound); assertEquals(sequenceNumber, compressedFile.getSequenceNumber()); - assertEquals(minimumGeneration, compressedFile.getMinimumGeneration()); - assertEquals(newMaximumGeneration, compressedFile.getMaximumGeneration()); + assertEquals(lowerBound, compressedFile.getLowerBound()); + assertEquals(newMaximumUpperBound, compressedFile.getUpperBound()); assertEquals(origin, compressedFile.getOrigin()); assertEquals(timestamp, compressedFile.getTimestamp()); } diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesReadWriteTests.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesReadWriteTests.java index 377812d2d038..c697243b0e95 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesReadWriteTests.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesReadWriteTests.java @@ -18,6 +18,8 @@ import static com.swirlds.common.test.fixtures.io.FileManipulation.corruptFile; import static com.swirlds.common.test.fixtures.io.FileManipulation.truncateFile; +import static com.swirlds.platform.event.AncientMode.BIRTH_ROUND_THRESHOLD; +import static com.swirlds.platform.event.AncientMode.GENERATION_THRESHOLD; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -29,12 +31,14 @@ import com.swirlds.common.io.IOIterator; import com.swirlds.common.io.utility.FileUtils; import com.swirlds.common.test.fixtures.RandomUtils; +import com.swirlds.platform.event.AncientMode; import com.swirlds.platform.event.GossipEvent; import com.swirlds.platform.event.preconsensus.PcesFile; import com.swirlds.platform.event.preconsensus.PcesFileIterator; import com.swirlds.platform.event.preconsensus.PcesMutableFile; import com.swirlds.platform.test.fixtures.event.generator.StandardGraphGenerator; import com.swirlds.platform.test.fixtures.event.source.StandardEventSource; +import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -46,14 +50,15 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Random; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; @DisplayName("PCES Read Write Tests") class PcesReadWriteTests { @@ -80,9 +85,14 @@ void afterEach() throws IOException { FileUtils.deleteDirectory(testDirectory); } - @Test + protected static Stream buildArguments() { + return Stream.of(Arguments.of(GENERATION_THRESHOLD), Arguments.of(BIRTH_ROUND_THRESHOLD)); + } + + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Write Then Read Test") - void writeThenReadTest() throws IOException { + void writeThenReadTest(@NonNull final AncientMode ancientMode) throws IOException { final Random random = RandomUtils.getRandomPrintSeed(); final int numEvents = 100; @@ -99,15 +109,21 @@ void writeThenReadTest() throws IOException { events.add(generator.generateEvent().getBaseEvent()); } - long maximumGeneration = Long.MIN_VALUE; + long upperBound = Long.MIN_VALUE; for (final GossipEvent event : events) { - maximumGeneration = Math.max(maximumGeneration, event.getGeneration()); + upperBound = Math.max(upperBound, event.getAncientIndicator(ancientMode)); } - maximumGeneration += random.nextInt(0, 10); + upperBound += random.nextInt(0, 10); final PcesFile file = PcesFile.of( - RandomUtils.randomInstant(random), random.nextInt(0, 100), 0, maximumGeneration, 0, testDirectory); + ancientMode, + RandomUtils.randomInstant(random), + random.nextInt(0, 100), + 0, + upperBound, + 0, + testDirectory); final PcesMutableFile mutableFile = file.getMutableFile(); for (final GossipEvent event : events) { @@ -125,9 +141,10 @@ void writeThenReadTest() throws IOException { } } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Read Files After Minimum Test") - void readFilesAfterMinimumTest() throws IOException { + void readFilesAfterMinimumTest(@NonNull final AncientMode ancientMode) throws IOException { final Random random = RandomUtils.getRandomPrintSeed(); final int numEvents = 100; @@ -144,21 +161,22 @@ void readFilesAfterMinimumTest() throws IOException { events.add(generator.generateEvent().getBaseEvent()); } - long maximumGeneration = Long.MIN_VALUE; + long upperBound = Long.MIN_VALUE; for (final GossipEvent event : events) { - maximumGeneration = Math.max(maximumGeneration, event.getGeneration()); + upperBound = Math.max(upperBound, event.getAncientIndicator(ancientMode)); } - final long middle = maximumGeneration / 2; + final long middle = upperBound / 2; - maximumGeneration += random.nextInt(0, 10); + upperBound += random.nextInt(0, 10); final PcesFile file = PcesFile.of( + ancientMode, RandomUtils.randomInstant(random), random.nextInt(0, 100), 0, - maximumGeneration, - maximumGeneration, + upperBound, + upperBound, testDirectory); final PcesMutableFile mutableFile = file.getMutableFile(); @@ -172,11 +190,11 @@ void readFilesAfterMinimumTest() throws IOException { final List deserializedEvents = new ArrayList<>(); iterator.forEachRemaining(deserializedEvents::add); - // We don't want any events with a generation less than the middle + // We don't want any events with an ancient indicator less than the middle final Iterator it = events.iterator(); while (it.hasNext()) { final GossipEvent event = it.next(); - if (event.getGeneration() < middle) { + if (event.getAncientIndicator(ancientMode) < middle) { it.remove(); } } @@ -187,12 +205,14 @@ void readFilesAfterMinimumTest() throws IOException { } } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Read Empty File Test") - void readEmptyFileTest() throws IOException { + void readEmptyFileTest(@NonNull final AncientMode ancientMode) throws IOException { final Random random = RandomUtils.getRandomPrintSeed(); final PcesFile file = PcesFile.of( + ancientMode, RandomUtils.randomInstant(random), random.nextInt(0, 100), random.nextLong(0, 1000), @@ -208,74 +228,78 @@ void readEmptyFileTest() throws IOException { } @ParameterizedTest - @ValueSource(booleans = {true, false}) + @MethodSource("buildArguments") @DisplayName("Truncated Event Test") - void truncatedEventTest(final boolean truncateOnBoundary) throws IOException { - final Random random = RandomUtils.getRandomPrintSeed(); - - final int numEvents = 100; + void truncatedEventTest(@NonNull final AncientMode ancientMode) throws IOException { + for (final boolean truncateOnBoundary : List.of(true, false)) { + final Random random = RandomUtils.getRandomPrintSeed(); + + final int numEvents = 100; + + final StandardGraphGenerator generator = new StandardGraphGenerator( + random.nextLong(), + new StandardEventSource(), + new StandardEventSource(), + new StandardEventSource(), + new StandardEventSource()); + + final List events = new ArrayList<>(); + for (int i = 0; i < numEvents; i++) { + events.add(generator.generateEvent().getBaseEvent()); + } - final StandardGraphGenerator generator = new StandardGraphGenerator( - random.nextLong(), - new StandardEventSource(), - new StandardEventSource(), - new StandardEventSource(), - new StandardEventSource()); + long upperBound = Long.MIN_VALUE; + for (final GossipEvent event : events) { + upperBound = Math.max(upperBound, event.getAncientIndicator(ancientMode)); + } - final List events = new ArrayList<>(); - for (int i = 0; i < numEvents; i++) { - events.add(generator.generateEvent().getBaseEvent()); - } + upperBound += random.nextInt(0, 10); - long maximumGeneration = Long.MIN_VALUE; - for (final GossipEvent event : events) { - maximumGeneration = Math.max(maximumGeneration, event.getGeneration()); - } + final PcesFile file = PcesFile.of( + ancientMode, + RandomUtils.randomInstant(random), + random.nextInt(0, 100), + 0, + upperBound, + upperBound, + testDirectory); - maximumGeneration += random.nextInt(0, 10); + final Map byteBoundaries = new HashMap<>(); - final PcesFile file = PcesFile.of( - RandomUtils.randomInstant(random), - random.nextInt(0, 100), - 0, - maximumGeneration, - maximumGeneration, - testDirectory); - - final Map byteBoundaries = new HashMap<>(); - - final PcesMutableFile mutableFile = file.getMutableFile(); - for (int i = 0; i < events.size(); i++) { - final GossipEvent event = events.get(i); - mutableFile.writeEvent(event); - byteBoundaries.put(i, (int) mutableFile.fileSize()); - } + final PcesMutableFile mutableFile = file.getMutableFile(); + for (int i = 0; i < events.size(); i++) { + final GossipEvent event = events.get(i); + mutableFile.writeEvent(event); + byteBoundaries.put(i, (int) mutableFile.fileSize()); + } - mutableFile.close(); + mutableFile.close(); - final int lastEventIndex = - random.nextInt(0, events.size() - 2 /* make sure we always truncate at least one event */); + final int lastEventIndex = + random.nextInt(0, events.size() - 2 /* make sure we always truncate at least one event */); - final int truncationPosition = byteBoundaries.get(lastEventIndex) + (truncateOnBoundary ? 0 : 1); + final int truncationPosition = byteBoundaries.get(lastEventIndex) + (truncateOnBoundary ? 0 : 1); - truncateFile(file.getPath(), truncationPosition); + truncateFile(file.getPath(), truncationPosition); - final PcesFileIterator iterator = file.iterator(Long.MIN_VALUE); - final List deserializedEvents = new ArrayList<>(); - iterator.forEachRemaining(deserializedEvents::add); + final PcesFileIterator iterator = file.iterator(Long.MIN_VALUE); + final List deserializedEvents = new ArrayList<>(); + iterator.forEachRemaining(deserializedEvents::add); - assertEquals(truncateOnBoundary, !iterator.hasPartialEvent()); + assertEquals(truncateOnBoundary, !iterator.hasPartialEvent()); - assertEquals(lastEventIndex + 1, deserializedEvents.size()); + assertEquals(lastEventIndex + 1, deserializedEvents.size()); - for (int i = 0; i < deserializedEvents.size(); i++) { - assertEquals(events.get(i), deserializedEvents.get(i)); + for (int i = 0; i < deserializedEvents.size(); i++) { + assertEquals(events.get(i), deserializedEvents.get(i)); + } } } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Corrupted Events Test") - void corruptedEventsTest() throws IOException { + void corruptedEventsTest(@NonNull final AncientMode ancientMode) throws IOException { final Random random = RandomUtils.getRandomPrintSeed(); final int numEvents = 100; @@ -292,15 +316,21 @@ void corruptedEventsTest() throws IOException { events.add(generator.generateEvent().getBaseEvent()); } - long maximumGeneration = Long.MIN_VALUE; + long upperBound = Long.MIN_VALUE; for (final GossipEvent event : events) { - maximumGeneration = Math.max(maximumGeneration, event.getGeneration()); + upperBound = Math.max(upperBound, event.getAncientIndicator(ancientMode)); } - maximumGeneration += random.nextInt(0, 10); + upperBound += random.nextInt(0, 10); final PcesFile file = PcesFile.of( - RandomUtils.randomInstant(random), random.nextInt(0, 100), 0, maximumGeneration, 0, testDirectory); + ancientMode, + RandomUtils.randomInstant(random), + random.nextInt(0, 100), + 0, + upperBound, + 0, + testDirectory); final Map byteBoundaries = new HashMap<>(); @@ -329,9 +359,10 @@ void corruptedEventsTest() throws IOException { assertThrows(IOException.class, iterator::next); } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Write Invalid Event Test") - void writeInvalidEventTest() throws IOException { + void writeInvalidEventTest(@NonNull final AncientMode ancientMode) throws IOException { final Random random = RandomUtils.getRandomPrintSeed(); final int numEvents = 100; @@ -348,30 +379,31 @@ void writeInvalidEventTest() throws IOException { events.add(generator.generateEvent().getBaseEvent()); } - long minimumGeneration = Long.MAX_VALUE; - long maximumGeneration = Long.MIN_VALUE; + long lowerBound = Long.MAX_VALUE; + long upperBound = Long.MIN_VALUE; for (final GossipEvent event : events) { - minimumGeneration = Math.min(minimumGeneration, event.getGeneration()); - maximumGeneration = Math.max(maximumGeneration, event.getGeneration()); + lowerBound = Math.min(lowerBound, event.getAncientIndicator(ancientMode)); + upperBound = Math.max(upperBound, event.getAncientIndicator(ancientMode)); } - // Intentionally choose minimum and maximum generations that do not permit all generated events - final long restrictedMinimumGeneration = minimumGeneration + (minimumGeneration + maximumGeneration) / 4; - final long restrictedMaximumGeneration = maximumGeneration - (minimumGeneration + maximumGeneration) / 4; + // Intentionally choose minimum and maximum boundaries that do not permit all generated events + final long restrictedLowerBound = lowerBound + (lowerBound + upperBound) / 4; + final long restrictedUpperBound = upperBound - (lowerBound + upperBound) / 4; final PcesFile file = PcesFile.of( + ancientMode, RandomUtils.randomInstant(random), random.nextInt(0, 100), - restrictedMinimumGeneration, - restrictedMaximumGeneration, + restrictedLowerBound, + restrictedUpperBound, 0, testDirectory); final PcesMutableFile mutableFile = file.getMutableFile(); final List validEvents = new ArrayList<>(); for (final GossipEvent event : events) { - if (event.getGeneration() >= restrictedMinimumGeneration - && event.getGeneration() <= restrictedMaximumGeneration) { + if (event.getAncientIndicator(ancientMode) >= restrictedLowerBound + && event.getAncientIndicator(ancientMode) <= restrictedUpperBound) { mutableFile.writeEvent(event); validEvents.add(event); } else { @@ -389,9 +421,10 @@ void writeInvalidEventTest() throws IOException { assertFalse(iterator.hasNext()); } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Span Compression Test") - void spanCompressionTest() throws IOException { + void spanCompressionTest(@NonNull final AncientMode ancientMode) throws IOException { final Random random = RandomUtils.getRandomPrintSeed(0); final int numEvents = 100; @@ -408,20 +441,21 @@ void spanCompressionTest() throws IOException { events.add(generator.generateEvent().getBaseEvent()); } - long minimumGeneration = Long.MAX_VALUE; - long maximumGeneration = Long.MIN_VALUE; + long lowerBound = Long.MAX_VALUE; + long upperBound = Long.MIN_VALUE; for (final GossipEvent event : events) { - minimumGeneration = Math.min(minimumGeneration, event.getGeneration()); - maximumGeneration = Math.max(maximumGeneration, event.getGeneration()); + lowerBound = Math.min(lowerBound, event.getAncientIndicator(ancientMode)); + upperBound = Math.max(upperBound, event.getAncientIndicator(ancientMode)); } - maximumGeneration += random.nextInt(1, 10); + upperBound += random.nextInt(1, 10); final PcesFile file = PcesFile.of( + ancientMode, RandomUtils.randomInstant(random), random.nextInt(0, 100), - minimumGeneration, - maximumGeneration, + lowerBound, + upperBound, 0, testDirectory); @@ -431,17 +465,15 @@ void spanCompressionTest() throws IOException { } mutableFile.close(); - final PcesFile compressedFile = mutableFile.compressGenerationalSpan(0); + final PcesFile compressedFile = mutableFile.compressSpan(0); assertEquals(file.getPath().getParent(), compressedFile.getPath().getParent()); assertEquals(file.getSequenceNumber(), compressedFile.getSequenceNumber()); - assertEquals(file.getMinimumGeneration(), compressedFile.getMinimumGeneration()); - assertTrue(maximumGeneration > compressedFile.getMaximumGeneration()); - assertEquals( - mutableFile.getUtilizedGenerationalSpan(), - compressedFile.getMaximumGeneration() - compressedFile.getMinimumGeneration()); + assertEquals(file.getLowerBound(), compressedFile.getLowerBound()); + assertTrue(upperBound > compressedFile.getUpperBound()); + assertEquals(mutableFile.getUtilizedSpan(), compressedFile.getUpperBound() - compressedFile.getLowerBound()); assertNotEquals(file.getPath(), compressedFile.getPath()); - assertNotEquals(file.getMaximumGeneration(), compressedFile.getMaximumGeneration()); + assertNotEquals(file.getUpperBound(), compressedFile.getUpperBound()); assertTrue(Files.exists(compressedFile.getPath())); assertFalse(Files.exists(file.getPath())); @@ -454,9 +486,10 @@ void spanCompressionTest() throws IOException { } } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Partial Span Compression Test") - void partialSpanCompressionTest() throws IOException { + void partialSpanCompressionTest(@NonNull final AncientMode ancientMode) throws IOException { final Random random = RandomUtils.getRandomPrintSeed(0); final int numEvents = 100; @@ -473,21 +506,22 @@ void partialSpanCompressionTest() throws IOException { events.add(generator.generateEvent().getBaseEvent()); } - long minimumEventGeneration = Long.MAX_VALUE; - long maximumEventGeneration = Long.MIN_VALUE; + long lowerBound = Long.MAX_VALUE; + long upperBound = Long.MIN_VALUE; for (final GossipEvent event : events) { - minimumEventGeneration = Math.min(minimumEventGeneration, event.getGeneration()); - maximumEventGeneration = Math.max(maximumEventGeneration, event.getGeneration()); + lowerBound = Math.min(lowerBound, event.getAncientIndicator(ancientMode)); + upperBound = Math.max(upperBound, event.getAncientIndicator(ancientMode)); } - final long maximumFileGeneration = maximumEventGeneration + random.nextInt(10, 20); + final long maximumFileBoundary = upperBound + random.nextInt(10, 20); final long uncompressedSpan = 5; final PcesFile file = PcesFile.of( + ancientMode, RandomUtils.randomInstant(random), random.nextInt(0, 100), - minimumEventGeneration, - maximumFileGeneration, + lowerBound, + maximumFileBoundary, 0, testDirectory); @@ -497,17 +531,17 @@ void partialSpanCompressionTest() throws IOException { } mutableFile.close(); - final PcesFile compressedFile = mutableFile.compressGenerationalSpan(maximumEventGeneration + uncompressedSpan); + final PcesFile compressedFile = mutableFile.compressSpan(upperBound + uncompressedSpan); assertEquals(file.getPath().getParent(), compressedFile.getPath().getParent()); assertEquals(file.getSequenceNumber(), compressedFile.getSequenceNumber()); - assertEquals(file.getMinimumGeneration(), compressedFile.getMinimumGeneration()); - assertEquals(maximumEventGeneration + uncompressedSpan, compressedFile.getMaximumGeneration()); + assertEquals(file.getLowerBound(), compressedFile.getLowerBound()); + assertEquals(upperBound + uncompressedSpan, compressedFile.getUpperBound()); assertEquals( - mutableFile.getUtilizedGenerationalSpan(), - compressedFile.getMaximumGeneration() - compressedFile.getMinimumGeneration() - uncompressedSpan); + mutableFile.getUtilizedSpan(), + compressedFile.getUpperBound() - compressedFile.getLowerBound() - uncompressedSpan); assertNotEquals(file.getPath(), compressedFile.getPath()); - assertNotEquals(file.getMaximumGeneration(), compressedFile.getMaximumGeneration()); + assertNotEquals(file.getUpperBound(), compressedFile.getUpperBound()); assertTrue(Files.exists(compressedFile.getPath())); assertFalse(Files.exists(file.getPath())); @@ -520,10 +554,11 @@ void partialSpanCompressionTest() throws IOException { } } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Empty File Test") - void emptyFileTest() throws IOException { - final PcesFile file = PcesFile.of(Instant.now(), 0, 0, 100, 0, testDirectory); + void emptyFileTest(@NonNull final AncientMode ancientMode) throws IOException { + final PcesFile file = PcesFile.of(ancientMode, Instant.now(), 0, 0, 100, 0, testDirectory); final Path path = file.getPath(); diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesUtilitiesTests.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesUtilitiesTests.java index 26f34a8eb9ea..458fbab85903 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesUtilitiesTests.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesUtilitiesTests.java @@ -16,61 +16,76 @@ package com.swirlds.platform.test.event.preconsensus; +import static com.swirlds.platform.event.AncientMode.BIRTH_ROUND_THRESHOLD; +import static com.swirlds.platform.event.AncientMode.GENERATION_THRESHOLD; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertThrows; import com.swirlds.base.test.fixtures.time.FakeTime; +import com.swirlds.platform.event.AncientMode; import com.swirlds.platform.event.preconsensus.PcesFile; import com.swirlds.platform.event.preconsensus.PcesUtilities; +import edu.umd.cs.findbugs.annotations.NonNull; import java.nio.file.Path; import java.time.Duration; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; /** * Tests for {@link PcesUtilities} */ class PcesUtilitiesTests { private FakeTime time; - private PcesFile previousFileDescriptor; @BeforeEach void setup() { time = new FakeTime(); time.tick(Duration.ofSeconds(100)); - previousFileDescriptor = PcesFile.of(time.now(), 2, 10, 20, 5, Path.of("root")); } - @Test + protected static Stream buildArguments() { + return Stream.of(Arguments.of(GENERATION_THRESHOLD), Arguments.of(BIRTH_ROUND_THRESHOLD)); + } + + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Standard operation") - void standardOperation() { + void standardOperation(@NonNull final AncientMode ancientMode) { + final PcesFile previousFileDescriptor = PcesFile.of(ancientMode, time.now(), 2, 10, 20, 5, Path.of("root")); final PcesFile currentFileDescriptor = PcesFile.of( + ancientMode, time.now(), previousFileDescriptor.getSequenceNumber() + 1, - previousFileDescriptor.getMinimumGeneration(), - previousFileDescriptor.getMaximumGeneration(), + previousFileDescriptor.getLowerBound(), + previousFileDescriptor.getUpperBound(), previousFileDescriptor.getOrigin(), Path.of("root")); assertDoesNotThrow(() -> PcesUtilities.fileSanityChecks( false, previousFileDescriptor.getSequenceNumber(), - previousFileDescriptor.getMinimumGeneration(), - previousFileDescriptor.getMaximumGeneration(), + previousFileDescriptor.getLowerBound(), + previousFileDescriptor.getUpperBound(), previousFileDescriptor.getOrigin(), previousFileDescriptor.getTimestamp(), currentFileDescriptor)); } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Decreasing sequence number") - void decreasingSequenceNumber() { + void decreasingSequenceNumber(@NonNull final AncientMode ancientMode) { + final PcesFile previousFileDescriptor = PcesFile.of(ancientMode, time.now(), 2, 10, 20, 5, Path.of("root")); final PcesFile currentFileDescriptor = PcesFile.of( + ancientMode, time.now(), previousFileDescriptor.getSequenceNumber() - 1, - previousFileDescriptor.getMinimumGeneration(), - previousFileDescriptor.getMaximumGeneration(), + previousFileDescriptor.getLowerBound(), + previousFileDescriptor.getUpperBound(), previousFileDescriptor.getOrigin(), Path.of("root")); @@ -79,42 +94,48 @@ void decreasingSequenceNumber() { () -> PcesUtilities.fileSanityChecks( false, previousFileDescriptor.getSequenceNumber(), - previousFileDescriptor.getMinimumGeneration(), - previousFileDescriptor.getMaximumGeneration(), + previousFileDescriptor.getLowerBound(), + previousFileDescriptor.getUpperBound(), previousFileDescriptor.getOrigin(), previousFileDescriptor.getTimestamp(), currentFileDescriptor)); } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Decreasing sequence number with gaps permitted") - void decreasingSequenceNumberWithGapsPermitted() { + void decreasingSequenceNumberWithGapsPermitted(@NonNull final AncientMode ancientMode) { + final PcesFile previousFileDescriptor = PcesFile.of(ancientMode, time.now(), 2, 10, 20, 5, Path.of("root")); final PcesFile currentFileDescriptor = PcesFile.of( + ancientMode, time.now(), previousFileDescriptor.getSequenceNumber() - 1, - previousFileDescriptor.getMinimumGeneration(), - previousFileDescriptor.getMaximumGeneration(), + previousFileDescriptor.getLowerBound(), + previousFileDescriptor.getUpperBound(), previousFileDescriptor.getOrigin(), Path.of("root")); assertDoesNotThrow(() -> PcesUtilities.fileSanityChecks( true, previousFileDescriptor.getSequenceNumber(), - previousFileDescriptor.getMinimumGeneration(), - previousFileDescriptor.getMaximumGeneration(), + previousFileDescriptor.getLowerBound(), + previousFileDescriptor.getUpperBound(), previousFileDescriptor.getOrigin(), previousFileDescriptor.getTimestamp(), currentFileDescriptor)); } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Non-increasing sequence number") - void nonIncreasingSequenceNumber() { + void nonIncreasingSequenceNumber(@NonNull final AncientMode ancientMode) { + final PcesFile previousFileDescriptor = PcesFile.of(ancientMode, time.now(), 2, 10, 20, 5, Path.of("root")); final PcesFile currentFileDescriptor = PcesFile.of( + ancientMode, time.now(), previousFileDescriptor.getSequenceNumber(), - previousFileDescriptor.getMinimumGeneration(), - previousFileDescriptor.getMaximumGeneration(), + previousFileDescriptor.getLowerBound(), + previousFileDescriptor.getUpperBound(), previousFileDescriptor.getOrigin(), Path.of("root")); @@ -123,21 +144,24 @@ void nonIncreasingSequenceNumber() { () -> PcesUtilities.fileSanityChecks( false, previousFileDescriptor.getSequenceNumber(), - previousFileDescriptor.getMinimumGeneration(), - previousFileDescriptor.getMaximumGeneration(), + previousFileDescriptor.getLowerBound(), + previousFileDescriptor.getUpperBound(), previousFileDescriptor.getOrigin(), previousFileDescriptor.getTimestamp(), currentFileDescriptor)); } - @Test - @DisplayName("Decreasing minimum generation") - void decreasingMinimumGeneration() { + @ParameterizedTest + @MethodSource("buildArguments") + @DisplayName("Decreasing Lower Bound") + void decreasingMinimumLowerBound(@NonNull final AncientMode ancientMode) { + final PcesFile previousFileDescriptor = PcesFile.of(ancientMode, time.now(), 2, 10, 20, 5, Path.of("root")); final PcesFile currentFileDescriptor = PcesFile.of( + ancientMode, time.now(), previousFileDescriptor.getSequenceNumber() + 1, - previousFileDescriptor.getMinimumGeneration() - 1, - previousFileDescriptor.getMaximumGeneration(), + previousFileDescriptor.getLowerBound() - 1, + previousFileDescriptor.getUpperBound(), previousFileDescriptor.getOrigin(), Path.of("root")); @@ -146,21 +170,24 @@ void decreasingMinimumGeneration() { () -> PcesUtilities.fileSanityChecks( false, previousFileDescriptor.getSequenceNumber(), - previousFileDescriptor.getMinimumGeneration(), - previousFileDescriptor.getMaximumGeneration(), + previousFileDescriptor.getLowerBound(), + previousFileDescriptor.getUpperBound(), previousFileDescriptor.getOrigin(), previousFileDescriptor.getTimestamp(), currentFileDescriptor)); } - @Test - @DisplayName("Decreasing maximum generation") - void decreasingMaximumGeneration() { + @ParameterizedTest + @MethodSource("buildArguments") + @DisplayName("Decreasing Upper Bound") + void decreasingUpperBound(@NonNull final AncientMode ancientMode) { + final PcesFile previousFileDescriptor = PcesFile.of(ancientMode, time.now(), 2, 10, 20, 5, Path.of("root")); final PcesFile currentFileDescriptor = PcesFile.of( + ancientMode, time.now(), previousFileDescriptor.getSequenceNumber() + 1, - previousFileDescriptor.getMinimumGeneration(), - previousFileDescriptor.getMaximumGeneration() - 1, + previousFileDescriptor.getLowerBound(), + previousFileDescriptor.getUpperBound() - 1, previousFileDescriptor.getOrigin(), Path.of("root")); @@ -169,21 +196,24 @@ void decreasingMaximumGeneration() { () -> PcesUtilities.fileSanityChecks( false, previousFileDescriptor.getSequenceNumber(), - previousFileDescriptor.getMinimumGeneration(), - previousFileDescriptor.getMaximumGeneration(), + previousFileDescriptor.getLowerBound(), + previousFileDescriptor.getUpperBound(), previousFileDescriptor.getOrigin(), previousFileDescriptor.getTimestamp(), currentFileDescriptor)); } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Decreasing timestamp") - void decreasingTimestamp() { + void decreasingTimestamp(@NonNull final AncientMode ancientMode) { + final PcesFile previousFileDescriptor = PcesFile.of(ancientMode, time.now(), 2, 10, 20, 5, Path.of("root")); final PcesFile currentFileDescriptor = PcesFile.of( + ancientMode, previousFileDescriptor.getTimestamp().minusSeconds(10), previousFileDescriptor.getSequenceNumber() + 1, - previousFileDescriptor.getMinimumGeneration(), - previousFileDescriptor.getMaximumGeneration(), + previousFileDescriptor.getLowerBound(), + previousFileDescriptor.getUpperBound(), previousFileDescriptor.getOrigin(), Path.of("root")); @@ -192,21 +222,24 @@ void decreasingTimestamp() { () -> PcesUtilities.fileSanityChecks( false, previousFileDescriptor.getSequenceNumber(), - previousFileDescriptor.getMinimumGeneration(), - previousFileDescriptor.getMaximumGeneration(), + previousFileDescriptor.getLowerBound(), + previousFileDescriptor.getUpperBound(), previousFileDescriptor.getOrigin(), previousFileDescriptor.getTimestamp(), currentFileDescriptor)); } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Decreasing origin") - void decreasingOrigin() { + void decreasingOrigin(@NonNull final AncientMode ancientMode) { + final PcesFile previousFileDescriptor = PcesFile.of(ancientMode, time.now(), 2, 10, 20, 5, Path.of("root")); final PcesFile currentFileDescriptor = PcesFile.of( + ancientMode, time.now(), previousFileDescriptor.getSequenceNumber() + 1, - previousFileDescriptor.getMinimumGeneration(), - previousFileDescriptor.getMaximumGeneration(), + previousFileDescriptor.getLowerBound(), + previousFileDescriptor.getUpperBound(), previousFileDescriptor.getOrigin() - 1, Path.of("root")); @@ -215,8 +248,8 @@ void decreasingOrigin() { () -> PcesUtilities.fileSanityChecks( false, previousFileDescriptor.getSequenceNumber(), - previousFileDescriptor.getMinimumGeneration(), - previousFileDescriptor.getMaximumGeneration(), + previousFileDescriptor.getLowerBound(), + previousFileDescriptor.getUpperBound(), previousFileDescriptor.getOrigin(), previousFileDescriptor.getTimestamp(), currentFileDescriptor)); diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesWriterTests.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesWriterTests.java index 79df31259754..a9e49d4b37f3 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesWriterTests.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/event/preconsensus/PcesWriterTests.java @@ -19,13 +19,14 @@ import static com.swirlds.common.units.DataUnit.UNIT_BYTES; import static com.swirlds.common.units.DataUnit.UNIT_KILOBYTES; import static com.swirlds.common.utility.CompareTo.isGreaterThanOrEqualTo; -import static com.swirlds.platform.event.preconsensus.PcesFileManager.NO_MINIMUM_GENERATION; +import static com.swirlds.platform.event.AncientMode.BIRTH_ROUND_THRESHOLD; +import static com.swirlds.platform.event.AncientMode.GENERATION_THRESHOLD; +import static com.swirlds.platform.event.preconsensus.PcesFileManager.NO_LOWER_BOUND; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import com.swirlds.base.test.fixtures.time.FakeTime; -import com.swirlds.base.time.Time; import com.swirlds.common.config.TransactionConfig_; import com.swirlds.common.constructable.ConstructableRegistry; import com.swirlds.common.constructable.ConstructableRegistryException; @@ -42,6 +43,8 @@ import com.swirlds.common.test.fixtures.TransactionGenerator; import com.swirlds.common.test.fixtures.io.FileManipulation; import com.swirlds.config.api.Configuration; +import com.swirlds.platform.consensus.NonAncientEventWindow; +import com.swirlds.platform.event.AncientMode; import com.swirlds.platform.event.GossipEvent; import com.swirlds.platform.event.preconsensus.EventDurabilityNexus; import com.swirlds.platform.event.preconsensus.PcesConfig_; @@ -53,6 +56,7 @@ import com.swirlds.platform.event.preconsensus.PcesSequencer; import com.swirlds.platform.event.preconsensus.PcesUtilities; import com.swirlds.platform.event.preconsensus.PcesWriter; +import com.swirlds.platform.eventhandling.EventConfig_; import com.swirlds.platform.system.transaction.ConsensusTransactionImpl; import com.swirlds.platform.system.transaction.SwirldTransaction; import com.swirlds.platform.test.fixtures.event.generator.StandardGraphGenerator; @@ -74,14 +78,15 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; @DisplayName("PcesWriter Tests") class PcesWriterTests { @@ -92,16 +97,12 @@ class PcesWriterTests { @TempDir Path testDirectory; - private FakeTime time; private final NodeId selfId = new NodeId(0); private final int numEvents = 1_000; - private PlatformContext platformContext; - private StandardGraphGenerator generator; - private int generationsUntilAncient; - private PcesSequencer sequencer; - private PcesFileTracker pcesFiles; - private PcesWriter writer; - private EventDurabilityNexus eventDurabilityNexus; + + protected static Stream buildArguments() { + return Stream.of(Arguments.of(GENERATION_THRESHOLD), Arguments.of(BIRTH_ROUND_THRESHOLD)); + } /** * Perform verification on a stream written by a {@link PcesWriter}. @@ -109,16 +110,18 @@ class PcesWriterTests { * @param events the events that were written to the stream * @param platformContext the platform context * @param truncatedFileCount the expected number of truncated files + * @param ancientMode the ancient mode */ private void verifyStream( @NonNull final List events, @NonNull final PlatformContext platformContext, - final int truncatedFileCount) + final int truncatedFileCount, + @NonNull final AncientMode ancientMode) throws IOException { - long lastGeneration = Long.MIN_VALUE; + long lastAncientIdentifier = Long.MIN_VALUE; for (final GossipEvent event : events) { - lastGeneration = Math.max(lastGeneration, event.getGeneration()); + lastAncientIdentifier = Math.max(lastAncientIdentifier, event.getAncientIndicator(ancientMode)); } final PcesFileTracker pcesFiles = PcesFileReader.readFilesFromDisk( @@ -126,7 +129,8 @@ private void verifyStream( TestRecycleBin.getInstance(), PcesUtilities.getDatabaseDirectory(platformContext, selfId), 0, - false); + false, + ancientMode); // Verify that the events were written correctly final PcesMultiFileIterator eventsIterator = pcesFiles.getEventIterator(0, 0); @@ -139,10 +143,10 @@ private void verifyStream( assertEquals(truncatedFileCount, eventsIterator.getTruncatedFileCount()); // Make sure things look good when iterating starting in the middle of the stream that was written - final long startingGeneration = lastGeneration / 2; - final IOIterator eventsIterator2 = pcesFiles.getEventIterator(startingGeneration, 0); + final long startingLowerBound = lastAncientIdentifier / 2; + final IOIterator eventsIterator2 = pcesFiles.getEventIterator(startingLowerBound, 0); for (final GossipEvent event : events) { - if (event.getGeneration() < startingGeneration) { + if (event.getAncientIndicator(ancientMode) < startingLowerBound) { continue; } assertTrue(eventsIterator2.hasNext()); @@ -150,8 +154,8 @@ private void verifyStream( } assertFalse(eventsIterator2.hasNext()); - // Iterating from a high generation should yield no events - final IOIterator eventsIterator3 = pcesFiles.getEventIterator(lastGeneration + 1, 0); + // Iterating from a high ancient indicator should yield no events + final IOIterator eventsIterator3 = pcesFiles.getEventIterator(lastAncientIdentifier + 1, 0); assertFalse(eventsIterator3.hasNext()); // Do basic validation on event files @@ -172,17 +176,17 @@ private void verifyStream( nextSequenceNumber++; assertTrue(isGreaterThanOrEqualTo(file.getTimestamp(), previousTimestamp)); previousTimestamp = file.getTimestamp(); - assertTrue(file.getMinimumGeneration() <= file.getMaximumGeneration()); - assertTrue(file.getMinimumGeneration() >= previousMinimum); - previousMinimum = file.getMinimumGeneration(); - assertTrue(file.getMaximumGeneration() >= previousMaximum); - previousMaximum = file.getMaximumGeneration(); + assertTrue(file.getLowerBound() <= file.getUpperBound()); + assertTrue(file.getLowerBound() >= previousMinimum); + previousMinimum = file.getLowerBound(); + assertTrue(file.getUpperBound() >= previousMaximum); + previousMaximum = file.getUpperBound(); final IOIterator fileEvents = file.iterator(0); while (fileEvents.hasNext()) { final GossipEvent event = fileEvents.next(); - assertTrue(event.getGeneration() >= file.getMinimumGeneration()); - assertTrue(event.getGeneration() <= file.getMaximumGeneration()); + assertTrue(event.getAncientIndicator(ancientMode) >= file.getLowerBound()); + assertTrue(event.getAncientIndicator(ancientMode) <= file.getUpperBound()); } } } @@ -238,19 +242,6 @@ static void beforeAll() throws ConstructableRegistryException { void beforeEach() throws IOException { FileUtils.deleteDirectory(testDirectory); Files.createDirectories(testDirectory); - - platformContext = buildContext(); - - final Random random = RandomUtils.getRandomPrintSeed(); - generator = buildGraphGenerator(random); - generationsUntilAncient = random.nextInt(50, 100); - sequencer = new PcesSequencer(); - pcesFiles = new PcesFileTracker(); - - time = new FakeTime(Duration.ofMillis(1)); - final PcesFileManager fileManager = new PcesFileManager(platformContext, time, pcesFiles, selfId, 0); - writer = new PcesWriter(platformContext, fileManager); - eventDurabilityNexus = new EventDurabilityNexus(); } @AfterEach @@ -258,18 +249,21 @@ void afterEach() throws IOException { FileUtils.deleteDirectory(testDirectory); } - private PlatformContext buildContext() { + @NonNull + private PlatformContext buildContext(@NonNull final AncientMode ancientMode) { final Configuration configuration = new TestConfigBuilder() .withValue(PcesConfig_.DATABASE_DIRECTORY, testDirectory) .withValue(PcesConfig_.PREFERRED_FILE_SIZE_MEGABYTES, 5) .withValue(TransactionConfig_.MAX_TRANSACTION_BYTES_PER_EVENT, Integer.MAX_VALUE) .withValue(TransactionConfig_.MAX_TRANSACTION_COUNT_PER_EVENT, Integer.MAX_VALUE) .withValue(TransactionConfig_.TRANSACTION_MAX_BYTES, Integer.MAX_VALUE) + .withValue(EventConfig_.USE_BIRTH_ROUND_ANCIENT_THRESHOLD, ancientMode == BIRTH_ROUND_THRESHOLD) .getOrCreateConfig(); final Metrics metrics = new NoOpMetrics(); - return new DefaultPlatformContext(configuration, metrics, CryptographyHolder.get(), Time.getCurrent()); + return new DefaultPlatformContext( + configuration, metrics, CryptographyHolder.get(), new FakeTime(Duration.ofMillis(1))); } /** @@ -279,16 +273,34 @@ private PlatformContext buildContext() { * number. This simulates the components being wired together. * * @param mostRecentDurableSequenceNumber the most recent durable sequence number + * @param eventDurabilityNexus the event durability nexus */ - private void passValueToDurabilityNexus(@Nullable final Long mostRecentDurableSequenceNumber) { + private void passValueToDurabilityNexus( + @Nullable final Long mostRecentDurableSequenceNumber, + @NonNull final EventDurabilityNexus eventDurabilityNexus) { if (mostRecentDurableSequenceNumber != null) { eventDurabilityNexus.setLatestDurableSequenceNumber(mostRecentDurableSequenceNumber); } } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Standard Operation Test") - void standardOperationTest() throws IOException { + void standardOperationTest(@NonNull final AncientMode ancientMode) throws IOException { + + final Random random = RandomUtils.getRandomPrintSeed(); + + final PlatformContext platformContext = buildContext(ancientMode); + + final StandardGraphGenerator generator = buildGraphGenerator(random); + final int stepsUntilAncient = random.nextInt(50, 100); + final PcesSequencer sequencer = new PcesSequencer(); + final PcesFileTracker pcesFiles = new PcesFileTracker(ancientMode); + + final PcesFileManager fileManager = new PcesFileManager(platformContext, pcesFiles, selfId, 0); + final PcesWriter writer = new PcesWriter(platformContext, fileManager); + final EventDurabilityNexus eventDurabilityNexus = new EventDurabilityNexus(); + final List events = new LinkedList<>(); for (int i = 0; i < numEvents; i++) { events.add(generator.generateEvent().getBaseEvent()); @@ -298,19 +310,21 @@ void standardOperationTest() throws IOException { final Collection rejectedEvents = new HashSet<>(); - long minimumGenerationNonAncient = 0; + long lowerBound = 0; final Iterator iterator = events.iterator(); while (iterator.hasNext()) { final GossipEvent event = iterator.next(); sequencer.assignStreamSequenceNumber(event); - passValueToDurabilityNexus(writer.writeEvent(event)); + passValueToDurabilityNexus(writer.writeEvent(event), eventDurabilityNexus); - minimumGenerationNonAncient = - Math.max(minimumGenerationNonAncient, event.getGeneration() - generationsUntilAncient); - passValueToDurabilityNexus(writer.setMinimumGenerationNonAncient(minimumGenerationNonAncient)); + lowerBound = Math.max(lowerBound, event.getAncientIndicator(ancientMode) - stepsUntilAncient); + passValueToDurabilityNexus( + writer.updateNonAncientEventBoundary( + new NonAncientEventWindow(1, lowerBound, lowerBound, ancientMode)), + eventDurabilityNexus); - if (event.getGeneration() < minimumGenerationNonAncient) { + if (event.getAncientIndicator(ancientMode) < lowerBound) { // Although it's not common, it's possible that the generator will generate // an event that is ancient (since it isn't aware of what we consider to be ancient) rejectedEvents.add(event); @@ -321,14 +335,29 @@ void standardOperationTest() throws IOException { events.forEach(event -> assertTrue(eventDurabilityNexus.isEventDurable(event))); rejectedEvents.forEach(event -> assertFalse(eventDurabilityNexus.isEventDurable(event))); - verifyStream(events, platformContext, 0); + verifyStream(events, platformContext, 0, ancientMode); writer.closeCurrentMutableFile(); } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Ancient Event Test") - void ancientEventTest() throws IOException { + void ancientEventTest(@NonNull final AncientMode ancientMode) throws IOException { + + final Random random = RandomUtils.getRandomPrintSeed(); + + final PlatformContext platformContext = buildContext(ancientMode); + + final StandardGraphGenerator generator = buildGraphGenerator(random); + final int stepsUntilAncient = random.nextInt(50, 100); + final PcesSequencer sequencer = new PcesSequencer(); + final PcesFileTracker pcesFiles = new PcesFileTracker(ancientMode); + + final PcesFileManager fileManager = new PcesFileManager(platformContext, pcesFiles, selfId, 0); + final PcesWriter writer = new PcesWriter(platformContext, fileManager); + final EventDurabilityNexus eventDurabilityNexus = new EventDurabilityNexus(); + // We will add this event at the very end, it should be ancient by then final GossipEvent ancientEvent = generator.generateEvent().getBaseEvent(); @@ -341,19 +370,21 @@ void ancientEventTest() throws IOException { final Collection rejectedEvents = new HashSet<>(); - long minimumGenerationNonAncient = 0; + long lowerBound = 0; final Iterator iterator = events.iterator(); while (iterator.hasNext()) { final GossipEvent event = iterator.next(); sequencer.assignStreamSequenceNumber(event); - passValueToDurabilityNexus(writer.writeEvent(event)); + passValueToDurabilityNexus(writer.writeEvent(event), eventDurabilityNexus); - minimumGenerationNonAncient = - Math.max(minimumGenerationNonAncient, event.getGeneration() - generationsUntilAncient); - passValueToDurabilityNexus(writer.setMinimumGenerationNonAncient(minimumGenerationNonAncient)); + lowerBound = Math.max(lowerBound, event.getAncientIndicator(ancientMode) - stepsUntilAncient); + passValueToDurabilityNexus( + writer.updateNonAncientEventBoundary( + new NonAncientEventWindow(1, lowerBound, lowerBound, ancientMode)), + eventDurabilityNexus); - if (event.getGeneration() < minimumGenerationNonAncient) { + if (event.getAncientIndicator(ancientMode) < lowerBound) { // Although it's not common, it's actually possible that the generator will generate // an event that is ancient (since it isn't aware of what we consider to be ancient) rejectedEvents.add(event); @@ -363,34 +394,53 @@ void ancientEventTest() throws IOException { // Add the ancient event sequencer.assignStreamSequenceNumber(ancientEvent); - if (minimumGenerationNonAncient > ancientEvent.getGeneration()) { + if (lowerBound > ancientEvent.getAncientIndicator(ancientMode)) { // This is probably not possible... but just in case make sure this event is ancient try { - passValueToDurabilityNexus(writer.setMinimumGenerationNonAncient(ancientEvent.getGeneration() + 1)); + passValueToDurabilityNexus( + writer.updateNonAncientEventBoundary(new NonAncientEventWindow( + 1, + ancientEvent.getAncientIndicator(ancientMode) + 1, + ancientEvent.getAncientIndicator(ancientMode) + 1, + ancientMode)), + eventDurabilityNexus); } catch (final IllegalArgumentException e) { - // ignore, more likely than not this event is way older than the actual ancient generation + // ignore, more likely than not this event is way older than the actual ancient threshold } } - passValueToDurabilityNexus(writer.writeEvent(ancientEvent)); + passValueToDurabilityNexus(writer.writeEvent(ancientEvent), eventDurabilityNexus); rejectedEvents.add(ancientEvent); assertEquals(GossipEvent.STALE_EVENT_STREAM_SEQUENCE_NUMBER, ancientEvent.getStreamSequenceNumber()); events.forEach(event -> assertTrue(eventDurabilityNexus.isEventDurable(event))); rejectedEvents.forEach(event -> assertFalse(eventDurabilityNexus.isEventDurable(event))); - verifyStream(events, platformContext, 0); + verifyStream(events, platformContext, 0, ancientMode); writer.closeCurrentMutableFile(); } /** - * In this test, keep adding events without increasing the first non-ancient generation. This will force the - * preferred generations per file to eventually be reached and exceeded. + * In this test, keep adding events without increasing the first non-ancient threshold. This will force the + * preferred span per file to eventually be reached and exceeded. */ - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("Overflow Test") - void overflowTest() throws IOException { + void overflowTest(@NonNull final AncientMode ancientMode) throws IOException { + final Random random = RandomUtils.getRandomPrintSeed(); + + final PlatformContext platformContext = buildContext(ancientMode); + + final StandardGraphGenerator generator = buildGraphGenerator(random); + final PcesSequencer sequencer = new PcesSequencer(); + final PcesFileTracker pcesFiles = new PcesFileTracker(ancientMode); + + final PcesFileManager fileManager = new PcesFileManager(platformContext, pcesFiles, selfId, 0); + final PcesWriter writer = new PcesWriter(platformContext, fileManager); + final EventDurabilityNexus eventDurabilityNexus = new EventDurabilityNexus(); + final List events = new LinkedList<>(); for (int i = 0; i < numEvents; i++) { events.add(generator.generateEvent().getBaseEvent()); @@ -400,24 +450,38 @@ void overflowTest() throws IOException { for (final GossipEvent event : events) { sequencer.assignStreamSequenceNumber(event); - passValueToDurabilityNexus(writer.writeEvent(event)); + passValueToDurabilityNexus(writer.writeEvent(event), eventDurabilityNexus); } writer.closeCurrentMutableFile(); - verifyStream(events, platformContext, 0); + verifyStream(events, platformContext, 0, ancientMode); - // Without advancing the first non-ancient generation, - // we should never be able to increase the minimum generation from 0. + // Without advancing the first non-ancient threshold, + // we should never be able to increase the lower bound from 0. for (final Iterator it = pcesFiles.getFileIterator(0, 0); it.hasNext(); ) { final PcesFile file = it.next(); - assertEquals(0, file.getMinimumGeneration()); + assertEquals(0, file.getLowerBound()); } } - @Test + @ParameterizedTest + @MethodSource("buildArguments") @DisplayName("beginStreamingEvents() Test") - void beginStreamingEventsTest() { + void beginStreamingEventsTest(@NonNull final AncientMode ancientMode) throws IOException { + final Random random = RandomUtils.getRandomPrintSeed(); + + final PlatformContext platformContext = buildContext(ancientMode); + + final StandardGraphGenerator generator = buildGraphGenerator(random); + final int stepsUntilAncient = random.nextInt(50, 100); + final PcesSequencer sequencer = new PcesSequencer(); + final PcesFileTracker pcesFiles = new PcesFileTracker(ancientMode); + + final PcesFileManager fileManager = new PcesFileManager(platformContext, pcesFiles, selfId, 0); + final PcesWriter writer = new PcesWriter(platformContext, fileManager); + final EventDurabilityNexus eventDurabilityNexus = new EventDurabilityNexus(); + final List events = new LinkedList<>(); for (int i = 0; i < numEvents; i++) { events.add(generator.generateEvent().getBaseEvent()); @@ -426,118 +490,155 @@ void beginStreamingEventsTest() { // We intentionally do not call writer.beginStreamingNewEvents(). This should cause all events // passed into the writer to be more or less ignored. - long minimumGenerationNonAncient = 0; + long lowerBound = 0; for (final GossipEvent event : events) { sequencer.assignStreamSequenceNumber(event); - passValueToDurabilityNexus(writer.writeEvent(event)); + passValueToDurabilityNexus(writer.writeEvent(event), eventDurabilityNexus); - minimumGenerationNonAncient = - Math.max(minimumGenerationNonAncient, event.getGeneration() - generationsUntilAncient); - passValueToDurabilityNexus(writer.setMinimumGenerationNonAncient(minimumGenerationNonAncient)); + lowerBound = Math.max(lowerBound, event.getAncientIndicator(ancientMode) - stepsUntilAncient); + passValueToDurabilityNexus( + writer.updateNonAncientEventBoundary( + new NonAncientEventWindow(1, lowerBound, lowerBound, ancientMode)), + eventDurabilityNexus); } assertTrue(eventDurabilityNexus.isEventDurable(events.get(events.size() - 1))); // We shouldn't find any events in the stream. - assertFalse(() -> pcesFiles - .getFileIterator(PcesFileManager.NO_MINIMUM_GENERATION, 0) - .hasNext()); + assertFalse(() -> + pcesFiles.getFileIterator(PcesFileManager.NO_LOWER_BOUND, 0).hasNext()); writer.closeCurrentMutableFile(); } @ParameterizedTest - @ValueSource(booleans = {true, false}) + @MethodSource("buildArguments") @DisplayName("Discontinuity Test") - void discontinuityTest(final boolean truncateLastFile) throws IOException { - final List eventsBeforeDiscontinuity = new LinkedList<>(); - final List eventsAfterDiscontinuity = new LinkedList<>(); - for (int i = 0; i < numEvents; i++) { - final GossipEvent event = generator.generateEvent().getBaseEvent(); - if (i < numEvents / 2) { - eventsBeforeDiscontinuity.add(event); - } else { - eventsAfterDiscontinuity.add(event); + void discontinuityTest(@NonNull final AncientMode ancientMode) throws IOException { + for (final boolean truncateLastFile : List.of(true, false)) { + beforeEach(); + + final Random random = RandomUtils.getRandomPrintSeed(); + + final PlatformContext platformContext = buildContext(ancientMode); + + final StandardGraphGenerator generator = buildGraphGenerator(random); + final int stepsUntilAncient = random.nextInt(50, 100); + final PcesSequencer sequencer = new PcesSequencer(); + final PcesFileTracker pcesFiles = new PcesFileTracker(ancientMode); + + final PcesFileManager fileManager = new PcesFileManager(platformContext, pcesFiles, selfId, 0); + final PcesWriter writer = new PcesWriter(platformContext, fileManager); + final EventDurabilityNexus eventDurabilityNexus = new EventDurabilityNexus(); + + final List eventsBeforeDiscontinuity = new LinkedList<>(); + final List eventsAfterDiscontinuity = new LinkedList<>(); + for (int i = 0; i < numEvents; i++) { + final GossipEvent event = generator.generateEvent().getBaseEvent(); + if (i < numEvents / 2) { + eventsBeforeDiscontinuity.add(event); + } else { + eventsAfterDiscontinuity.add(event); + } } - } - writer.beginStreamingNewEvents(new DoneStreamingPcesTrigger()); + writer.beginStreamingNewEvents(new DoneStreamingPcesTrigger()); - final Collection rejectedEvents = new HashSet<>(); + final Collection rejectedEvents = new HashSet<>(); - long minimumGenerationNonAncient = 0; - final Iterator iterator1 = eventsBeforeDiscontinuity.iterator(); - while (iterator1.hasNext()) { - final GossipEvent event = iterator1.next(); + long lowerBound = 0; + final Iterator iterator1 = eventsBeforeDiscontinuity.iterator(); + while (iterator1.hasNext()) { + final GossipEvent event = iterator1.next(); - sequencer.assignStreamSequenceNumber(event); - passValueToDurabilityNexus(writer.writeEvent(event)); + sequencer.assignStreamSequenceNumber(event); + passValueToDurabilityNexus(writer.writeEvent(event), eventDurabilityNexus); - minimumGenerationNonAncient = - Math.max(minimumGenerationNonAncient, event.getGeneration() - generationsUntilAncient); - passValueToDurabilityNexus(writer.setMinimumGenerationNonAncient(minimumGenerationNonAncient)); + lowerBound = Math.max(lowerBound, event.getAncientIndicator(ancientMode) - stepsUntilAncient); + passValueToDurabilityNexus( + writer.updateNonAncientEventBoundary( + new NonAncientEventWindow(1, lowerBound, lowerBound, ancientMode)), + eventDurabilityNexus); - if (event.getGeneration() < minimumGenerationNonAncient) { - // Although it's not common, it's actually possible that the generator will generate - // an event that is ancient (since it isn't aware of what we consider to be ancient) - rejectedEvents.add(event); - iterator1.remove(); + if (event.getAncientIndicator(ancientMode) < lowerBound) { + // Although it's not common, it's actually possible that the generator will generate + // an event that is ancient (since it isn't aware of what we consider to be ancient) + rejectedEvents.add(event); + iterator1.remove(); + } } - } - eventsBeforeDiscontinuity.forEach(event -> assertTrue(eventDurabilityNexus.isEventDurable(event))); + eventsBeforeDiscontinuity.forEach(event -> assertTrue(eventDurabilityNexus.isEventDurable(event))); - passValueToDurabilityNexus(writer.registerDiscontinuity(100)); + passValueToDurabilityNexus(writer.registerDiscontinuity(100), eventDurabilityNexus); - if (truncateLastFile) { - // Remove a single byte from the last file. This will corrupt the last event that was written. - final Iterator it = pcesFiles.getFileIterator(NO_MINIMUM_GENERATION, 0); - while (it.hasNext()) { - final PcesFile file = it.next(); - if (!it.hasNext()) { - FileManipulation.truncateNBytesFromFile(file.getPath(), 1); + if (truncateLastFile) { + // Remove a single byte from the last file. This will corrupt the last event that was written. + final Iterator it = pcesFiles.getFileIterator(NO_LOWER_BOUND, 0); + while (it.hasNext()) { + final PcesFile file = it.next(); + if (!it.hasNext()) { + FileManipulation.truncateNBytesFromFile(file.getPath(), 1); + } } - } - eventsBeforeDiscontinuity.remove(eventsBeforeDiscontinuity.size() - 1); - } + eventsBeforeDiscontinuity.remove(eventsBeforeDiscontinuity.size() - 1); + } - final Iterator iterator2 = eventsAfterDiscontinuity.iterator(); - while (iterator2.hasNext()) { - final GossipEvent event = iterator2.next(); + final Iterator iterator2 = eventsAfterDiscontinuity.iterator(); + while (iterator2.hasNext()) { + final GossipEvent event = iterator2.next(); - sequencer.assignStreamSequenceNumber(event); - passValueToDurabilityNexus(writer.writeEvent(event)); + sequencer.assignStreamSequenceNumber(event); + passValueToDurabilityNexus(writer.writeEvent(event), eventDurabilityNexus); - minimumGenerationNonAncient = - Math.max(minimumGenerationNonAncient, event.getGeneration() - generationsUntilAncient); - passValueToDurabilityNexus(writer.setMinimumGenerationNonAncient(minimumGenerationNonAncient)); + lowerBound = Math.max(lowerBound, event.getAncientIndicator(ancientMode) - stepsUntilAncient); + passValueToDurabilityNexus( + writer.updateNonAncientEventBoundary( + new NonAncientEventWindow(1, lowerBound, lowerBound, ancientMode)), + eventDurabilityNexus); - if (event.getGeneration() < minimumGenerationNonAncient) { - // Although it's not common, it's actually possible that the generator will generate - // an event that is ancient (since it isn't aware of what we consider to be ancient) - rejectedEvents.add(event); - iterator2.remove(); + if (event.getAncientIndicator(ancientMode) < lowerBound) { + // Although it's not common, it's actually possible that the generator will generate + // an event that is ancient (since it isn't aware of what we consider to be ancient) + rejectedEvents.add(event); + iterator2.remove(); + } } - } - assertTrue( - eventDurabilityNexus.isEventDurable(eventsAfterDiscontinuity.get(eventsAfterDiscontinuity.size() - 1))); - eventsAfterDiscontinuity.forEach(event -> assertTrue(eventDurabilityNexus.isEventDurable(event))); - rejectedEvents.forEach(event -> assertFalse(eventDurabilityNexus.isEventDurable(event))); + assertTrue(eventDurabilityNexus.isEventDurable( + eventsAfterDiscontinuity.get(eventsAfterDiscontinuity.size() - 1))); + eventsAfterDiscontinuity.forEach(event -> assertTrue(eventDurabilityNexus.isEventDurable(event))); + rejectedEvents.forEach(event -> assertFalse(eventDurabilityNexus.isEventDurable(event))); - verifyStream(eventsBeforeDiscontinuity, platformContext, truncateLastFile ? 1 : 0); + verifyStream(eventsBeforeDiscontinuity, platformContext, truncateLastFile ? 1 : 0, ancientMode); - writer.closeCurrentMutableFile(); + writer.closeCurrentMutableFile(); + } } /** - * In this test, increase the first non-ancient generation as events are added. When this happens, we - * should never have to include more than the preferred number of events in each file. + * In this test, increase the lower bound as events are added. When this happens, we should never + * have to include more than the preferred number of events in each file. */ - @Test - @DisplayName("Advance Non Ancient Generation Test") - void advanceNonAncientGenerationTest() throws IOException { + @ParameterizedTest + @MethodSource("buildArguments") + @DisplayName("Advance Non Ancient Boundary Test") + void advanceNonAncientBoundaryTest(@NonNull final AncientMode ancientMode) throws IOException { + final Random random = RandomUtils.getRandomPrintSeed(); + + final PlatformContext platformContext = buildContext(ancientMode); + final FakeTime time = (FakeTime) platformContext.getTime(); + + final StandardGraphGenerator generator = buildGraphGenerator(random); + final int stepsUntilAncient = random.nextInt(50, 100); + final PcesSequencer sequencer = new PcesSequencer(); + final PcesFileTracker pcesFiles = new PcesFileTracker(ancientMode); + + final PcesFileManager fileManager = new PcesFileManager(platformContext, pcesFiles, selfId, 0); + final PcesWriter writer = new PcesWriter(platformContext, fileManager); + final EventDurabilityNexus eventDurabilityNexus = new EventDurabilityNexus(); + final List events = new LinkedList<>(); for (int i = 0; i < numEvents; i++) { events.add(generator.generateEvent().getBaseEvent()); @@ -547,23 +648,25 @@ void advanceNonAncientGenerationTest() throws IOException { final Set rejectedEvents = new HashSet<>(); - long minimumGenerationNonAncient = 0; + long lowerBound = 0; for (final GossipEvent event : events) { sequencer.assignStreamSequenceNumber(event); - passValueToDurabilityNexus(writer.writeEvent(event)); + passValueToDurabilityNexus(writer.writeEvent(event), eventDurabilityNexus); assertFalse(eventDurabilityNexus.isEventDurable(event)); time.tick(Duration.ofSeconds(1)); - if (event.getGeneration() < minimumGenerationNonAncient) { + if (event.getAncientIndicator(ancientMode) < lowerBound) { // This event is ancient and will have been rejected. rejectedEvents.add(event); } - minimumGenerationNonAncient = - Math.max(minimumGenerationNonAncient, event.getGeneration() - generationsUntilAncient); - passValueToDurabilityNexus(writer.setMinimumGenerationNonAncient(minimumGenerationNonAncient)); + lowerBound = Math.max(lowerBound, event.getAncientIndicator(ancientMode) - stepsUntilAncient); + passValueToDurabilityNexus( + writer.updateNonAncientEventBoundary( + new NonAncientEventWindow(1, lowerBound, lowerBound, ancientMode)), + eventDurabilityNexus); } // Remove the rejected events from the list @@ -571,39 +674,40 @@ void advanceNonAncientGenerationTest() throws IOException { events.forEach(event -> assertTrue(eventDurabilityNexus.isEventDurable(event))); rejectedEvents.forEach(event -> assertFalse(eventDurabilityNexus.isEventDurable(event))); - verifyStream(events, platformContext, 0); + verifyStream(events, platformContext, 0, ancientMode); // Advance the time so that all files are GC eligible according to the clock. time.tick(Duration.ofDays(1)); // Prune old files. - final long minimumGenerationToStore = events.get(events.size() - 1).getGeneration() / 2; - writer.setMinimumGenerationToStore(minimumGenerationToStore); + final long lowerBoundToStore = events.get(events.size() - 1).getAncientIndicator(ancientMode) / 2; + writer.setMinimumAncientIdentifierToStore(lowerBoundToStore); // We shouldn't see any files that are incapable of storing events above the minimum - final PcesFileTracker pcesFiles = PcesFileReader.readFilesFromDisk( + final PcesFileTracker pcesFiles2 = PcesFileReader.readFilesFromDisk( platformContext, TestRecycleBin.getInstance(), PcesUtilities.getDatabaseDirectory(platformContext, selfId), 0, - false); + false, + ancientMode); - pcesFiles - .getFileIterator(NO_MINIMUM_GENERATION, 0) - .forEachRemaining(file -> assertTrue(file.getMaximumGeneration() >= minimumGenerationToStore)); + pcesFiles2 + .getFileIterator(NO_LOWER_BOUND, 0) + .forEachRemaining(file -> assertTrue(file.getUpperBound() >= lowerBoundToStore)); writer.closeCurrentMutableFile(); - // Since we were very careful to always advance the first non-ancient generation, we should - // find lots of files with a minimum generation exceeding 0. - boolean foundNonZeroMinimumGeneration = false; - for (final Iterator fileIterator = pcesFiles.getFileIterator(0, 0); fileIterator.hasNext(); ) { + // Since we were very careful to always advance the first non-ancient threshold, we should + // find lots of files with a lower bound exceeding 0. + boolean foundNonZeroBoundary = false; + for (final Iterator fileIterator = pcesFiles2.getFileIterator(0, 0); fileIterator.hasNext(); ) { final PcesFile file = fileIterator.next(); - if (file.getMinimumGeneration() > 0) { - foundNonZeroMinimumGeneration = true; + if (file.getLowerBound() > 0) { + foundNonZeroBoundary = true; break; } } - assertTrue(foundNonZeroMinimumGeneration); + assertTrue(foundNonZeroBoundary); } }