From 40c0478b216680b8d9b750c02d3d656b608629c9 Mon Sep 17 00:00:00 2001 From: Nikita Smirnov <46124551+Nikita-Smirnov-Exactpro@users.noreply.github.com> Date: Wed, 6 Mar 2024 18:32:36 +0400 Subject: [PATCH] [Th2-5165] Use cache in BookInfo (#253) * bookId, page name, page start are unique page identifier * Added book info metrics * [TH2-5165] Removed ThreadSafeProvider class * [TH2-5165] Corrected java doc in BookInfo * [TH2-5165] Corrected after review * [TH2-5165] Corrected after review * [TH2-5165] Fixed problem with Instant MIN/MAX conversion problem * [TH2-5165] Corrected book cache behaviour when book has gap * [TH2-5165] Added testAddPageInTheMiddleOfExist * [TH2-5165] corrected after review * Update cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/BaseCradleCassandraTest.java Co-authored-by: Oleg Smirnov * [TH2-5165] corrected after review * [TH2-5165] removed java 17 API * [TH2-5165] corrected book info metrics * [TH2-5165] added testAddTheSecondPageBeforeThreshold test * [TH2-5165] added randomAccessDaysInvalidateInterval option * [TH2-5165] corrected after review * [TH2-5165] revert BOOK_LABEL to book * [TH2-5165] revert logic to skip register load for empty day page interval * [TH2-5165] revert logic to skip register load for empty day page interval * [TH2-5165] merged with dev-version-5 * [TH2-5165] added BookWithoutPageTest * [TH2-5165] corrected after review * [TH2-5165] Updated github workflow --------- Co-authored-by: Oleg Smirnov --- ...ish-sonatype.yml => build-dev-release.yml} | 9 +- .github/workflows/build-release.yml | 16 + ...ublish-sonatype.yml => build-sanpshot.yml} | 14 +- .github/workflows/integration-tests.yml | 26 + .github/workflows/java-publish-sonatype.yml | 41 - README.md | 19 + build.gradle | 4 + cradle-cassandra/build.gradle | 1 - .../cassandra/CassandraCradleStorage.java | 70 +- .../cassandra/CassandraStorageSettings.java | 25 +- .../cassandra/ReadThroughBookCache.java | 91 ++- .../cassandra/dao/books/PageEntity.java | 9 +- .../cassandra/dao/books/PageNameEntity.java | 4 +- .../cassandra/dao/books/PageOperator.java | 31 +- .../intervals/CassandraIntervalsWorker.java | 42 +- .../AbstractMessageIteratorProvider.java | 168 ++-- .../CassandraGroupedMessageFilter.java | 49 +- .../CassandraStoredMessageFilter.java | 82 +- .../GroupedMessageIteratorProvider.java | 156 ++-- .../MessageBatchesIteratorProvider.java | 10 +- .../messages/MessagesIteratorProvider.java | 10 +- .../EntityStatisticsIteratorProvider.java | 57 +- .../MessageStatisticsIteratorProvider.java | 53 +- .../testevents/CassandraTestEventFilter.java | 49 +- .../testevents/TestEventIteratorProvider.java | 159 ++-- .../PagesInIntervalIteratorProvider.java | 6 +- .../SessionsStatisticsIteratorProvider.java | 123 --- .../cradle/cassandra/utils/FilterUtils.java | 91 ++- .../cradle/cassandra/utils/StorageUtils.java | 64 ++ .../cassandra/workers/MessagesWorker.java | 20 +- .../cassandra/ReadThroughBookCacheTest.java | 26 +- .../GroupedMessageBatchEntityTest.java | 7 +- .../dao/messages/MessageBatchEntityTest.java | 4 +- .../MessagesIteratorProviderTest.java | 10 +- .../dao/testevents/TestEventEntityTest.java | 4 +- .../integration/BaseCradleCassandraTest.java | 37 +- .../integration/BookWithoutPageTest.java | 229 ++++++ .../integration/CassandraCradleHelper.java | 25 +- .../GroupedMessageIteratorProviderTest.java | 50 +- .../MessageBatchIteratorProviderTest.java | 14 +- .../messages/MessageIteratorProviderTest.java | 9 +- .../integration/pages/PagesApiRemoveTest.java | 195 +++++ .../integration/pages/PagesApiTest.java | 169 +++- .../TestEventIteratorProviderTest.java | 28 +- .../PagesInIntervalIteratorProviderTest.java | 15 +- cradle-core/build.gradle | 2 + .../java/com/exactpro/cradle/BookCache.java | 6 +- .../java/com/exactpro/cradle/BookInfo.java | 720 ++++++++++++++---- .../com/exactpro/cradle/BookInfoMetrics.java | 190 +++++ .../java/com/exactpro/cradle/BookManager.java | 21 +- .../exactpro/cradle/CoreStorageSettings.java | 3 +- .../com/exactpro/cradle/CradleStorage.java | 180 ++--- .../exactpro/cradle/DummyCradleStorage.java | 12 +- .../main/java/com/exactpro/cradle/PageId.java | 40 +- .../java/com/exactpro/cradle/PageInfo.java | 42 +- .../java/com/exactpro/cradle/PagesLoader.java | 30 + .../cradle/filters/AbstractFilter.java | 4 +- .../cradle/resultset/EmptyResultSet.java | 14 +- .../exactpro/cradle/utils/TestEventUtils.java | 4 +- .../com/exactpro/cradle/BookInfoTest.java | 645 ++++++++++++++++ .../CradleStorageBookPageNamingTest.java | 16 +- .../exactpro/cradle/CradleStorageTest.java | 19 +- .../cradle/InMemoryCradleStorage.java | 431 +++++++++++ .../com/exactpro/cradle/TestPageLoader.java | 37 + .../com/exactpro/cradle/TestPagesLoader.java | 42 + .../java/com/exactpro/cradle/TestUtils.java | 12 +- .../cradle/testevents/EventBatchTest.java | 38 +- .../cradle/testevents/EventSingleTest.java | 35 +- gradle.properties | 2 +- gradle/wrapper/gradle-wrapper.properties | 4 +- 70 files changed, 3708 insertions(+), 1162 deletions(-) rename .github/workflows/{dev-release-java-publish-sonatype.yml => build-dev-release.yml} (74%) create mode 100644 .github/workflows/build-release.yml rename .github/workflows/{dev-java-publish-sonatype.yml => build-sanpshot.yml} (71%) create mode 100644 .github/workflows/integration-tests.yml delete mode 100644 .github/workflows/java-publish-sonatype.yml delete mode 100644 cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/resultset/SessionsStatisticsIteratorProvider.java create mode 100644 cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/BookWithoutPageTest.java create mode 100644 cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/pages/PagesApiRemoveTest.java create mode 100644 cradle-core/src/main/java/com/exactpro/cradle/BookInfoMetrics.java create mode 100644 cradle-core/src/main/java/com/exactpro/cradle/PagesLoader.java create mode 100644 cradle-core/src/test/java/com/exactpro/cradle/BookInfoTest.java create mode 100644 cradle-core/src/test/java/com/exactpro/cradle/InMemoryCradleStorage.java create mode 100644 cradle-core/src/test/java/com/exactpro/cradle/TestPageLoader.java create mode 100644 cradle-core/src/test/java/com/exactpro/cradle/TestPagesLoader.java diff --git a/.github/workflows/dev-release-java-publish-sonatype.yml b/.github/workflows/build-dev-release.yml similarity index 74% rename from .github/workflows/dev-release-java-publish-sonatype.yml rename to .github/workflows/build-dev-release.yml index 0c23d75c8..992b2a0be 100644 --- a/.github/workflows/dev-release-java-publish-sonatype.yml +++ b/.github/workflows/build-dev-release.yml @@ -1,17 +1,14 @@ -name: Build and publish dev-release Java distributions to sonatype. +name: Build and publish dev release jar to sonatype repository -on: - push: - tags: - - \d+.\d+.\d+-dev +on: workflow_dispatch jobs: build: uses: th2-net/.github/.github/workflows/compound-java.yml@main with: build-target: 'Sonatype' - runsOn: ubuntu-latest devRelease: true + createTag: true secrets: sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml new file mode 100644 index 000000000..669bca926 --- /dev/null +++ b/.github/workflows/build-release.yml @@ -0,0 +1,16 @@ +name: Build and publish release jar to sonatype repository + +on: workflow_dispatch + +jobs: + build: + uses: th2-net/.github/.github/workflows/compound-java.yml@main + with: + build-target: 'Sonatype' + devRelease: false + createTag: true + secrets: + sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} + sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} + sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }} + sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} \ No newline at end of file diff --git a/.github/workflows/dev-java-publish-sonatype.yml b/.github/workflows/build-sanpshot.yml similarity index 71% rename from .github/workflows/dev-java-publish-sonatype.yml rename to .github/workflows/build-sanpshot.yml index a0f035f9b..e8180f36b 100644 --- a/.github/workflows/dev-java-publish-sonatype.yml +++ b/.github/workflows/build-sanpshot.yml @@ -1,19 +1,21 @@ -name: Dev build and publish Java distributions to sonatype snapshot repository +name: Build and publish jar to sonatype snapshot repository + on: push: branches-ignore: - - master - - version-* + - master + - version-* + - dependabot* + paths-ignore: + - README.md jobs: build-job: uses: th2-net/.github/.github/workflows/compound-java-dev.yml@main with: - multiproject: true build-target: 'Sonatype' - runsOn: ubuntu-latest secrets: sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }} - sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} + sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} \ No newline at end of file diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..44bbbe4a2 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,26 @@ +name: "Run integration tests for cradle API" + +on: + push: + branches: + - '*' + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 'zulu' '11' + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: '11' + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + - name: Build with Gradle + run: ./gradlew --info clean integrationTest + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: integration-test-results + path: build/reports/tests/integrationTest/ diff --git a/.github/workflows/java-publish-sonatype.yml b/.github/workflows/java-publish-sonatype.yml deleted file mode 100644 index 5d1e85f99..000000000 --- a/.github/workflows/java-publish-sonatype.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Build and release Java distributions to sonatype. -on: - push: - branches: - - master - - version-* - paths: - - gradle.properties - -jobs: - build: - uses: th2-net/.github/.github/workflows/compound-java.yml@main - with: - scanner-enabled: false - build-target: 'Sonatype' - runsOn: ubuntu-latest - secrets: - sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }} - sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }} - sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }} - sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }} - scan-job: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Create lockfiles - run: ./gradlew createLockFiles - - name: Run Trivy vulnerability scanner in repo mode - uses: aquasecurity/trivy-action@master - with: - scan-type: 'fs' - ignore-unfixed: false - format: 'sarif' - output: 'trivy-results.sarif' - severity: 'CRITICAL,HIGH,MEDIUM' - exit-code: '0' - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: 'trivy-results.sarif' \ No newline at end of file diff --git a/README.md b/README.md index 267e440d9..fbd7824d7 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,25 @@ Events in a batch can have a reference only to the parent of the batch or other Test events have mandatory parameters that are verified when storing an event. These are: id, name (for non-batch events), start timestamp. +## Metrics + +* `cradle_page_cache_size` (type: Gauge, labels: book, cache) - Size of page cache. +* `cradle_page_cache_page_request_total` (type: Counter, labels: book, cache, method) - Page requests number from cache +* `cradle_page_cache_invalidate_total` (type: Counter, labels: book, cache, cause) - Cache invalidates +* `cradle_page_cache_page_loads_total` (type: Summary, labels: book, cache) - Page loads number to cache + * `_count` - loaded page day intervals + * `_sum` - loaded pages + +### Labels: + * cache: HOT, RANDOM + * method: GET, NEXT, PREVIOUS, FIND, ITERATE, REFRESH + * cause: EXPLICIT, REPLACED, COLLECTED, EXPIRED, SIZE + +## Release notes + +### 5.2.0 +* Added page cache for each book to avoid memory leak + ## Changes ### 5.1.5 diff --git a/build.gradle b/build.gradle index 0e7371513..87d816e93 100644 --- a/build.gradle +++ b/build.gradle @@ -55,6 +55,10 @@ subprojects { targetCompatibility = JavaVersion.VERSION_11 //Java version to generate classes for. compileJava.options.debugOptions.debugLevel = 'source,lines,vars' // Include debug information + dependencies { + implementation 'io.prometheus:simpleclient_dropwizard:0.16.0' + } + jar { manifest { attributes( diff --git a/cradle-cassandra/build.gradle b/cradle-cassandra/build.gradle index f7deaaafd..3b14a4788 100644 --- a/cradle-cassandra/build.gradle +++ b/cradle-cassandra/build.gradle @@ -13,7 +13,6 @@ dependencies { implementation "com.datastax.oss:java-driver-query-builder" implementation "com.datastax.oss:java-driver-mapper-processor" implementation "com.datastax.oss:java-driver-mapper-runtime" - implementation 'io.prometheus:simpleclient_dropwizard:0.16.0' implementation 'com.google.guava:guava' // this section is required to bypass failing vulnerability check caused by cassandra driver's transitive dependencies diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraCradleStorage.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraCradleStorage.java index 0cd7160b7..02090bd47 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraCradleStorage.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraCradleStorage.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Exactpro (Exactpro Systems Limited) + * Copyright 2023-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -73,7 +73,6 @@ import com.exactpro.cradle.cassandra.keyspaces.CradleInfoKeyspaceCreator; import com.exactpro.cradle.cassandra.metrics.DriverMetrics; import com.exactpro.cradle.cassandra.resultset.CassandraCradleResultSet; -import com.exactpro.cradle.cassandra.resultset.SessionsStatisticsIteratorProvider; import com.exactpro.cradle.cassandra.retries.FixedNumberRetryPolicy; import com.exactpro.cradle.cassandra.retries.PageSizeAdjustingPolicy; import com.exactpro.cradle.cassandra.retries.SelectExecutionPolicy; @@ -128,6 +127,9 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static com.exactpro.cradle.cassandra.utils.StorageUtils.toLocalDate; +import static com.exactpro.cradle.cassandra.utils.StorageUtils.toLocalTime; + public class CassandraCradleStorage extends CradleStorage { private final Logger LOGGER = LoggerFactory.getLogger(CassandraCradleStorage.class); @@ -211,7 +213,11 @@ protected void doInit(boolean prepareStorage) throws CradleStorageException .setTimeout(timeout) .setPageSize(resultPageSize); operators = createOperators(connection.getSession(), settings); - bookCache = new ReadThroughBookCache(operators, readAttrs, settings.getSchemaVersion()); + bookCache = new ReadThroughBookCache(operators, + readAttrs, + settings.getSchemaVersion(), + settings.getRandomAccessDaysCacheSize(), + settings.getRandomAccessDaysInvalidateInterval()); bookManager = new BookManager(getBookCache(), settings.getBookRefreshIntervalMillis()); bookManager.start(); @@ -294,12 +300,12 @@ protected void doAddPages(BookId bookId, List pages, PageInfo lastPage String bookName = bookId.getName(); for (PageInfo page : pages) { - String pageName = page.getId().getName(); + String pageName = page.getName(); try { PageNameEntity nameEntity = new PageNameEntity(bookName, pageName, page.getStarted(), page.getComment(), page.getEnded()); if (!pageNameOp.writeNew(nameEntity, writeAttrs).wasApplied()) - throw new IOException("Query to insert page '"+nameEntity.getName()+"' was not applied. Probably, page already exists"); + throw new IOException("Query to insert page '"+nameEntity.getName()+"' book '" + bookId.getName() + "' was not applied. Probably, page already exists"); PageEntity entity = new PageEntity(bookName, pageName, page.getStarted(), page.getComment(), page.getEnded(), page.getUpdated()); pageOp.write(entity, writeAttrs); } @@ -943,29 +949,6 @@ protected Counter doGetCount(BookId bookId, EntityType entityType, Interval inte } } - private CompletableFuture> doGetSessionsAsync(BookId bookId, Interval interval, SessionRecordType recordType) throws CradleStorageException { - String queryInfo = String.format("%s Aliases in book %s from %s to %s", - recordType.name(), - bookId.getName(), - interval.getStart().toString(), - interval.getEnd().toString()); - - List frameIntervals = StorageUtils.sliceInterval(interval); - - SessionsStatisticsIteratorProvider iteratorProvider = new SessionsStatisticsIteratorProvider( - queryInfo, - operators, - getBookCache().getBook(bookId), - composingService, - selectExecutor, - readAttrs, - frameIntervals, - recordType); - - return iteratorProvider.nextIterator() - .thenApplyAsync(it -> new CassandraCradleResultSet<>(it, iteratorProvider)); - } - @Override protected CompletableFuture> doGetSessionAliasesAsync(BookId bookId, Interval interval) throws CradleStorageException { String queryInfo = String.format("Session Aliases in book %s from pages that fall within %s to %s", @@ -1311,8 +1294,8 @@ protected void removeTestEventData(PageId pageId) { protected void removePageData(PageInfo pageInfo) throws CradleStorageException { - String book = pageInfo.getId().getBookId().getName(); - String page = pageInfo.getId().getName(); + String book = pageInfo.getBookName(); + String page = pageInfo.getName(); // remove sessions operators.getPageSessionsOperator().remove(book, page, writeAttrs); @@ -1371,23 +1354,25 @@ private void removeEntityStatistics(PageId pageId) { @Override protected CompletableFuture> doGetPagesAsync (BookId bookId, Interval interval) { + Instant start = interval.getStart(); + Instant end = interval.getEnd(); String queryInfo = String.format( "Getting pages for book %s between %s - %s ", bookId.getName(), - interval.getStart(), - interval.getEnd()); + start, + end); PageEntity startPage = operators.getPageOperator() .getPageForLessOrEqual( bookId.getName(), - LocalDate.ofInstant(interval.getStart(), TIMEZONE_OFFSET), - LocalTime.ofInstant(interval.getStart(), TIMEZONE_OFFSET), + toLocalDate(start), + toLocalTime(start), readAttrs).one(); LocalDate startDate = startPage == null ? LocalDate.MIN : startPage.getStartDate(); LocalTime startTime = startPage == null ? LocalTime.MIN : startPage.getStartTime(); - LocalDate endDate = LocalDate.ofInstant(interval.getEnd(), TIMEZONE_OFFSET); - LocalTime endTime = LocalTime.ofInstant(interval.getEnd(), TIMEZONE_OFFSET); + LocalDate endDate = toLocalDate(end); + LocalTime endTime = toLocalTime(end); return operators.getPageOperator().getPagesForInterval( bookId.getName(), @@ -1408,16 +1393,15 @@ protected CompletableFuture> doGetPagesAsync (BookId bookId, @Override protected Iterator doGetPages(BookId bookId, Interval interval) throws CradleStorageException { - String queryInfo = String.format( - "Getting pages for book %s between %s - %s ", - bookId.getName(), - interval.getStart(), - interval.getEnd()); - try { return doGetPagesAsync(bookId, interval).get(); } catch (InterruptedException | ExecutionException e) { - throw new CradleStorageException("Error while " + queryInfo, e); + String error = String.format( + "Error while Getting pages for book %s between %s - %s ", + bookId.getName(), + interval.getStart(), + interval.getEnd()); + throw new CradleStorageException(error, e); } } } diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraStorageSettings.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraStorageSettings.java index 5b33ec986..c7df0f4c0 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraStorageSettings.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraStorageSettings.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,6 +33,9 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class CassandraStorageSettings extends CoreStorageSettings { public static final String SCHEMA_VERSION = "5.3.0"; + public static final int RANDOM_ACCESS_DAYS_CACHE_SIZE = 10; + /** One day in milliseconds */ + public static final long RANDOM_ACCESS_DAYS_CACHE_INVALIDATE_INTERVAL = 24 * 60 * 60 * 1_000; public static final CassandraConsistencyLevel DEFAULT_CONSISTENCY_LEVEL = CassandraConsistencyLevel.LOCAL_QUORUM; public static final int DEFAULT_KEYSPACE_REPL_FACTOR = 1; @@ -53,9 +56,13 @@ public class CassandraStorageSettings extends CoreStorageSettings { public static final long DEFAULT_TIMEOUT = 5000; public static final CompressionType DEFAULT_COMPRESSION_TYPE = CompressionType.ZLIB; + //we need to use Instant.EPOCH instead of Instant.MIN. + //when cassandra driver tries to convert Instant.MIN to milliseconds using toEpochMilli() it causes long overflow. + public static final Instant MIN_EPOCH_INSTANT = Instant.EPOCH; //we need to use Instant.ofEpochMilli(Long.MAX_VALUE) instead of Instant.MAX. //when cassandra driver tries to convert Instant.MAX to milliseconds using toEpochMilli() it causes long overflow. - public static final Instant DEFAULT_PAGE_REMOVE_TIME = Instant.ofEpochMilli(Long.MAX_VALUE); + public static final Instant MAX_EPOCH_INSTANT = Instant.ofEpochMilli(Long.MAX_VALUE); + public static final Instant DEFAULT_PAGE_REMOVE_TIME = MAX_EPOCH_INSTANT; @JsonIgnore private NetworkTopologyStrategy networkTopologyStrategy; @@ -66,6 +73,8 @@ public class CassandraStorageSettings extends CoreStorageSettings { private CassandraConsistencyLevel readConsistencyLevel = DEFAULT_CONSISTENCY_LEVEL; private String keyspace; private String schemaVersion = SCHEMA_VERSION; + private int randomAccessDaysCacheSize = RANDOM_ACCESS_DAYS_CACHE_SIZE; + private long randomAccessDaysInvalidateInterval = RANDOM_ACCESS_DAYS_CACHE_INVALIDATE_INTERVAL; private int keyspaceReplicationFactor = DEFAULT_KEYSPACE_REPL_FACTOR; private int maxParallelQueries = DEFAULT_MAX_PARALLEL_QUERIES; // FIXME: remove @@ -122,6 +131,8 @@ public CassandraStorageSettings(CassandraStorageSettings settings) { this.keyspace = settings.getKeyspace(); this.schemaVersion = settings.getSchemaVersion(); + this.randomAccessDaysCacheSize = settings.getRandomAccessDaysCacheSize(); + this.randomAccessDaysInvalidateInterval = settings.getRandomAccessDaysInvalidateInterval(); this.keyspaceReplicationFactor = settings.getKeyspaceReplicationFactor(); this.maxParallelQueries = settings.getMaxParallelQueries(); @@ -196,6 +207,14 @@ public String getSchemaVersion() { return schemaVersion; } + public int getRandomAccessDaysCacheSize() { + return randomAccessDaysCacheSize; + } + + public long getRandomAccessDaysInvalidateInterval() { + return randomAccessDaysInvalidateInterval; + } + public int getKeyspaceReplicationFactor() { return keyspaceReplicationFactor; } @@ -386,6 +405,8 @@ public String toString() { ", readConsistencyLevel=" + readConsistencyLevel + ", keyspace='" + keyspace + '\'' + ", schemaVersion='" + schemaVersion + '\'' + + ", randomAccessDaysCacheSize='" + randomAccessDaysCacheSize + '\'' + + ", randomAccessDaysInvalidateInterval='" + randomAccessDaysInvalidateInterval + '\'' + ", keyspaceReplicationFactor=" + keyspaceReplicationFactor + ", maxParallelQueries=" + maxParallelQueries + ", resultPageSize=" + resultPageSize + diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/ReadThroughBookCache.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/ReadThroughBookCache.java index 27408046a..d61715a6f 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/ReadThroughBookCache.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/ReadThroughBookCache.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,6 +26,9 @@ import com.exactpro.cradle.errors.BookNotFoundException; import com.exactpro.cradle.utils.CradleStorageException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -34,6 +37,8 @@ import java.util.function.Function; import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_PAGE_REMOVE_TIME; +import static com.exactpro.cradle.cassandra.utils.StorageUtils.toLocalDate; +import static com.exactpro.cradle.cassandra.utils.StorageUtils.toLocalTime; public class ReadThroughBookCache implements BookCache { @@ -41,17 +46,21 @@ public class ReadThroughBookCache implements BookCache { private final Map books; private final Function readAttrs; private final String schemaVersion; + private final int raCacheSize; + private final long raCacheInvalidateInterval; public ReadThroughBookCache(CassandraOperators operators, Function readAttrs, - String schemaVersion) { + String schemaVersion, int raCacheSize, long raCacheInvalidateInterval) { this.operators = operators; this.books = new ConcurrentHashMap<>(); this.readAttrs = readAttrs; this.schemaVersion = schemaVersion; + this.raCacheSize = raCacheSize; + this.raCacheInvalidateInterval = raCacheInvalidateInterval; } - public BookInfo getBook (BookId bookId) throws BookNotFoundException { + public BookInfo getBook(BookId bookId) throws BookNotFoundException { try { return books.computeIfAbsent(bookId, bookId1 -> { try { @@ -80,6 +89,51 @@ public Collection loadPageInfo(BookId bookId, boolean loadRemoved) { return result; } + public Collection loadPageInfo(BookId bookId, Instant start, Instant end, boolean loadRemoved) { + Collection result = new ArrayList<>(); + LocalDate startDate = start != null ? toLocalDate(start) : LocalDate.MIN; + LocalTime startTime = start != null ? toLocalTime(start) : LocalTime.MIN; + LocalDate endDate = end != null ? toLocalDate(end) : LocalDate.MAX; + LocalTime endTime = end != null ? toLocalTime(end) : LocalTime.MAX; + for (PageEntity pageEntity : operators.getPageOperator().get( + bookId.getName(), + startDate, + startTime, + endDate, + endTime, + readAttrs + )) { + if (loadRemoved || pageEntity.getRemoved() == null || pageEntity.getRemoved().equals(DEFAULT_PAGE_REMOVE_TIME)) { + result.add(pageEntity.toPageInfo()); + } + } + return result; + } + + public PageInfo loadLastPageInfo(BookId bookId, boolean loadRemoved) { + for (PageEntity pageEntity : operators.getPageOperator().getLast( + bookId.getName(), + readAttrs + )) { + if (loadRemoved || pageEntity.getRemoved() == null || pageEntity.getRemoved().equals(DEFAULT_PAGE_REMOVE_TIME)) { + return pageEntity.toPageInfo(); + } + } + return null; + } + + public PageInfo loadFirstPageInfo(BookId bookId, boolean loadRemoved) { + for (PageEntity pageEntity : operators.getPageOperator().getFirst( + bookId.getName(), + readAttrs + )) { + if (loadRemoved || pageEntity.getRemoved() == null || pageEntity.getRemoved().equals(DEFAULT_PAGE_REMOVE_TIME)) { + return pageEntity.toPageInfo(); + } + } + return null; + } + public BookInfo loadBook(BookId bookId) throws CradleStorageException { BookEntity bookEntity = operators.getBookOperator().get(bookId.getName(), readAttrs); @@ -97,16 +151,33 @@ public BookInfo loadBook(BookId bookId) throws CradleStorageException { return toBookInfo(bookEntity); } - private BookInfo toBookInfo(BookEntity entity) throws CradleStorageException { - BookId bookId = new BookId(entity.getName()); - Collection pages = loadPageInfo(bookId, false); + private Collection loadPageInfo(BookId bookId, Instant start, Instant end) { + return loadPageInfo(bookId, start, end, false); + } - return new BookInfo(new BookId(entity.getName()), entity.getFullName(), entity.getDesc(), entity.getCreated(), pages); + private PageInfo loadLastPageInfo(BookId bookId) { + return loadLastPageInfo(bookId, false); } - @Override - public void updateCachedBook(BookInfo bookInfo) { - books.put(bookInfo.getId(), bookInfo); + private PageInfo loadFirstPageInfo(BookId bookId) { + return loadFirstPageInfo(bookId, false); + } + + private BookInfo toBookInfo(BookEntity entity) throws CradleStorageException { + try { + return new BookInfo( + new BookId(entity.getName()), + entity.getFullName(), + entity.getDesc(), + entity.getCreated(), + raCacheSize, + raCacheInvalidateInterval, + this::loadPageInfo, + this::loadFirstPageInfo, + this::loadLastPageInfo); + } catch (RuntimeException e) { + throw new CradleStorageException("Inconsistent state of book '"+entity.getName(), e); + } } @Override diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/books/PageEntity.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/books/PageEntity.java index bc6a0c27e..480657c22 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/books/PageEntity.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/books/PageEntity.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,6 +30,7 @@ import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_PAGE_REMOVE_TIME; +@SuppressWarnings("DefaultAnnotationParam") @Entity @CqlName(PageEntity.TABLE_NAME) @PropertyStrategy(mutable = false) @@ -112,7 +113,7 @@ public PageEntity(String book, String name, Instant started, String comment, Ins } public PageEntity(PageInfo pageInfo) { - this(pageInfo.getId().getBookId().getName(), pageInfo.getId().getName(), pageInfo.getStarted(), pageInfo.getComment(), pageInfo.getEnded(), pageInfo.getUpdated()); + this(pageInfo.getBookName(), pageInfo.getName(), pageInfo.getStarted(), pageInfo.getComment(), pageInfo.getEnded(), pageInfo.getUpdated()); } @@ -145,8 +146,8 @@ public Instant getRemoved() { } public PageInfo toPageInfo() { - return new PageInfo(new PageId(new BookId(book), name), - TimeUtils.toInstant(getStartDate(), getStartTime()), + Instant start = TimeUtils.toInstant(getStartDate(), getStartTime()); + return new PageInfo(new PageId(new BookId(book), start, name), TimeUtils.toInstant(getEndDate(), getEndTime()), getComment(), getUpdated(), diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/books/PageNameEntity.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/books/PageNameEntity.java index 599e3b095..0e0ffe245 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/books/PageNameEntity.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/books/PageNameEntity.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -94,7 +94,7 @@ public PageNameEntity(String book, String name, Instant started, String comment, } public PageNameEntity(PageInfo pageInfo) { - this(pageInfo.getId().getBookId().getName(), pageInfo.getId().getName(), pageInfo.getStarted(), pageInfo.getComment(), pageInfo.getEnded()); + this(pageInfo.getBookName(), pageInfo.getName(), pageInfo.getStarted(), pageInfo.getComment(), pageInfo.getEnded()); } diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/books/PageOperator.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/books/PageOperator.java index 750e3833f..fd6fbde9e 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/books/PageOperator.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/books/PageOperator.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,6 +43,35 @@ public interface PageOperator { PagingIterable get(String book, LocalDate startDate, LocalTime startTime, Function attributes); + @Query("SELECT * FROM ${qualifiedTableId} " + + "WHERE " + + FIELD_BOOK +"=:book AND " + + "(" + FIELD_START_DATE + ", " + FIELD_START_TIME + ") >= (:startDate, :startTime) " + + "AND " + + "(" + FIELD_START_DATE + ", " + FIELD_START_TIME + ") <= (:endDate, :endTime)") + PagingIterable get(String book, LocalDate startDate, LocalTime startTime, + LocalDate endDate, LocalTime endTime, + Function attributes); + + @Query("SELECT * FROM ${qualifiedTableId} " + + "WHERE " + + FIELD_BOOK +"=:book " + + "ORDER BY " + + FIELD_START_DATE + " DESC, " + + FIELD_START_TIME + " DESC " + + "LIMIT 1") + PagingIterable getLast(String book, Function attributes); + + @Query("SELECT * FROM ${qualifiedTableId} " + + "WHERE " + + FIELD_BOOK +"=:book " + + "ORDER BY " + + FIELD_START_DATE + " ASC, " + + FIELD_START_TIME + " ASC " + + "LIMIT 1") + PagingIterable getFirst(String book, Function attributes); + + @Update(nullSavingStrategy = NullSavingStrategy.SET_TO_NULL) ResultSet update(PageEntity entity, Function attributes); diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/intervals/CassandraIntervalsWorker.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/intervals/CassandraIntervalsWorker.java index 66b248453..06bb88c6d 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/intervals/CassandraIntervalsWorker.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/intervals/CassandraIntervalsWorker.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,6 +43,8 @@ import java.util.function.Function; import static com.exactpro.cradle.CradleStorage.TIMEZONE_OFFSET; +import static com.exactpro.cradle.cassandra.utils.StorageUtils.toLocalDate; +import static com.exactpro.cradle.cassandra.utils.StorageUtils.toLocalTime; public class CassandraIntervalsWorker extends Worker implements IntervalsWorker { @@ -119,15 +121,15 @@ public CompletableFuture storeIntervalAsync(Interval interval) { IntervalEntity.IntervalEntityBuilder builder = IntervalEntity.builder() .setBook(interval.getBookId().getName()) - .setStartDate(LocalDate.ofInstant(interval.getStart(), TIMEZONE_OFFSET)) - .setStartTime(LocalTime.ofInstant(interval.getStart(), TIMEZONE_OFFSET)) + .setStartDate(toLocalDate(interval.getStart())) + .setStartTime(toLocalTime(interval.getStart())) .setCrawlerName(interval.getCrawlerName()) .setCrawlerType(interval.getCrawlerType()) .setCrawlerVersion(interval.getCrawlerVersion()) - .setEndDate(LocalDate.ofInstant(interval.getEnd(), TIMEZONE_OFFSET)) - .setEndTime(LocalTime.ofInstant(interval.getEnd(), TIMEZONE_OFFSET)) - .setLastUpdateDate(LocalDate.ofInstant(interval.getLastUpdate(), TIMEZONE_OFFSET)) - .setLastUpdateTime(LocalTime.ofInstant(interval.getLastUpdate(), TIMEZONE_OFFSET)) + .setEndDate(toLocalDate(interval.getEnd())) + .setEndTime(toLocalTime(interval.getEnd())) + .setLastUpdateDate(toLocalDate(interval.getLastUpdate())) + .setLastUpdateTime(toLocalTime(interval.getLastUpdate())) .setRecoveryState(interval.getRecoveryState()) .setProcessed(interval.isProcessed()); @@ -142,9 +144,9 @@ public CompletableFuture storeIntervalAsync(Interval interval) { @Override public Iterable getIntervalsPerDay(BookId bookId, Instant from, Instant to, String crawlerName, String crawlerVersion, String crawlerType) throws CradleStorageException { - LocalDate date = LocalDate.ofInstant(from, TIMEZONE_OFFSET); - LocalTime fromTime = LocalTime.ofInstant(from, TIMEZONE_OFFSET); - LocalTime toTime = LocalTime.ofInstant(to, TIMEZONE_OFFSET); + LocalDate date = toLocalDate(from); + LocalTime fromTime = toLocalTime(from); + LocalTime toTime = toLocalTime(to); String queryInfo = String.format("Getting intervals for crawler %s:%s for day %s between times %s-%s in book %s", crawlerName, @@ -164,9 +166,9 @@ public Iterable getIntervalsPerDay(BookId bookId, Instant from, Instan @Override public CompletableFuture> getIntervalsPerDayAsync(BookId bookId, Instant from, Instant to, String crawlerName, String crawlerVersion, String crawlerType) throws CradleStorageException { - LocalDate date = LocalDate.ofInstant(from, TIMEZONE_OFFSET); - LocalTime fromTime = LocalTime.ofInstant(from, TIMEZONE_OFFSET); - LocalTime toTime = LocalTime.ofInstant(to, TIMEZONE_OFFSET); + LocalDate date = toLocalDate(from); + LocalTime fromTime = toLocalTime(from); + LocalTime toTime = toLocalTime(to); String queryInfo = String.format("Getting intervals for crawler %s:%s for day %s between times %s-%s in book %s", crawlerName, @@ -200,7 +202,7 @@ public Iterable getIntervals(BookId bookId, Instant from, Instant to, Iterable result = new ArrayList<>(); - if (fromDateTime.toLocalDate().compareTo(toDateTime.toLocalDate()) == 0) + if (fromDateTime.toLocalDate().isEqual(toDateTime.toLocalDate())) { return getIntervalsPerDay(bookId, from, to, crawlerName, crawlerVersion, crawlerType); } @@ -300,8 +302,8 @@ public CompletableFuture setIntervalLastUpdateTimeAndDateAsync(Interva LocalDateTime dateTime = LocalDateTime.ofInstant(newLastUpdateTime, TIMEZONE_OFFSET); - LocalDate startDate = LocalDate.ofInstant(interval.getStart(), TIMEZONE_OFFSET); - LocalTime startTime = LocalTime.ofInstant(interval.getStart(), TIMEZONE_OFFSET); + LocalDate startDate = toLocalDate(interval.getStart()); + LocalTime startTime = toLocalTime(interval.getStart()); LocalDate newDate = dateTime.toLocalDate(); LocalTime newTime = dateTime.toLocalTime(); @@ -352,8 +354,8 @@ public CompletableFuture updateRecoveryStateAsync(Interval interval, S LocalTime newLastUpdateTime = newLastUpdateDateTime.toLocalTime(); LocalDate newLastUpdateDate = newLastUpdateDateTime.toLocalDate(); - LocalDate startDate = LocalDate.ofInstant(interval.getStart(), TIMEZONE_OFFSET); - LocalTime startTime = LocalTime.ofInstant(interval.getStart(), TIMEZONE_OFFSET); + LocalDate startDate = toLocalDate(interval.getStart()); + LocalTime startTime = toLocalTime(interval.getStart()); LocalDate oldUpdateDate = LocalDate.from(interval.getLastUpdate().atOffset(TIMEZONE_OFFSET)); LocalTime oldUpdateTime = LocalTime.from(interval.getLastUpdate().atOffset(TIMEZONE_OFFSET)); @@ -403,8 +405,8 @@ public CompletableFuture setIntervalProcessedAsync(Interval interval, LocalDate newLastUpdateDate = newLastUpdateDateTime.toLocalDate(); - LocalDate startDate = LocalDate.ofInstant(interval.getStart(), TIMEZONE_OFFSET); - LocalTime startTime = LocalTime.ofInstant(interval.getStart(), TIMEZONE_OFFSET); + LocalDate startDate = toLocalDate(interval.getStart()); + LocalTime startTime = toLocalTime(interval.getStart()); LocalDate oldUpdateDate = LocalDate.from(interval.getLastUpdate().atOffset(TIMEZONE_OFFSET)); LocalTime oldUpdateTime = LocalTime.from(interval.getLastUpdate().atOffset(TIMEZONE_OFFSET)); diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/AbstractMessageIteratorProvider.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/AbstractMessageIteratorProvider.java index 7f9ff2eab..09c623a40 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/AbstractMessageIteratorProvider.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/AbstractMessageIteratorProvider.java @@ -20,7 +20,6 @@ import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.core.cql.Row; import com.exactpro.cradle.BookInfo; -import com.exactpro.cradle.Order; import com.exactpro.cradle.PageId; import com.exactpro.cradle.PageInfo; import com.exactpro.cradle.cassandra.dao.CassandraOperators; @@ -32,7 +31,7 @@ import com.exactpro.cradle.cassandra.iterators.PagedIterator; import com.exactpro.cradle.cassandra.resultset.IteratorProvider; import com.exactpro.cradle.cassandra.retries.SelectQueryExecutor; -import com.exactpro.cradle.cassandra.utils.FilterUtils; +import com.exactpro.cradle.cassandra.utils.StorageUtils; import com.exactpro.cradle.cassandra.workers.MessagesWorker; import com.exactpro.cradle.filters.FilterForAny; import com.exactpro.cradle.filters.FilterForGreater; @@ -47,6 +46,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; @@ -57,11 +58,15 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; +import static com.exactpro.cradle.Order.REVERSE; import static com.exactpro.cradle.cassandra.dao.messages.MessageBatchEntity.FIELD_FIRST_MESSAGE_TIME; +import static com.exactpro.cradle.cassandra.utils.FilterUtils.findFirstTimestamp; +import static com.exactpro.cradle.cassandra.utils.FilterUtils.findLastTimestamp; +import static java.util.Objects.requireNonNull; abstract public class AbstractMessageIteratorProvider extends IteratorProvider { - private static final Logger logger = LoggerFactory.getLogger(AbstractMessageIteratorProvider.class); + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractMessageIteratorProvider.class); protected final MessageBatchOperator op; protected final MessageBatchEntityConverter converter; protected final BookInfo book; @@ -69,16 +74,17 @@ abstract public class AbstractMessageIteratorProvider extends IteratorProvide protected final SelectQueryExecutor selectQueryExecutor; protected final FilterForGreater leftBoundFilter; protected final FilterForLess rightBoundFilter; - protected PageInfo firstPage, lastPage; protected final Function readAttrs; protected final MessageFilter filter; /** limit must be strictly positive ( limit greater than 0 ) */ protected final int limit; protected final AtomicInteger returned; - protected CassandraStoredMessageFilter cassandraFilter; protected final MessageBatchIteratorFilter batchFilter; protected final MessageBatchIteratorCondition iterationCondition; - protected TakeWhileIterator takeWhileIterator; + + private final Iterator pageProvider; + // only get / set operation are guarded by lock + private TakeWhileIterator takeWhileIterator; public AbstractMessageIteratorProvider(String requestInfo, MessageFilter filter, CassandraOperators operators, BookInfo book, ExecutorService composingService, SelectQueryExecutor selectQueryExecutor, @@ -96,7 +102,6 @@ public AbstractMessageIteratorProvider(String requestInfo, MessageFilter filter, this.returned = new AtomicInteger(); this.leftBoundFilter = createLeftBoundFilter(filter); this.rightBoundFilter = createRightBoundFilter(filter); - this.cassandraFilter = createInitialFilter(filter); FilterForAny sequenceFilter = filter.getSequence(); MessageBatchIteratorFilter batchFilter; @@ -114,153 +119,115 @@ public AbstractMessageIteratorProvider(String requestInfo, MessageFilter filter, this.batchFilter = batchFilter; this.iterationCondition = iterationCondition; + + this.pageProvider = book.getPages( + requireNonNull(leftBoundFilter.getValue()), + requireNonNull(rightBoundFilter.getValue()), + filter.getOrder() + ); } - //TODO refactor this method to assigns firstPage outside the method protected FilterForGreater createLeftBoundFilter(MessageFilter filter) throws CradleStorageException { FilterForGreater result = filter.getTimestampFrom(); - firstPage = FilterUtils.findFirstPage(filter.getPageId(), result, book); - - if (result == null && firstPage == null) - return null; - - Instant leftBoundFromPage = firstPage == null ? Instant.MIN : firstPage.getStarted(); - if (result == null || (filter.getPageId() != null && leftBoundFromPage.isAfter(result.getValue()))) + Instant leftBoundFromPage = findFirstTimestamp(filter.getPageId(), result, book); + if (leftBoundFromPage == null) { + if (result != null) { + return result; + } + return FilterForGreater.forGreaterOrEquals(Instant.MIN); + } + if (result == null || (filter.getPageId() != null && leftBoundFromPage.isAfter(result.getValue()))) { return FilterForGreater.forGreaterOrEquals(leftBoundFromPage); + } // If the page wasn't specified in the filter, we should find a batch with a lower date, // which may contain messages that satisfy the original condition - LocalDateTime leftBoundLocalDate = TimeUtils.toLocalTimestamp(result.getValue()); - LocalTime nearestBatchTime = getNearestBatchTime(firstPage, filter.getSessionAlias(), + LocalDateTime leftBoundLocalDate = StorageUtils.toLocalDateTime(result.getValue()); + LocalTime nearestBatchTime = getNearestBatchTime(leftBoundFromPage, filter.getSessionAlias(), filter.getDirection().getLabel(), leftBoundLocalDate.toLocalDate(), leftBoundLocalDate.toLocalTime()); if (nearestBatchTime != null) { Instant nearestBatchInstant = TimeUtils.toInstant(leftBoundLocalDate.toLocalDate(), nearestBatchTime); - if (nearestBatchInstant.isBefore(result.getValue())) + if (nearestBatchInstant.isBefore(result.getValue())) { result = FilterForGreater.forGreaterOrEquals(nearestBatchInstant); - firstPage = FilterUtils.findFirstPage(filter.getPageId(), result, book); + } } - return result; } - private LocalTime getNearestBatchTime(PageInfo page, String sessionAlias, String direction, - LocalDate messageDate, LocalTime messageTime) throws CradleStorageException - { - while (page != null) - { + private LocalTime getNearestBatchTime(Instant timestamp, + String sessionAlias, + String direction, + LocalDate messageDate, + LocalTime messageTime) throws CradleStorageException { + if (timestamp == null) { + return null; + } + Iterator pageInfoIterator = book.getPages(null, timestamp, REVERSE); + while (pageInfoIterator.hasNext()) { + PageInfo page = pageInfoIterator.next(); CompletableFuture future = op.getNearestTime( - page.getId().getBookId().getName(), - page.getId().getName(), + page.getBookName(), + page.getName(), sessionAlias, direction, messageDate, messageTime, readAttrs); - try - { + try { Row row = future.get(); - if (row != null) + if (row != null) { return row.getLocalTime(FIELD_FIRST_MESSAGE_TIME); - } - catch (Exception e) - { + } + } catch (Exception e) { throw new CradleStorageException("Error while getting left bound ", e); } - if (TimeUtils.toLocalTimestamp(page.getStarted()).toLocalDate().isBefore(messageDate)) + if (StorageUtils.toLocalDateTime(page.getStarted()).toLocalDate().isBefore(messageDate)) { return null; - page = book.getPreviousPage(page.getStarted()); + } } return null; } - //TODO refactor this method to assign last page outside of this method. protected FilterForLess createRightBoundFilter(MessageFilter filter) { FilterForLess result = filter.getTimestampTo(); - lastPage = FilterUtils.findLastPage(filter.getPageId(), result, book); - Instant endOfPage = lastPage == null || lastPage.getEnded() == null ? Instant.now() : lastPage.getEnded(); - - return FilterForLess.forLessOrEquals(result == null || endOfPage.isBefore(result.getValue()) ? endOfPage : result.getValue()); - } + Instant lastTimestamp = findLastTimestamp(filter.getPageId(), result, book); + Instant endTimestamp = lastTimestamp == null ? Instant.now() : lastTimestamp; - protected CassandraStoredMessageFilter createInitialFilter(MessageFilter filter) { - if (filter.getOrder() == Order.DIRECT) { - return new CassandraStoredMessageFilter( - book.getId().getName(), - firstPage != null ? firstPage.getId().getName() : null, - filter.getSessionAlias(), - filter.getDirection().getLabel(), - leftBoundFilter, - rightBoundFilter, - filter.getLimit(), - filter.getOrder()); - } else { - return new CassandraStoredMessageFilter( - book.getId().getName(), - lastPage != null ? lastPage.getId().getName() : null, - filter.getSessionAlias(), - filter.getDirection().getLabel(), - leftBoundFilter, - rightBoundFilter, - filter.getLimit(), - filter.getOrder()); - } + return FilterForLess.forLessOrEquals(result == null || endTimestamp.isBefore(result.getValue()) + ? endTimestamp + : result.getValue()); } - protected CassandraStoredMessageFilter createNextFilter(CassandraStoredMessageFilter prevFilter, int updatedLimit) - { - PageInfo oldPage = book.getPage(new PageId(book.getId(), prevFilter.getPage())); - PageInfo newPage; - - if (filter.getOrder() == Order.DIRECT) { - if (oldPage.equals(lastPage)) - return null; - - newPage = book.getNextPage(oldPage.getStarted()); - } else { - if (oldPage.equals(firstPage)) { - return null; - } - - newPage = book.getPreviousPage(oldPage.getStarted()); - } - + protected CassandraStoredMessageFilter createFilter(@Nonnull PageInfo pageInfo, int updatedLimit) { return new CassandraStoredMessageFilter( - newPage.getId().getBookId().getName(), - newPage.getId().getName(), - prevFilter.getSessionAlias(), - prevFilter.getDirection(), + pageInfo.getId(), + filter.getSessionAlias(), + filter.getDirection().getLabel(), leftBoundFilter, rightBoundFilter, updatedLimit, filter.getOrder()); } - protected boolean interruptIteratorChecks () { - if (cassandraFilter == null) { - return true; - } - + protected @Nullable PageInfo nextPage() { if (takeWhileIterator != null && takeWhileIterator.isHalted()) { - logger.debug("Iterator was interrupted because iterator condition was not met"); - return true; + LOGGER.debug("Iterator was interrupted because iterator condition was not met"); + return null; } if (limit > 0 && returned.get() >= limit) { - logger.debug("Filtering interrupted because limit for records to return ({}) is reached ({})", limit, returned); - return true; + LOGGER.debug("Filtering interrupted because limit for records to return ({}) is reached ({})", limit, returned); + return null; } - return false; + return pageProvider.hasNext() ? pageProvider.next() : null; } - protected Iterator getBatchedIterator (MappedAsyncPagingIterable resultSet) { - PageId pageId = new PageId(book.getId(), cassandraFilter.getPage()); - // Updated limit should be smaller, since we already got entities from previous batch - cassandraFilter = createNextFilter(cassandraFilter, Math.max(limit - returned.get(),0)); - + protected Iterator getBatchedIterator(PageId pageId, + MappedAsyncPagingIterable resultSet) { PagedIterator pagedIterator = new PagedIterator<>( resultSet, selectQueryExecutor, @@ -269,6 +236,7 @@ protected Iterator getBatchedIterator (MappedAsyncPagingIter FilteringIterator filteringIterator = new FilteringIterator<>( pagedIterator, batchFilter::test); + // We need to store this iterator since // it gives info whether or no iterator was halted takeWhileIterator = new TakeWhileIterator<>( diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/CassandraGroupedMessageFilter.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/CassandraGroupedMessageFilter.java index 60e75904b..2252d0928 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/CassandraGroupedMessageFilter.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/CassandraGroupedMessageFilter.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,22 +19,32 @@ import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.core.metadata.schema.ClusteringOrder; import com.datastax.oss.driver.api.querybuilder.select.Select; -import com.datastax.oss.driver.shaded.guava.common.base.Preconditions; import com.exactpro.cradle.Order; +import com.exactpro.cradle.PageId; import com.exactpro.cradle.cassandra.dao.CassandraFilter; import com.exactpro.cradle.cassandra.utils.FilterUtils; import com.exactpro.cradle.filters.FilterForGreater; import com.exactpro.cradle.filters.FilterForLess; +import javax.annotation.Nonnull; import java.time.Instant; import java.util.StringJoiner; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.bindMarker; -import static com.exactpro.cradle.cassandra.dao.messages.GroupedMessageBatchEntity.*; -import static com.exactpro.cradle.cassandra.dao.messages.CassandraStoredMessageFilter.*; +import static com.exactpro.cradle.cassandra.dao.messages.CassandraStoredMessageFilter.DATE_FROM; +import static com.exactpro.cradle.cassandra.dao.messages.CassandraStoredMessageFilter.DATE_TO; +import static com.exactpro.cradle.cassandra.dao.messages.CassandraStoredMessageFilter.TIME_FROM; +import static com.exactpro.cradle.cassandra.dao.messages.CassandraStoredMessageFilter.TIME_TO; +import static com.exactpro.cradle.cassandra.dao.messages.GroupedMessageBatchEntity.FIELD_ALIAS_GROUP; +import static com.exactpro.cradle.cassandra.dao.messages.GroupedMessageBatchEntity.FIELD_BOOK; +import static com.exactpro.cradle.cassandra.dao.messages.GroupedMessageBatchEntity.FIELD_FIRST_MESSAGE_DATE; +import static com.exactpro.cradle.cassandra.dao.messages.GroupedMessageBatchEntity.FIELD_FIRST_MESSAGE_TIME; +import static com.exactpro.cradle.cassandra.dao.messages.GroupedMessageBatchEntity.FIELD_PAGE; +import static java.util.Objects.requireNonNull; public class CassandraGroupedMessageFilter implements CassandraFilter { - private final String book, page, groupName; + private final @Nonnull String groupName; + private final @Nonnull PageId pageId; /** limit must be strictly positive ( limit greater than 0 ) */ private final int limit; @@ -42,16 +52,14 @@ public class CassandraGroupedMessageFilter implements CassandraFilter messageTimeTo; private final Order order; - public CassandraGroupedMessageFilter(String book, - String page, + public CassandraGroupedMessageFilter(PageId pageId, String groupName, FilterForGreater messageTimeFrom, FilterForLess messageTimeTo, Order order, int limit) { - this.book = book; - this.page = page; - this.groupName = groupName; + this.pageId = requireNonNull(pageId, "page id can't be null because book and page names are part of partition"); + this.groupName = requireNonNull(groupName, "group name can't be null because it is part of partition"); this.messageTimeFrom = messageTimeFrom; this.messageTimeTo = messageTimeTo; this.order = order; @@ -88,8 +96,8 @@ public Select addConditions(Select select) { @Override public BoundStatementBuilder bindParameters(BoundStatementBuilder builder) { builder = builder - .setString(FIELD_BOOK, book) - .setString(FIELD_PAGE, page) + .setString(FIELD_BOOK, pageId.getBookId().getName()) + .setString(FIELD_PAGE, pageId.getName()) .setString(FIELD_ALIAS_GROUP, groupName); if (messageTimeFrom != null) @@ -100,15 +108,19 @@ public BoundStatementBuilder bindParameters(BoundStatementBuilder builder) { return builder; } - public String getBook() { - return book; + public @Nonnull PageId getPageId() { + return pageId; } - public String getPage() { - return page; + public @Nonnull String getBook() { + return pageId.getBookId().getName(); } - public String getGroupName() { + public @Nonnull String getPage() { + return pageId.getName(); + } + + public @Nonnull String getGroupName() { return groupName; } @@ -127,8 +139,7 @@ public FilterForLess getMessageTimeTo() { @Override public String toString() { return new StringJoiner(", ", CassandraGroupedMessageFilter.class.getSimpleName() + "[", "]") - .add("book='" + book + "'") - .add("page='" + page + "'") + .add("pageId='" + pageId + "'") .add("groupName='" + groupName + "'") .add("limit=" + limit) .add("messageTimeFrom " + messageTimeFrom) diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/CassandraStoredMessageFilter.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/CassandraStoredMessageFilter.java index 4ac1e81f6..7e1c6a64c 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/CassandraStoredMessageFilter.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/CassandraStoredMessageFilter.java @@ -21,6 +21,7 @@ import com.datastax.oss.driver.api.querybuilder.relation.MultiColumnRelationBuilder; import com.datastax.oss.driver.api.querybuilder.select.Select; import com.exactpro.cradle.Order; +import com.exactpro.cradle.PageId; import com.exactpro.cradle.cassandra.dao.CassandraFilter; import com.exactpro.cradle.cassandra.utils.FilterUtils; import com.exactpro.cradle.filters.ComparisonOperation; @@ -28,14 +29,21 @@ import com.exactpro.cradle.filters.FilterForGreater; import com.exactpro.cradle.filters.FilterForLess; +import javax.annotation.Nonnull; import java.time.Instant; import java.util.ArrayList; import java.util.List; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.bindMarker; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.tuple; - -import static com.exactpro.cradle.cassandra.dao.messages.MessageBatchEntity.*; +import static com.exactpro.cradle.cassandra.dao.messages.MessageBatchEntity.FIELD_BOOK; +import static com.exactpro.cradle.cassandra.dao.messages.MessageBatchEntity.FIELD_DIRECTION; +import static com.exactpro.cradle.cassandra.dao.messages.MessageBatchEntity.FIELD_FIRST_MESSAGE_DATE; +import static com.exactpro.cradle.cassandra.dao.messages.MessageBatchEntity.FIELD_FIRST_MESSAGE_TIME; +import static com.exactpro.cradle.cassandra.dao.messages.MessageBatchEntity.FIELD_PAGE; +import static com.exactpro.cradle.cassandra.dao.messages.MessageBatchEntity.FIELD_SEQUENCE; +import static com.exactpro.cradle.cassandra.dao.messages.MessageBatchEntity.FIELD_SESSION_ALIAS; +import static java.util.Objects.requireNonNull; public class CassandraStoredMessageFilter implements CassandraFilter { @@ -43,7 +51,9 @@ public class CassandraStoredMessageFilter implements CassandraFilter messageTimeFrom; private final FilterForLess messageTimeTo; @@ -54,27 +64,12 @@ public class CassandraStoredMessageFilter implements CassandraFilter messageTimeFrom, FilterForLess messageTimeTo) - { - this.book = book; - this.page = page; - this.sessionAlias = sessionAlias; - this.direction = direction; - this.messageTimeFrom = messageTimeFrom; - this.messageTimeTo = messageTimeTo; - this.sequence = null; - this.limit = 0; - this.order = Order.DIRECT; - } - - public CassandraStoredMessageFilter(String book, String page, String sessionAlias, String direction, + public CassandraStoredMessageFilter(PageId pageId, String sessionAlias, String direction, FilterForGreater messageTimeFrom, FilterForLess messageTimeTo, int limit, Order order) { - this.book = book; - this.page = page; - this.sessionAlias = sessionAlias; - this.direction = direction; + this.pageId = requireNonNull(pageId, "page id can't be null because book and page names are part of partition"); + this.sessionAlias = requireNonNull(sessionAlias, "session alias can't be null because it is part of partition"); + this.direction = requireNonNull(direction, "direction can't be null because it is part of partition"); this.messageTimeFrom = messageTimeFrom; this.messageTimeTo = messageTimeTo; this.sequence = null; @@ -86,13 +81,10 @@ public CassandraStoredMessageFilter(String book, String page, String sessionAlia public Select addConditions(Select select) { select = select .whereColumn(FIELD_BOOK).isEqualTo(bindMarker()) + .whereColumn(FIELD_PAGE).isEqualTo(bindMarker()) .whereColumn(FIELD_SESSION_ALIAS).isEqualTo(bindMarker()) .whereColumn(FIELD_DIRECTION).isEqualTo(bindMarker()); - if (page != null) { - select = select.whereColumn(FIELD_PAGE).isEqualTo(bindMarker()); - } - if (sequence != null) select = addMessageIdConditions(select); else @@ -118,14 +110,11 @@ public Select addConditions(Select select) { @Override public BoundStatementBuilder bindParameters(BoundStatementBuilder builder) { builder = builder - .setString(FIELD_BOOK, book) + .setString(FIELD_BOOK, pageId.getBookId().getName()) + .setString(FIELD_PAGE, pageId.getName()) .setString(FIELD_SESSION_ALIAS, sessionAlias) .setString(FIELD_DIRECTION, direction); - if (page != null) { - builder = builder.setString(FIELD_PAGE, page); - } - if (sequence != null) builder = bindMessageIdParameters(builder); else @@ -138,23 +127,23 @@ public BoundStatementBuilder bindParameters(BoundStatementBuilder builder) { return builder; } - public String getBook() - { - return book; + public @Nonnull String getBook() { + return pageId.getBookId().getName(); } - public String getPage() - { - return page; + public @Nonnull String getPage() { + return pageId.getName(); } - public String getSessionAlias() - { + public @Nonnull PageId getPageId() { + return pageId; + } + + public @Nonnull String getSessionAlias() { return sessionAlias; } - public String getDirection() - { + public @Nonnull String getDirection() { return direction; } @@ -169,14 +158,9 @@ public FilterForAny getSequence() public String toString() { List result = new ArrayList<>(10); - if (book != null) - result.add("book=" + book); - if (page != null) - result.add("page=" + page); - if (sessionAlias != null) - result.add("sessionAlias=" + sessionAlias); - if (direction != null) - result.add("direction=" + direction); + result.add("pageId=" + pageId); + result.add("sessionAlias=" + sessionAlias); + result.add("direction=" + direction); if (messageTimeFrom != null) result.add("timestamp" + messageTimeFrom); if (messageTimeTo != null) diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/GroupedMessageIteratorProvider.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/GroupedMessageIteratorProvider.java index 8af0a1f8a..55a46654f 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/GroupedMessageIteratorProvider.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/GroupedMessageIteratorProvider.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,14 +20,13 @@ import com.datastax.oss.driver.api.core.cql.Row; import com.exactpro.cradle.BookInfo; import com.exactpro.cradle.Order; -import com.exactpro.cradle.PageId; import com.exactpro.cradle.PageInfo; import com.exactpro.cradle.cassandra.dao.CassandraOperators; import com.exactpro.cradle.cassandra.dao.messages.converters.GroupedMessageBatchEntityConverter; import com.exactpro.cradle.cassandra.iterators.PagedIterator; import com.exactpro.cradle.cassandra.resultset.IteratorProvider; import com.exactpro.cradle.cassandra.retries.SelectQueryExecutor; -import com.exactpro.cradle.cassandra.utils.FilterUtils; +import com.exactpro.cradle.cassandra.utils.StorageUtils; import com.exactpro.cradle.cassandra.workers.MessagesWorker; import com.exactpro.cradle.filters.FilterForGreater; import com.exactpro.cradle.filters.FilterForLess; @@ -39,6 +38,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; @@ -49,7 +49,12 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; +import static com.exactpro.cradle.Order.REVERSE; import static com.exactpro.cradle.cassandra.dao.messages.MessageBatchEntity.FIELD_FIRST_MESSAGE_TIME; +import static com.exactpro.cradle.cassandra.utils.FilterUtils.findFirstTimestamp; +import static com.exactpro.cradle.cassandra.utils.FilterUtils.findLastTimestamp; +import static com.exactpro.cradle.filters.FilterForLess.forLessOrEquals; +import static java.lang.Math.max; public class GroupedMessageIteratorProvider extends IteratorProvider { @@ -63,12 +68,11 @@ public class GroupedMessageIteratorProvider extends IteratorProvider leftBoundFilter; protected final FilterForLess rightBoundFilter; - protected PageInfo firstPage, lastPage; + private final Iterator pageProvider; private final Function readAttrs; /** limit must be strictly positive ( limit greater than 0 ) */ private final int limit; private final AtomicInteger returned; - protected CassandraGroupedMessageFilter cassandraFilter; private final Order order; public GroupedMessageIteratorProvider(String requestInfo, @@ -91,40 +95,34 @@ public GroupedMessageIteratorProvider(String requestInfo, this.limit = filter.getLimit(); this.returned = new AtomicInteger(); this.leftBoundFilter = createLeftBoundFilter(filter); - this.firstPage = FilterUtils.findFirstPage(filter.getPageId(), leftBoundFilter, book); this.rightBoundFilter = createRightBoundFilter(filter); - this.lastPage = FilterUtils.findLastPage(filter.getPageId(), rightBoundFilter, book); + this.pageProvider = book.getPages( + leftBoundFilter.getValue(), + rightBoundFilter.getValue(), + order + ); this.order = order; - - // Filter should be initialized last as it might use above initialized properties - this.cassandraFilter = createInitialFilter(filter); - } - - private CassandraGroupedMessageFilter createInitialFilter(GroupedMessageFilter filter) { - return new CassandraGroupedMessageFilter( - book.getId().getName(), - getFirstPage().getId().getName(), - filter.getGroupName(), - leftBoundFilter, - rightBoundFilter, - order, - filter.getLimit()); } - //TODO refactor or split this method to avoid findFirstPage calculation multiple times. private FilterForGreater createLeftBoundFilter(GroupedMessageFilter filter) throws CradleStorageException { FilterForGreater result = filter.getFrom(); - var firstPageLocal = FilterUtils.findFirstPage(filter.getPageId(), result, book); - Instant leftBoundFromPage = firstPageLocal.getStarted(); - if (result == null || (filter.getPageId() != null && leftBoundFromPage.isAfter(result.getValue()))) + Instant leftBoundFromPage = findFirstTimestamp(filter.getPageId(), result, book); + if (leftBoundFromPage == null) { + if (result != null) { + return result; + } + return FilterForGreater.forGreaterOrEquals(Instant.MIN); + } + if (result == null || (filter.getPageId() != null && leftBoundFromPage.isAfter(result.getValue()))) { return FilterForGreater.forGreaterOrEquals(leftBoundFromPage); + } // If the page wasn't specified in the filter, we should find a batch with a lower date, // which may contain messages that satisfy the original condition - LocalDateTime leftBoundLocalDate = TimeUtils.toLocalTimestamp(result.getValue()); + LocalDateTime leftBoundLocalDate = StorageUtils.toLocalDateTime(result.getValue()); LocalTime nearestBatchTime = getNearestBatchTime( - firstPageLocal, + leftBoundFromPage, filter.getGroupName(), leftBoundLocalDate.toLocalDate(), leftBoundLocalDate.toLocalTime()); @@ -132,86 +130,63 @@ private FilterForGreater createLeftBoundFilter(GroupedMessageFilter fil if (nearestBatchTime != null) { Instant nearestBatchInstant = TimeUtils.toInstant(leftBoundLocalDate.toLocalDate(), nearestBatchTime); - if (nearestBatchInstant.isBefore(result.getValue())) + if (nearestBatchInstant.isBefore(result.getValue())) { result = FilterForGreater.forGreaterOrEquals(nearestBatchInstant); + } } return result; } - private LocalTime getNearestBatchTime(PageInfo page, String groupAlias, LocalDate messageDate, LocalTime messageTime) throws CradleStorageException - { - while (page != null) - { + private LocalTime getNearestBatchTime(Instant timestamp, + String groupAlias, + LocalDate messageDate, + LocalTime messageTime) throws CradleStorageException { + if (timestamp == null) { + return null; + } + Iterator pageInfoIterator = book.getPages(null, timestamp, REVERSE); + while (pageInfoIterator.hasNext()) { + PageInfo page = pageInfoIterator.next(); CompletableFuture future = op.getNearestTime( - page.getId().getBookId().getName(), - page.getId().getName(), + page.getBookName(), + page.getName(), groupAlias, messageDate, messageTime, readAttrs); - try - { + try { Row row = future.get(); if (row != null) return row.getLocalTime(FIELD_FIRST_MESSAGE_TIME); - } - catch (Exception e) - { + } catch (Exception e) { throw new CradleStorageException("Error while getting left bound ", e); } - if (TimeUtils.toLocalTimestamp(page.getStarted()).toLocalDate().isBefore(messageDate)) + if (StorageUtils.toLocalDateTime(page.getStarted()).toLocalDate().isBefore(messageDate)) { return null; - page = book.getPreviousPage(page.getStarted()); + } } return null; } - //TODO refactor or split this method to avoid findLastPage calculation multiple times. - protected FilterForLess createRightBoundFilter(GroupedMessageFilter filter) - { - FilterForLess result = filter.getTo(); - var lastPageLocal = FilterUtils.findLastPage(filter.getPageId(), result, book); - Instant endOfPage = lastPageLocal.getEnded() == null ? Instant.now() : lastPageLocal.getEnded(); - - return FilterForLess.forLessOrEquals(result == null || endOfPage.isBefore(result.getValue()) ? endOfPage : result.getValue()); - } - - protected CassandraGroupedMessageFilter createNextFilter(CassandraGroupedMessageFilter prevFilter, int updatedLimit) { - PageInfo prevPage = book.getPage(new PageId(book.getId(), prevFilter.getPage())); - if (prevPage.equals(getLastPage())) - return null; - - PageInfo nextPage = getNextPage(prevPage.getStarted()); - - return new CassandraGroupedMessageFilter( - book.getId().getName(), - nextPage.getId().getName(), - prevFilter.getGroupName(), - prevFilter.getMessageTimeFrom(), - prevFilter.getMessageTimeTo(), - order, - updatedLimit); - } - @Override public CompletableFuture> nextIterator() { - if (cassandraFilter == null) + if (!pageProvider.hasNext()) { return CompletableFuture.completedFuture(null); + } + PageInfo nextPage = pageProvider.next(); + if (limit > 0 && returned.get() >= limit) { logger.debug("Filtering interrupted because limit for records to return ({}) is reached ({})", limit, returned); return CompletableFuture.completedFuture(null); } + CassandraGroupedMessageFilter cassandraFilter = createFilter(nextPage, max(limit - returned.get(), 0)); + logger.debug("Getting next iterator for '{}' by filter {}", getRequestInfo(), cassandraFilter); return op.getByFilter(cassandraFilter, selectQueryExecutor, getRequestInfo(), readAttrs) - .thenApplyAsync(resultSet -> - { - PageId pageId = new PageId(book.getId(), cassandraFilter.getPage()); - // Updated limit should be smaller, since we already got entities from previous batch - cassandraFilter = createNextFilter(cassandraFilter, Math.max(limit - returned.get(), 0)); - + .thenApplyAsync(resultSet -> { PagedIterator pagedIterator = new PagedIterator<>( resultSet, selectQueryExecutor, @@ -220,26 +195,29 @@ public CompletableFuture> nextIterator() { return new ConvertingIterator<>( pagedIterator, - entity -> MessagesWorker.mapGroupedMessageBatchEntity(pageId, entity)); + entity -> MessagesWorker.mapGroupedMessageBatchEntity(nextPage.getId(), entity)); }, composingService) .thenApplyAsync(it -> new FilteredGroupedMessageBatchIterator(it, filter, limit, returned), composingService); } - private boolean isDirectOrder() { - return order == Order.DIRECT; - } - - private PageInfo getFirstPage() { - return isDirectOrder() ? firstPage : lastPage; - } + private FilterForLess createRightBoundFilter(GroupedMessageFilter filter) { + FilterForLess result = filter.getTo(); + Instant lastTimestamp = findLastTimestamp(filter.getPageId(), result, book); + Instant endTimestamp = lastTimestamp == null ? Instant.now() : lastTimestamp; - private PageInfo getLastPage() { - return isDirectOrder() ? lastPage : firstPage; + return forLessOrEquals(result == null || endTimestamp.isBefore(result.getValue()) + ? endTimestamp + : result.getValue() + ); } - private PageInfo getNextPage(Instant currentPageStart) { - return isDirectOrder() - ? book.getNextPage(currentPageStart) - : book.getPreviousPage(currentPageStart); + private CassandraGroupedMessageFilter createFilter(@Nonnull PageInfo pageInfo, int updatedLimit) { + return new CassandraGroupedMessageFilter( + pageInfo.getId(), + filter.getGroupName(), + leftBoundFilter, + rightBoundFilter, + order, + updatedLimit); } } diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/MessageBatchesIteratorProvider.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/MessageBatchesIteratorProvider.java index 4697df9cc..37e64dd57 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/MessageBatchesIteratorProvider.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/MessageBatchesIteratorProvider.java @@ -18,6 +18,7 @@ import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.exactpro.cradle.BookInfo; +import com.exactpro.cradle.PageInfo; import com.exactpro.cradle.cassandra.dao.CassandraOperators; import com.exactpro.cradle.cassandra.retries.SelectQueryExecutor; import com.exactpro.cradle.iterators.LimitedIterator; @@ -32,6 +33,8 @@ import java.util.concurrent.ExecutorService; import java.util.function.Function; +import static java.lang.Math.max; + public class MessageBatchesIteratorProvider extends AbstractMessageIteratorProvider { private static final Logger logger = LoggerFactory.getLogger(MessageBatchesIteratorProvider.class); @@ -45,13 +48,16 @@ public MessageBatchesIteratorProvider(String requestInfo, MessageFilter filter, @Override public CompletableFuture> nextIterator() { - if (interruptIteratorChecks()) { + PageInfo nextPage = nextPage(); + if (nextPage == null) { return CompletableFuture.completedFuture(null); } + CassandraStoredMessageFilter cassandraFilter = createFilter(nextPage, max(limit - returned.get(), 0)); + logger.debug("Getting next iterator for '{}' by filter {}", getRequestInfo(), cassandraFilter); return op.getByFilter(cassandraFilter, selectQueryExecutor, getRequestInfo(), readAttrs) - .thenApplyAsync(this::getBatchedIterator, composingService) + .thenApplyAsync(resultSet -> getBatchedIterator(nextPage.getId(), resultSet), composingService) .thenApply(it -> limit > 0 ? new LimitedIterator<>(it, limit) : it); } } \ No newline at end of file diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/MessagesIteratorProvider.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/MessagesIteratorProvider.java index 3e378943f..ccfc97341 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/MessagesIteratorProvider.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/messages/MessagesIteratorProvider.java @@ -18,6 +18,7 @@ import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.exactpro.cradle.BookInfo; +import com.exactpro.cradle.PageInfo; import com.exactpro.cradle.cassandra.dao.CassandraOperators; import com.exactpro.cradle.cassandra.retries.SelectQueryExecutor; import com.exactpro.cradle.messages.MessageFilter; @@ -31,6 +32,8 @@ import java.util.concurrent.ExecutorService; import java.util.function.Function; +import static java.lang.Math.max; + public class MessagesIteratorProvider extends AbstractMessageIteratorProvider { private static final Logger logger = LoggerFactory.getLogger(MessagesIteratorProvider.class); @@ -42,13 +45,16 @@ public MessagesIteratorProvider(String requestInfo, MessageFilter filter, Cassan @Override public CompletableFuture> nextIterator() { - if (interruptIteratorChecks()) { + PageInfo nextPage = nextPage(); + if (nextPage == null) { return CompletableFuture.completedFuture(null); } + CassandraStoredMessageFilter cassandraFilter = createFilter(nextPage, max(limit - returned.get(), 0)); + logger.debug("Getting next iterator for '{}' by filter {}", getRequestInfo(), cassandraFilter); return op.getByFilter(cassandraFilter, selectQueryExecutor, getRequestInfo(), readAttrs) - .thenApplyAsync(this::getBatchedIterator, composingService) + .thenApplyAsync(resultSet -> getBatchedIterator(nextPage.getId(), resultSet), composingService) .thenApplyAsync(it -> new FilteredMessageIterator(it, filter, limit, returned), composingService); } } \ No newline at end of file diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/statistics/EntityStatisticsIteratorProvider.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/statistics/EntityStatisticsIteratorProvider.java index 5d8b0ed5c..79181bf7e 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/statistics/EntityStatisticsIteratorProvider.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/statistics/EntityStatisticsIteratorProvider.java @@ -1,15 +1,31 @@ +/* + * Copyright 2022-2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.exactpro.cradle.cassandra.dao.statistics; import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; -import com.exactpro.cradle.*; +import com.exactpro.cradle.BookInfo; +import com.exactpro.cradle.EntityType; +import com.exactpro.cradle.FrameType; +import com.exactpro.cradle.PageInfo; import com.exactpro.cradle.cassandra.counters.FrameInterval; import com.exactpro.cradle.cassandra.dao.CassandraOperators; import com.exactpro.cradle.cassandra.iterators.PagedIterator; import com.exactpro.cradle.cassandra.resultset.IteratorProvider; import com.exactpro.cradle.cassandra.retries.SelectQueryExecutor; -import com.exactpro.cradle.cassandra.utils.FilterUtils; import com.exactpro.cradle.counters.CounterSample; -import com.exactpro.cradle.filters.FilterForGreater; import com.exactpro.cradle.iterators.ConvertingIterator; import java.time.Instant; @@ -18,16 +34,17 @@ import java.util.concurrent.ExecutorService; import java.util.function.Function; +import static com.exactpro.cradle.Order.DIRECT; + public class EntityStatisticsIteratorProvider extends IteratorProvider { - private CassandraOperators operators; - private BookInfo book; - private ExecutorService composingService; - private SelectQueryExecutor selectQueryExecutor; - private EntityType entityType; - private FrameType frameType; - private FrameInterval frameInterval; - private Function readAttrs; - private PageInfo currentPage; + private final CassandraOperators operators; + private final ExecutorService composingService; + private final SelectQueryExecutor selectQueryExecutor; + private final EntityType entityType; + private final FrameType frameType; + private final FrameInterval frameInterval; + private final Function readAttrs; + private final Iterator pageProvider; public EntityStatisticsIteratorProvider(String requestInfo, CassandraOperators operators, BookInfo book, ExecutorService composingService, SelectQueryExecutor selectQueryExecutor, EntityType entityType, @@ -36,21 +53,25 @@ public EntityStatisticsIteratorProvider(String requestInfo, CassandraOperators o Function readAttrs) { super(requestInfo); this.operators = operators; - this.book = book; this.composingService = composingService; this.selectQueryExecutor = selectQueryExecutor; this.entityType = entityType; this.frameType = frameType; this.frameInterval = frameInterval; this.readAttrs = readAttrs; - this.currentPage = FilterUtils.findFirstPage(null, FilterForGreater.forGreater(frameInterval.getInterval().getStart()),book); + this.pageProvider = book.getPages( + frameInterval.getInterval().getStart(), + frameInterval.getInterval().getEnd(), + DIRECT + ); } @Override public CompletableFuture> nextIterator() { - if(currentPage == null || frameInterval.getInterval().getEnd().isBefore(currentPage.getStarted()) ){ + if(!pageProvider.hasNext()){ return CompletableFuture.completedFuture(null); } + PageInfo pageInfo = pageProvider.next(); Instant actualStart = frameInterval.getFrameType().getFrameStart(frameInterval.getInterval().getStart()); Instant actualEnd = frameInterval.getFrameType().getFrameEnd(frameInterval.getInterval().getEnd()); @@ -59,16 +80,14 @@ public CompletableFuture> nextIterator() { EntityStatisticsEntityConverter entityStatsConverter = operators.getEntityStatisticsEntityConverter(); return entityStatsOperator.getStatistics( - currentPage.getId().getBookId().getName(), - currentPage.getId().getName(), + pageInfo.getBookName(), + pageInfo.getName(), entityType.getValue(), frameType.getValue(), actualStart, actualEnd, readAttrs) .thenApplyAsync(rs -> { - currentPage = book.getNextPage(currentPage.getStarted()); - PagedIterator pagedIterator = new PagedIterator<>( rs, selectQueryExecutor, diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/statistics/MessageStatisticsIteratorProvider.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/statistics/MessageStatisticsIteratorProvider.java index f3e3f9882..28b9ff4c8 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/statistics/MessageStatisticsIteratorProvider.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/statistics/MessageStatisticsIteratorProvider.java @@ -1,3 +1,18 @@ +/* + * Copyright 2022-2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.exactpro.cradle.cassandra.dao.statistics; import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; @@ -9,9 +24,7 @@ import com.exactpro.cradle.cassandra.iterators.PagedIterator; import com.exactpro.cradle.cassandra.resultset.IteratorProvider; import com.exactpro.cradle.cassandra.retries.SelectQueryExecutor; -import com.exactpro.cradle.cassandra.utils.FilterUtils; import com.exactpro.cradle.counters.CounterSample; -import com.exactpro.cradle.filters.FilterForGreater; import com.exactpro.cradle.iterators.ConvertingIterator; import java.time.Instant; @@ -20,17 +33,17 @@ import java.util.concurrent.ExecutorService; import java.util.function.Function; -public class MessageStatisticsIteratorProvider extends IteratorProvider { - private CassandraOperators operators; - private BookInfo book; - private ExecutorService composingService; - private SelectQueryExecutor selectQueryExecutor; - private String sessionAlias; - private Direction direction; - private FrameInterval frameInterval; - private Function readAttrs; - private PageInfo currentPage; +import static com.exactpro.cradle.Order.DIRECT; +public class MessageStatisticsIteratorProvider extends IteratorProvider { + private final CassandraOperators operators; + private final ExecutorService composingService; + private final SelectQueryExecutor selectQueryExecutor; + private final String sessionAlias; + private final Direction direction; + private final FrameInterval frameInterval; + private final Function readAttrs; + private final Iterator pageProvider; public MessageStatisticsIteratorProvider(String requestInfo, CassandraOperators operators, BookInfo book, ExecutorService composingService, SelectQueryExecutor selectQueryExecutor, @@ -39,22 +52,26 @@ public MessageStatisticsIteratorProvider(String requestInfo, CassandraOperators super(requestInfo); this.operators = operators; - this.book = book; this.composingService = composingService; this.selectQueryExecutor = selectQueryExecutor; this.sessionAlias = sessionAlias; this.direction = direction; this.frameInterval = frameInterval; this.readAttrs = readAttrs; - this.currentPage = FilterUtils.findFirstPage(null, FilterForGreater.forGreater(frameInterval.getInterval().getStart()),book); + this.pageProvider = book.getPages( + frameInterval.getInterval().getStart(), + frameInterval.getInterval().getEnd(), + DIRECT + ); } @Override public CompletableFuture> nextIterator() { - if(currentPage == null || frameInterval.getInterval().getEnd().isBefore(currentPage.getStarted()) ){ + if(!pageProvider.hasNext()){ return CompletableFuture.completedFuture(null); } + PageInfo pageInfo = pageProvider.next(); Instant actualStart = frameInterval.getFrameType().getFrameStart(frameInterval.getInterval().getStart()); Instant actualEnd = frameInterval.getFrameType().getFrameEnd(frameInterval.getInterval().getEnd()); @@ -63,8 +80,8 @@ public CompletableFuture> nextIterator() { MessageStatisticsEntityConverter messageStatsConverter = operators.getMessageStatisticsEntityConverter(); return messageStatsOperator.getStatistics( - currentPage.getId().getBookId().getName(), - currentPage.getId().getName(), + pageInfo.getBookName(), + pageInfo.getName(), sessionAlias, direction.getLabel(), frameInterval.getFrameType().getValue(), @@ -72,8 +89,6 @@ public CompletableFuture> nextIterator() { actualEnd, readAttrs) .thenApplyAsync(rs -> { - currentPage = book.getNextPage(currentPage.getStarted()); - PagedIterator pagedIterator = new PagedIterator<>( rs, selectQueryExecutor, diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/testevents/CassandraTestEventFilter.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/testevents/CassandraTestEventFilter.java index aff358abd..81440aaec 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/testevents/CassandraTestEventFilter.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/testevents/CassandraTestEventFilter.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,8 +19,8 @@ import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.core.metadata.schema.ClusteringOrder; import com.datastax.oss.driver.api.querybuilder.select.Select; -import com.datastax.oss.driver.shaded.guava.common.base.Preconditions; import com.exactpro.cradle.Order; +import com.exactpro.cradle.PageId; import com.exactpro.cradle.cassandra.dao.CassandraFilter; import com.exactpro.cradle.cassandra.utils.FilterUtils; import com.exactpro.cradle.filters.ComparisonOperation; @@ -28,12 +28,20 @@ import com.exactpro.cradle.filters.FilterForLess; import com.exactpro.cradle.testevents.StoredTestEventId; +import javax.annotation.Nonnull; import java.time.Instant; import java.util.ArrayList; import java.util.List; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.bindMarker; -import static com.exactpro.cradle.cassandra.dao.testevents.TestEventEntity.*; +import static com.exactpro.cradle.cassandra.dao.testevents.TestEventEntity.FIELD_BOOK; +import static com.exactpro.cradle.cassandra.dao.testevents.TestEventEntity.FIELD_ID; +import static com.exactpro.cradle.cassandra.dao.testevents.TestEventEntity.FIELD_PAGE; +import static com.exactpro.cradle.cassandra.dao.testevents.TestEventEntity.FIELD_PARENT_ID; +import static com.exactpro.cradle.cassandra.dao.testevents.TestEventEntity.FIELD_SCOPE; +import static com.exactpro.cradle.cassandra.dao.testevents.TestEventEntity.FIELD_START_DATE; +import static com.exactpro.cradle.cassandra.dao.testevents.TestEventEntity.FIELD_START_TIME; +import static java.util.Objects.requireNonNull; public class CassandraTestEventFilter implements CassandraFilter { private static final String START_DATE_FROM = "startDateFrom"; @@ -42,7 +50,8 @@ public class CassandraTestEventFilter implements CassandraFilter startTimestampFrom; private final FilterForLess startTimestampTo; private final String parentId; @@ -52,13 +61,12 @@ public class CassandraTestEventFilter implements CassandraFilter startTimestampFrom, FilterForLess startTimestampTo, StoredTestEventId id, String parentId, int limit, Order order) { - this.book = book; - this.page = page; - this.scope = scope; + this.pageId = requireNonNull(pageId, "page id can't be null because book and page names are part of partition"); + this.scope = requireNonNull(scope, "scope can't be null because it is part of partition"); this.startTimestampFrom = startTimestampFrom; this.startTimestampTo = startTimestampTo; this.id = id; @@ -110,8 +118,8 @@ public Select addConditions(Select select) { @Override public BoundStatementBuilder bindParameters(BoundStatementBuilder builder) { builder = builder - .setString(FIELD_BOOK, book) - .setString(FIELD_PAGE, page) + .setString(FIELD_BOOK, pageId.getBookId().getName()) + .setString(FIELD_PAGE, pageId.getName()) .setString(FIELD_SCOPE, scope); if (startTimestampFrom != null) @@ -133,16 +141,19 @@ public BoundStatementBuilder bindParameters(BoundStatementBuilder builder) { return builder; } + public @Nonnull PageId getPageId() { + return pageId; + } - public String getBook() { - return book; + public @Nonnull String getBook() { + return pageId.getBookId().getName(); } - public String getPage() { - return page; + public @Nonnull String getPage() { + return pageId.getName(); } - public String getScope() { + public @Nonnull String getScope() { return scope; } @@ -169,12 +180,8 @@ public Order getOrder() { @Override public String toString() { List result = new ArrayList<>(10); - if (book != null) - result.add("book=" + book); - if (page != null) - result.add("page=" + page); - if (scope != null) - result.add("scope=" + scope); + result.add("pageId=" + pageId); + result.add("scope=" + scope); if (startTimestampFrom != null) result.add("timestampFrom" + startTimestampFrom); if (startTimestampTo != null) diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/testevents/TestEventIteratorProvider.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/testevents/TestEventIteratorProvider.java index d45ff7afa..8cdb0365d 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/testevents/TestEventIteratorProvider.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/testevents/TestEventIteratorProvider.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,8 +18,6 @@ import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.exactpro.cradle.BookInfo; -import com.exactpro.cradle.Order; -import com.exactpro.cradle.PageId; import com.exactpro.cradle.PageInfo; import com.exactpro.cradle.cassandra.EventBatchDurationWorker; import com.exactpro.cradle.cassandra.dao.CassandraOperators; @@ -27,10 +25,8 @@ import com.exactpro.cradle.cassandra.iterators.PagedIterator; import com.exactpro.cradle.cassandra.resultset.IteratorProvider; import com.exactpro.cradle.cassandra.retries.SelectQueryExecutor; -import com.exactpro.cradle.cassandra.utils.FilterUtils; import com.exactpro.cradle.filters.ComparisonOperation; import com.exactpro.cradle.filters.FilterForGreater; -import com.exactpro.cradle.filters.FilterForLess; import com.exactpro.cradle.iterators.ConvertingIterator; import com.exactpro.cradle.iterators.FilteringIterator; import com.exactpro.cradle.iterators.LimitedIterator; @@ -40,6 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.time.Instant; import java.util.Iterator; import java.util.concurrent.CompletableFuture; @@ -47,7 +44,11 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; +import static com.exactpro.cradle.cassandra.utils.FilterUtils.findFirstPage; +import static com.exactpro.cradle.cassandra.utils.FilterUtils.findFirstTimestamp; +import static com.exactpro.cradle.cassandra.utils.FilterUtils.findLastTimestamp; import static com.exactpro.cradle.cassandra.workers.EventsWorker.mapTestEventEntity; +import static java.lang.Math.max; public class TestEventIteratorProvider extends IteratorProvider { /* @@ -58,21 +59,20 @@ public class TestEventIteratorProvider extends IteratorProvider * If the ending page was queried, no more queries will be done, meaning the end of data */ - private static final Logger logger = LoggerFactory.getLogger(TestEventIteratorProvider.class); + private static final Logger LOGGER = LoggerFactory.getLogger(TestEventIteratorProvider.class); private final TestEventOperator op; - private final BookInfo book; - private final ExecutorService composingService; + private final ExecutorService composingService; private final SelectQueryExecutor selectQueryExecutor; private final TestEventEntityConverter entityConverter; - private final PageInfo firstPage, lastPage; private final Function readAttrs; /** limit must be strictly positive ( limit greater than 0 ) */ private final int limit; - private final AtomicInteger returned; - private CassandraTestEventFilter cassandraFilter; - private final EventBatchDurationWorker eventBatchDurationWorker; - private final Instant actualFrom; + private final AtomicInteger returned = new AtomicInteger(); + private final Instant actualFrom; + private final Iterator pageProvider; + private final TestEventFilter filter; + private final FilterForGreater leftBoundFilter; public TestEventIteratorProvider(String requestInfo, TestEventFilter filter, CassandraOperators operators, BookInfo book, ExecutorService composingService, SelectQueryExecutor selectQueryExecutor, @@ -82,63 +82,40 @@ public TestEventIteratorProvider(String requestInfo, TestEventFilter filter, Cas super(requestInfo); this.op = operators.getTestEventOperator(); this.entityConverter = operators.getTestEventEntityConverter(); - this.book = book; - this.composingService = composingService; + this.composingService = composingService; this.selectQueryExecutor = selectQueryExecutor; - - // setup first & last pages according to filter order - PageInfo pageFrom = FilterUtils.findFirstPage(filter.getPageId(), effectiveStartTimestampFrom(filter), book); - PageInfo pageTo = FilterUtils.findLastPage(filter.getPageId(), effectiveStartTimestampTo(filter), book); - Order order = filter.getOrder(); - this.firstPage = (order == Order.DIRECT) ? pageFrom : pageTo; - this.lastPage = (order == Order.DIRECT) ? pageTo : pageFrom; - - this.eventBatchDurationWorker = eventBatchDurationWorker; this.actualFrom = actualFrom; - this.readAttrs = readAttrs; this.limit = filter.getLimit(); - this.returned = new AtomicInteger(); - this.cassandraFilter = createInitialFilter(filter); - } - - - private FilterForGreater effectiveStartTimestampFrom(TestEventFilter filter) { - if (filter.getOrder() == Order.DIRECT && filter.getId() != null) { - FilterForGreater result = new FilterForGreater<>(filter.getId().getStartTimestamp()); - result.setGreaterOrEquals(); - return result; - } - return filter.getStartTimestampFrom(); - } - - - private FilterForLess effectiveStartTimestampTo(TestEventFilter filter) { - if (filter.getOrder() == Order.REVERSE && filter.getId() != null) { - FilterForLess result = new FilterForLess<>(filter.getId().getStartTimestamp()); - result.setLessOrEquals(); - return result; - } - return filter.getStartTimestampTo(); + this.leftBoundFilter = createLeftBoundFilter(book, eventBatchDurationWorker, actualFrom, readAttrs, filter); + this.filter = filter; + + this.pageProvider = book.getPages( + // we must calculate the first timestamp by origin filter because + // left bound filter is calculated according to max duration of event batch on the first page + findFirstTimestamp(filter.getPageId(), filter.getStartTimestampFrom(), book), + findLastTimestamp(filter.getPageId(), filter.getStartTimestampTo(), book), + filter.getOrder() + ); } @Override public CompletableFuture> nextIterator() { - - if (cassandraFilter == null) + if (!pageProvider.hasNext()) { return CompletableFuture.completedFuture(null); + } + PageInfo nextPage = pageProvider.next(); if (limit > 0 && returned.get() >= limit) { - logger.debug("Filtering interrupted because limit for records to return ({}) is reached ({})", limit, returned); + LOGGER.debug("Filtering interrupted because limit for records to return ({}) is reached ({})", limit, returned); return CompletableFuture.completedFuture(null); } + + CassandraTestEventFilter cassandraFilter = createFilter(nextPage, max(limit - returned.get(), 0)); - logger.debug("Getting next iterator for '{}' by filter {}", getRequestInfo(), cassandraFilter); + LOGGER.debug("Getting next iterator for '{}' by filter {}", getRequestInfo(), cassandraFilter); return op.getByFilter(cassandraFilter, selectQueryExecutor, getRequestInfo(), readAttrs) .thenApplyAsync(resultSet -> { - PageId pageId = new PageId(book.getId(), cassandraFilter.getPage()); - cassandraFilter = createNextFilter(cassandraFilter, Math.max(limit - returned.get(),0)); - PagedIterator pagedIterator = new PagedIterator<>( resultSet, selectQueryExecutor, @@ -146,7 +123,7 @@ public CompletableFuture> nextIterator() { getRequestInfo()); ConvertingIterator convertingIterator = new ConvertingIterator<>( pagedIterator, entity -> - mapTestEventEntity(pageId, entity)); + mapTestEventEntity(nextPage.getId(), entity)); FilteringIterator filteringIterator = new FilteringIterator<>( convertingIterator, convertedEntity -> !convertedEntity.getLastStartTimestamp().isBefore(actualFrom)); @@ -156,61 +133,18 @@ public CompletableFuture> nextIterator() { }, composingService); } - - private CassandraTestEventFilter createInitialFilter(TestEventFilter filter) { - /* - Only initial filter needs to be adjusted with max duration, - since `createNextFilter` just passes timestamps from previous to next filters - */ - long duration = eventBatchDurationWorker.getMaxDuration(filter.getBookId().getName(), firstPage.getId().getName(), filter.getScope(), readAttrs); - FilterForGreater newFrom; - - ComparisonOperation operation = filter.getStartTimestampFrom() == null ? ComparisonOperation.GREATER : filter.getStartTimestampFrom().getOperation(); - if (operation.equals(ComparisonOperation.GREATER)) { - newFrom = FilterForGreater.forGreater(actualFrom.minusMillis(duration)); - } else { - newFrom = FilterForGreater.forGreaterOrEquals(actualFrom.minusMillis(duration)); - } - - String parentId = getParentIdString(filter); + private CassandraTestEventFilter createFilter(@Nonnull PageInfo pageInfo, int updatedLimit) { return new CassandraTestEventFilter( - book.getId().getName(), - firstPage.getId().getName(), + pageInfo.getId(), filter.getScope(), - newFrom, + leftBoundFilter, filter.getStartTimestampTo(), filter.getId(), - parentId, - filter.getLimit(), - filter.getOrder()); - } - - private CassandraTestEventFilter createNextFilter(CassandraTestEventFilter prevFilter, Integer updatedLimit) { - - PageInfo prevPage = book.getPage(new PageId(book.getId(), prevFilter.getPage())); - if (prevPage == lastPage) - return null; - - // calculate next page according to filter order - PageInfo nextPage; - if (prevFilter.getOrder() == Order.DIRECT) - nextPage = book.getNextPage(prevPage.getStarted()); - else - nextPage = book.getPreviousPage(prevPage.getStarted()); - - return new CassandraTestEventFilter( - book.getId().getName(), - nextPage.getId().getName(), - prevFilter.getScope(), - prevFilter.getStartTimestampFrom(), - prevFilter.getStartTimestampTo(), - prevFilter.getId(), - prevFilter.getParentId(), + getParentIdString(filter), updatedLimit, - prevFilter.getOrder()); + filter.getOrder()); } - private String getParentIdString(TestEventFilter filter) { if (filter.isRoot()) @@ -222,4 +156,23 @@ private String getParentIdString(TestEventFilter filter) { return null; } + + private static FilterForGreater createLeftBoundFilter(BookInfo bookInfo, EventBatchDurationWorker eventBatchDurationWorker, Instant actualFrom, Function readAttrs, TestEventFilter filter) { + /* + Only initial filter needs to be adjusted with max duration, + since `createNextFilter` just passes timestamps from previous to next filters + */ + PageInfo firstPage = findFirstPage(filter.getPageId(), filter.getStartTimestampFrom(), bookInfo); + long duration = 0; + if (firstPage != null) { + duration = eventBatchDurationWorker.getMaxDuration(filter.getBookId().getName(), firstPage.getName(), filter.getScope(), readAttrs); + } + + ComparisonOperation operation = filter.getStartTimestampFrom() == null ? ComparisonOperation.GREATER : filter.getStartTimestampFrom().getOperation(); + if (operation.equals(ComparisonOperation.GREATER)) { + return FilterForGreater.forGreater(actualFrom.minusMillis(duration)); + } else { + return FilterForGreater.forGreaterOrEquals(actualFrom.minusMillis(duration)); + } + } } diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/resultset/PagesInIntervalIteratorProvider.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/resultset/PagesInIntervalIteratorProvider.java index c095d6f06..641af06d9 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/resultset/PagesInIntervalIteratorProvider.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/resultset/PagesInIntervalIteratorProvider.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Exactpro (Exactpro Systems Limited) + * Copyright 2023-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -73,13 +73,13 @@ private Queue getPagesInInterval(BookId bookId, BookCache bookCache, Int return bookCache.loadPageInfo(bookId, false) .stream() .filter(page -> checkInterval(page, start, end)) - .map(page -> page.getId().getName()) + .map(PageInfo::getName) .collect(Collectors.toCollection(LinkedList::new)); } public static boolean checkInterval(PageInfo page, Instant start, Instant end) { var pageStart = page.getStarted(); - Objects.requireNonNull(pageStart, String.format("Page \"%s\" has null start time", page.getId().getName())); + Objects.requireNonNull(pageStart, String.format("Page \"%s\" has null start time", page.getName())); var pageEnd = defaultIfNull(page.getEnded(), Instant.MAX); return !pageEnd.isBefore(start) && !pageStart.isAfter(end); } diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/resultset/SessionsStatisticsIteratorProvider.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/resultset/SessionsStatisticsIteratorProvider.java deleted file mode 100644 index b6a93a29f..000000000 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/resultset/SessionsStatisticsIteratorProvider.java +++ /dev/null @@ -1,123 +0,0 @@ -package com.exactpro.cradle.cassandra.resultset; - -import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; -import com.exactpro.cradle.BookInfo; -import com.exactpro.cradle.PageInfo; -import com.exactpro.cradle.SessionRecordType; -import com.exactpro.cradle.cassandra.counters.FrameInterval; -import com.exactpro.cradle.cassandra.dao.CassandraOperators; -import com.exactpro.cradle.cassandra.dao.statistics.SessionStatisticsEntity; -import com.exactpro.cradle.cassandra.dao.statistics.SessionStatisticsEntityConverter; -import com.exactpro.cradle.cassandra.dao.statistics.SessionStatisticsOperator; -import com.exactpro.cradle.cassandra.iterators.PagedIterator; -import com.exactpro.cradle.cassandra.retries.SelectQueryExecutor; -import com.exactpro.cradle.cassandra.utils.FilterUtils; -import com.exactpro.cradle.filters.FilterForGreater; -import com.exactpro.cradle.iterators.ConvertingIterator; -import com.exactpro.cradle.iterators.UniqueIterator; - -import java.time.Instant; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.function.Function; - -/** - * Iterator provider for sessions which provides different iterators for - * frameIntervals and pages - */ -public class SessionsStatisticsIteratorProvider extends IteratorProvider{ - - - private final CassandraOperators operators; - private final BookInfo bookInfo; - private final ExecutorService composingService; - private final SelectQueryExecutor selectQueryExecutor; - private final Function readAttrs; - private final List frameIntervals; - private final SessionRecordType recordType; - private Integer frameIndex; - /* - This set will be used during creation of all next iterators - to guarantee that unique elements will be returned - across all iterators - */ - private final Set registry; - private PageInfo curPage; - - public SessionsStatisticsIteratorProvider (String requestInfo, - CassandraOperators operators, - BookInfo bookInfo, - ExecutorService composingService, - SelectQueryExecutor selectQueryExecutor, - Function readAttrs, - List frameIntervals, - SessionRecordType recordType) { - super(requestInfo); - this.operators = operators; - this.bookInfo = bookInfo; - this.composingService = composingService; - this.selectQueryExecutor = selectQueryExecutor; - this.readAttrs = readAttrs; - this.frameIntervals = frameIntervals; - this.recordType = recordType; - - this.registry = new HashSet<>(); - /* - Since intervals are created in strictly increasing, non-overlapping order - the first page is set in regards to first interval - */ - this.frameIndex = 0; - this.curPage = FilterUtils.findFirstPage(null, FilterForGreater.forGreater(frameIntervals.get(frameIndex).getInterval().getStart()), bookInfo); - } - - @Override - public CompletableFuture> nextIterator() { - // All intervals have been processed, there can't be next iterator - if (frameIndex == frameIntervals.size()) { - return CompletableFuture.completedFuture(null); - } - - FrameInterval frameInterval = frameIntervals.get(frameIndex); - - Instant actualStart = frameInterval.getInterval().getStart(); - Instant actualEnd = frameInterval.getInterval().getEnd(); - - SessionStatisticsOperator sessionStatisticsOperator = operators.getSessionStatisticsOperator(); - SessionStatisticsEntityConverter converter = operators.getSessionStatisticsEntityConverter(); - - return sessionStatisticsOperator.getStatistics( - curPage.getId().getBookId().getName(), - curPage.getId().getName(), - recordType.getValue(), - frameInterval.getFrameType().getValue(), - actualStart, - actualEnd, - readAttrs).thenApplyAsync(rs -> { - /* - At this point we need to either update page - or move to new interval - */ - if (curPage.getEnded() != null && curPage.getEnded().isBefore(frameInterval.getInterval().getEnd())) { - // Page finishes sooner than this interval - curPage = bookInfo.getNextPage(curPage.getStarted()); - } else { - // Interval finishes sooner than page - frameIndex ++; - } - - PagedIterator pagedIterator = new PagedIterator<>(rs, - selectQueryExecutor, - converter::getEntity, - getRequestInfo()); - ConvertingIterator convertingIterator = new ConvertingIterator<>( - pagedIterator, - SessionStatisticsEntity::getSession); - - return new UniqueIterator<>(convertingIterator, registry); - }, composingService); - } -} diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/utils/FilterUtils.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/utils/FilterUtils.java index 0e5c95f0b..e1122b611 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/utils/FilterUtils.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/utils/FilterUtils.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,6 +40,9 @@ import com.exactpro.cradle.filters.FilterForLess; import com.exactpro.cradle.utils.TimeUtils; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + public class FilterUtils { /** @@ -195,8 +198,7 @@ public static FilterForLess filterTimeTo(FilterForLess filte ? FilterForLess.forLess(ldt.toLocalTime()) : FilterForLess.forLessOrEquals(ldt.toLocalTime()); } - - + /** * Finds page to start filtering from * @param pageId specified in filter. Can be null @@ -242,4 +244,87 @@ public static PageInfo findLastPage(PageId pageId, FilterForLess timest } return book.getLastPage(); } + + /** + * Finds timestamp to start filtering from + * @param pageId specified in filter. Can be null + * @param timestampFrom specified in filter. Can be null + * @param book to filter data from + * @return timestamp to start filtering from. + * If pageId is specified, it will be start timestamp of that page. + * else if timestampFrom is specified, it will be return. + * else if first page of the book exist, it will be return. + * else null + */ + public static @Nullable Instant findFirstTimestamp(@Nullable PageId pageId, + @Nullable Instant timestampFrom, + @Nonnull BookInfo book) { + if (pageId != null) { + return pageId.getStart(); + } + if (timestampFrom != null) { + return timestampFrom; + } + PageInfo firstPage = book.getFirstPage(); + return firstPage == null ? null : firstPage.getStarted(); + } + + /** + * Finds timestamp to start filtering from + * @param pageId specified in filter. Can be null + * @param timestampFrom specified in filter. Can be null + * @param book to filter data from + * @return timestamp to start filtering from. + * If pageId is specified, it will be start timestamp of that page. + * else if timestampFrom is specified, it will be return. + * else if first page of the book exist, it will be return. + * else null + */ + public static @Nullable Instant findFirstTimestamp(@Nullable PageId pageId, + @Nullable FilterForGreater timestampFrom, + @Nonnull BookInfo book) { + return findFirstTimestamp(pageId, timestampFrom == null ? null : timestampFrom.getValue(), book); + } + + /** + * Finds timestamp to end filtering at + * @param pageId specified in filter. Can be null + * @param timestampTo specified in filter. Can be null + * @param book to filter data from + * @return page to end filtering at. + * If pageId is specified, it will be end timestamp of that page. + * else if timestampTo is specified, it will be return. + * else if last page of the book exist, it will be return. + * else null + */ + public static @Nullable Instant findLastTimestamp(@Nullable PageId pageId, + @Nullable Instant timestampTo, + @Nonnull BookInfo book) + { + if (pageId != null) { + return book.getPage(pageId).getEnded(); + } + if (timestampTo != null) { + return timestampTo; + } + PageInfo lastPage = book.getLastPage(); + return lastPage == null ? null : lastPage.getEnded(); + } + + /** + * Finds timestamp to end filtering at + * @param pageId specified in filter. Can be null + * @param timestampTo specified in filter. Can be null + * @param book to filter data from + * @return page to end filtering at. + * If pageId is specified, it will be start timestamp of that page. + * else if timestampTo is specified, it will be return. + * else if last page of the book exist, it will be return. + * else null + */ + public static @Nullable Instant findLastTimestamp(@Nullable PageId pageId, + @Nullable FilterForLess timestampTo, + @Nonnull BookInfo book) { + return findLastTimestamp(pageId, timestampTo == null ? null : timestampTo.getValue(), book); + } } diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/utils/StorageUtils.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/utils/StorageUtils.java index af782f1be..a16848151 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/utils/StorageUtils.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/utils/StorageUtils.java @@ -1,15 +1,43 @@ +/* + * Copyright 2022-2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.exactpro.cradle.cassandra.utils; import com.exactpro.cradle.FrameType; import com.exactpro.cradle.cassandra.counters.FrameInterval; import com.exactpro.cradle.counters.Interval; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; import java.util.ArrayList; import java.util.List; +import static com.exactpro.cradle.CradleStorage.TIMEZONE_OFFSET; +import static com.exactpro.cradle.cassandra.CassandraStorageSettings.MAX_EPOCH_INSTANT; +import static com.exactpro.cradle.cassandra.CassandraStorageSettings.MIN_EPOCH_INSTANT; + public class StorageUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(StorageUtils.class); + + private StorageUtils() { } + public static List sliceInterval (Interval interval) { List slices = new ArrayList<>(); @@ -92,4 +120,40 @@ public static List sliceInterval (Interval interval) { return slices; } + + public static LocalDate toLocalDate(Instant instant) { + if (instant.isBefore(MIN_EPOCH_INSTANT)) { + LOGGER.trace("Replaces {} by {}", instant, MIN_EPOCH_INSTANT); + return LocalDate.MIN; + } + if (instant.isAfter(MAX_EPOCH_INSTANT)) { + LOGGER.trace("Replaces {} by {}", instant, MAX_EPOCH_INSTANT); + return LocalDate.MAX; + } + return LocalDate.ofInstant(instant, TIMEZONE_OFFSET); + } + + public static LocalTime toLocalTime(Instant instant) { + if (instant.isBefore(MIN_EPOCH_INSTANT)) { + LOGGER.trace("Replaces {} by {}", instant, MIN_EPOCH_INSTANT); + return LocalTime.MIN; + } + if (instant.isAfter(MAX_EPOCH_INSTANT)) { + LOGGER.trace("Replaces {} by {}", instant, MAX_EPOCH_INSTANT); + return LocalTime.MAX; + } + return LocalTime.ofInstant(instant, TIMEZONE_OFFSET); + } + + public static LocalDateTime toLocalDateTime(Instant instant) { + if (instant.isBefore(MIN_EPOCH_INSTANT)) { + LOGGER.trace("Replaces {} by {}", instant, MIN_EPOCH_INSTANT); + return LocalDateTime.MIN; + } + if (instant.isAfter(MAX_EPOCH_INSTANT)) { + LOGGER.trace("Replaces {} by {}", instant, MAX_EPOCH_INSTANT); + return LocalDateTime.MAX; + } + return LocalDateTime.ofInstant(instant, TIMEZONE_OFFSET); + } } diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/workers/MessagesWorker.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/workers/MessagesWorker.java index e234aaf12..bb67a39ff 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/workers/MessagesWorker.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/workers/MessagesWorker.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -68,6 +68,7 @@ import java.time.LocalTime; import java.util.EnumMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -81,6 +82,8 @@ import java.util.zip.DataFormatException; import static com.exactpro.cradle.CradleStorage.EMPTY_MESSAGE_INDEX; +import static com.exactpro.cradle.Order.DIRECT; +import static com.exactpro.cradle.Order.REVERSE; import static com.exactpro.cradle.cassandra.dao.messages.MessageBatchEntity.FIELD_FIRST_MESSAGE_TIME; import static com.exactpro.cradle.cassandra.dao.messages.MessageBatchEntity.FIELD_LAST_SEQUENCE; import static com.exactpro.cradle.cassandra.dao.messages.MessageBatchEntity.FIELD_SEQUENCE; @@ -242,9 +245,9 @@ private CompletableFuture getNearestTimeAndSequenceBefore( LocalTime messageTime, long sequence, Function readAttrs ) { String queryInfo = format("get nearest time and sequence before %s for page '%s'", - TimeUtils.toInstant(messageDate, messageTime), page.getId().getName()); + TimeUtils.toInstant(messageDate, messageTime), page.getName()); return selectQueryExecutor.executeSingleRowResultQuery( - () -> mbOperator.getNearestBatchTimeAndSequenceBefore(page.getId().getBookId().getName(), page.getId().getName(), sessionAlias, + () -> mbOperator.getNearestBatchTimeAndSequenceBefore(page.getBookName(), page.getName(), sessionAlias, direction, messageDate, messageTime, sequence, readAttrs), Function.identity(), queryInfo) .thenComposeAsync(row -> { @@ -536,11 +539,12 @@ private CompletableFuture>> storePageSessions(BookId public long getBoundarySequence(String sessionAlias, Direction direction, BookInfo book, boolean first) throws CradleStorageException { MessageBatchOperator mbOp = getOperators().getMessageBatchOperator(); - PageInfo currentPage = first ? book.getFirstPage() : book.getLastPage(); + Iterator pageIterator = book.getPages(null, null, first ? DIRECT : REVERSE); + PageInfo currentPage = pageIterator.hasNext() ? pageIterator.next() : null; Row row = null; while (row == null && currentPage != null) { - String page = currentPage.getId().getName(); + String page = currentPage.getName(); String bookName = book.getId().getName(); String queryInfo = format("get %s sequence for book '%s' page '%s', session alias '%s', " + "direction '%s'", (first ? "first" : "last"), bookName, page, sessionAlias, direction); @@ -553,9 +557,9 @@ public long getBoundarySequence(String sessionAlias, Direction direction, BookIn throw new CradleStorageException("Error occurs while " + queryInfo, e); } - if (row == null) - currentPage = first ? book.getNextPage(currentPage.getStarted()) - : book.getPreviousPage(currentPage.getStarted()); + if (row == null) { + currentPage = pageIterator.hasNext() ? pageIterator.next() : null; + } } if (row == null) { logger.debug("There is no messages yet in book '{}' with session alias '{}' and direction '{}'", diff --git a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/ReadThroughBookCacheTest.java b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/ReadThroughBookCacheTest.java index 6f5570c31..a4a2ee191 100644 --- a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/ReadThroughBookCacheTest.java +++ b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/ReadThroughBookCacheTest.java @@ -34,11 +34,14 @@ import java.time.LocalTime; import java.time.ZoneId; import java.time.ZoneOffset; +import java.util.ArrayList; import java.util.List; import java.util.function.Function; import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_PAGE_REMOVE_TIME; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; @@ -59,7 +62,7 @@ public class ReadThroughBookCacheTest { private final BookOperator bookOperator = mock(BookOperator.class); private final CassandraOperators operators = mock(CassandraOperators.class); private final Function readAttrs = mock(Function.class); - private final ReadThroughBookCache cache = new ReadThroughBookCache(operators, readAttrs, schemaVersion); + private final ReadThroughBookCache cache = new ReadThroughBookCache(operators, readAttrs, schemaVersion, 1, Long.MAX_VALUE); List previousFormatPages = List.of( new PageEntity(bookId.getName(), @@ -107,6 +110,25 @@ public void beforeMethod() { doReturn(pageOperator).when(operators).getPageOperator(); doReturn(bookOperator).when(operators).getBookOperator(); doReturn(pagingIterable).when(pageOperator).getAll(same(bookId.getName()), same(readAttrs)); + doAnswer(invocation -> { + LocalDate startDate = invocation.getArgument(1); + LocalTime startTime = invocation.getArgument(2); + LocalDate endDate = invocation.getArgument(3); + LocalTime endTime = invocation.getArgument(4); + + List result = new ArrayList<>(); + for (PageEntity pageEntity : pagingIterable) { + if ((startDate == null || !startDate.isAfter(pageEntity.getStartDate())) && + (startTime == null || !startTime.isAfter(pageEntity.getStartTime())) && + (endDate == null || pageEntity.getEndDate() == null || !endDate.isBefore(pageEntity.getEndDate())) && + (endTime == null || pageEntity.getEndTime() == null || !endTime.isBefore(pageEntity.getEndTime()))) { + result.add(pageEntity); + } + } + PagingIterable iterable = mock(PagingIterable.class); + doReturn(result.iterator()).when(iterable).iterator(); + return iterable; + }).when(pageOperator).get(same(bookId.getName()), any(), any(), any(), any(), same(readAttrs)); doReturn(bookEntity).when(bookOperator).get(same(bookId.getName()), same(readAttrs)); } @@ -128,7 +150,7 @@ public void testLoadPageInfoNewFormat(boolean loadRemoved) { public void testGetBookPreviousFormat() throws BookNotFoundException { doReturn(previousFormatPages.iterator()).when(pagingIterable).iterator(); - assertEquals(cache.getBook(bookId).getId(), bookId); + assertEquals(cache. getBook(bookId).getId(), bookId); } @Test diff --git a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/dao/messages/GroupedMessageBatchEntityTest.java b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/dao/messages/GroupedMessageBatchEntityTest.java index 00c17b9dc..394af0214 100644 --- a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/dao/messages/GroupedMessageBatchEntityTest.java +++ b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/dao/messages/GroupedMessageBatchEntityTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2022-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2022-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,7 +41,8 @@ public class GroupedMessageBatchEntityTest { @Test public void validationTest() throws IOException, DataFormatException, CradleStorageException, CompressException { - PageId pageId = new PageId(new BookId("Test_Book_1"), "Test_Page_1"); + Instant timestamp = Instant.parse("2022-06-10T23:59:58.987Z"); + PageId pageId = new PageId(new BookId("Test_Book_1"), timestamp, "Test_Page_1"); String group = "test-group"; GroupedMessageBatchToStore batch = new GroupedMessageBatchToStore(group, 10_000, new CoreStorageSettings().calculateStoreActionRejectionThreshold()); @@ -49,7 +50,7 @@ public void validationTest() throws IOException, DataFormatException, CradleStor .bookId(pageId.getBookId()) .sessionAlias("TEST_Session") .direction(Direction.FIRST) - .timestamp(Instant.parse("2022-06-10T23:59:58.987Z")) + .timestamp(timestamp) .sequence(1) .content(createContent(40)) .metadata("key_test", "value_test") diff --git a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/dao/messages/MessageBatchEntityTest.java b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/dao/messages/MessageBatchEntityTest.java index f49c07701..9f2408028 100644 --- a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/dao/messages/MessageBatchEntityTest.java +++ b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/dao/messages/MessageBatchEntityTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,7 +42,7 @@ public class MessageBatchEntityTest { @Test public void messageEntity() throws IOException, DataFormatException, CradleStorageException, CompressException { - PageId pageId = new PageId(new BookId("Test_Book_1"), "Test_Page_1"); + PageId pageId = new PageId(new BookId("Test_Book_1"), Instant.now(), "Test_Page_1"); MessageBatchToStore batch = MessageBatchToStore.singleton(MessageToStore.builder() .bookId(pageId.getBookId()) .sessionAlias("TEST_Session") diff --git a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/dao/messages/MessagesIteratorProviderTest.java b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/dao/messages/MessagesIteratorProviderTest.java index 78e1baec6..fb110ad10 100644 --- a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/dao/messages/MessagesIteratorProviderTest.java +++ b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/dao/messages/MessagesIteratorProviderTest.java @@ -40,7 +40,15 @@ public void createIteratorProviderWithEmptyBook() throws CradleStorageException "", messageFilter, operators, - new BookInfo(DEFAULT_BOOK_ID, "book_name", "", Instant.now(), Collections.emptyList()), + new BookInfo(DEFAULT_BOOK_ID, + "book_name", + "", + Instant.now(), + 1, + Long.MAX_VALUE, + (bookId, start, end) -> Collections.emptyList(), + bookId -> null, + bookId -> null), null, null, null diff --git a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/dao/testevents/TestEventEntityTest.java b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/dao/testevents/TestEventEntityTest.java index af7b1c52f..fd0ae408f 100644 --- a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/dao/testevents/TestEventEntityTest.java +++ b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/dao/testevents/TestEventEntityTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,7 +48,7 @@ public class TestEventEntityTest { private final BookId book = new BookId("Book1"); - private final PageId page = new PageId(book, "Page1"); + private final PageId page = new PageId(book, Instant.now(), "Page1"); private final String scope = "Scope1"; private final Instant startTimestamp = Instant.now(); diff --git a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/BaseCradleCassandraTest.java b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/BaseCradleCassandraTest.java index 384591eb9..b7946315c 100644 --- a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/BaseCradleCassandraTest.java +++ b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/BaseCradleCassandraTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,14 +22,18 @@ import com.exactpro.cradle.messages.MessageToStore; import com.exactpro.cradle.testevents.*; import com.exactpro.cradle.utils.CradleStorageException; +import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.UUID; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -52,28 +56,22 @@ public abstract class BaseCradleCassandraTest { private static final List DEFAULT_PAGES = List.of( new PageInfo( - new PageId(DEFAULT_BOOK_ID, DEFAULT_PAGE_PREFIX + 0), - DEFAULT_DATA_START, + new PageId(DEFAULT_BOOK_ID, DEFAULT_DATA_START, DEFAULT_PAGE_PREFIX + 0), DEFAULT_DATA_START.plus(10, ChronoUnit.MINUTES), ""), new PageInfo( - new PageId(DEFAULT_BOOK_ID, DEFAULT_PAGE_PREFIX + 1), - DEFAULT_DATA_START.plus(10, ChronoUnit.MINUTES), + new PageId(DEFAULT_BOOK_ID, DEFAULT_DATA_START.plus(10, ChronoUnit.MINUTES), DEFAULT_PAGE_PREFIX + 1), DEFAULT_DATA_START.plus(20, ChronoUnit.MINUTES), ""), new PageInfo( - new PageId(DEFAULT_BOOK_ID, DEFAULT_PAGE_PREFIX + 2), - DEFAULT_DATA_START.plus(20, ChronoUnit.MINUTES), + new PageId(DEFAULT_BOOK_ID, DEFAULT_DATA_START.plus(20, ChronoUnit.MINUTES), DEFAULT_PAGE_PREFIX + 2), DEFAULT_DATA_START.plus(30, ChronoUnit.MINUTES), ""), new PageInfo( - new PageId(DEFAULT_BOOK_ID, DEFAULT_PAGE_PREFIX + 3), - DEFAULT_DATA_START.plus(30, ChronoUnit.MINUTES), + new PageId(DEFAULT_BOOK_ID, DEFAULT_DATA_START.plus(30, ChronoUnit.MINUTES), DEFAULT_PAGE_PREFIX + 3), DEFAULT_DATA_START.plus(40, ChronoUnit.MINUTES), ""), new PageInfo( - new PageId(DEFAULT_BOOK_ID, DEFAULT_PAGE_PREFIX + 4), - DEFAULT_DATA_START.plus(40, ChronoUnit.MINUTES), + new PageId(DEFAULT_BOOK_ID, DEFAULT_DATA_START.plus(40, ChronoUnit.MINUTES), DEFAULT_PAGE_PREFIX + 4), DEFAULT_DATA_START.plus(50, ChronoUnit.MINUTES), ""), new PageInfo( - new PageId(DEFAULT_BOOK_ID, DEFAULT_PAGE_PREFIX + 5), - DEFAULT_DATA_START.plus(50, ChronoUnit.MINUTES), + new PageId(DEFAULT_BOOK_ID, DEFAULT_DATA_START.plus(50, ChronoUnit.MINUTES), DEFAULT_PAGE_PREFIX + 5), DEFAULT_DATA_START.plus(60, ChronoUnit.MINUTES), "")); @@ -81,12 +79,11 @@ public abstract class BaseCradleCassandraTest { protected CqlSession session; protected CassandraCradleStorage storage; protected Instant dataStart = DEFAULT_DATA_START; - protected Instant dataEnd = DEFAULT_DATA_END; protected BookId bookId = DEFAULT_BOOK_ID; /* Following method should be used in beforeClass if extending class - wants to implement it's own logic of initializing books and pages + wants to implement its own logic of initializing books and pages */ protected void startUp() throws IOException, InterruptedException, CradleStorageException { startUp(false); @@ -113,7 +110,7 @@ protected void startUp(boolean generateBookPages) throws IOException, Interrupte bookId, DEFAULT_PAGES.stream().map( el -> new PageToAdd( - el.getId().getName(), + el.getName(), el.getStarted(), el.getComment())).collect(Collectors.toList())); } @@ -167,4 +164,12 @@ protected TestEventToStore generateTestEvent (String scope, Instant start, long return batch; } + @NotNull + protected static Map toMap(Collection result) { + return result.stream() + .collect(Collectors.toUnmodifiableMap( + PageInfo::getId, + Function.identity() + )); + } } diff --git a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/BookWithoutPageTest.java b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/BookWithoutPageTest.java new file mode 100644 index 000000000..fc0a04255 --- /dev/null +++ b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/BookWithoutPageTest.java @@ -0,0 +1,229 @@ +/* + * Copyright 2023-2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.cradle.cassandra.integration; + +import com.exactpro.cradle.BookToAdd; +import com.exactpro.cradle.Direction; +import com.exactpro.cradle.counters.Interval; +import com.exactpro.cradle.messages.GroupedMessageFilter; +import com.exactpro.cradle.messages.GroupedMessageFilterBuilder; +import com.exactpro.cradle.messages.MessageFilter; +import com.exactpro.cradle.messages.MessageFilterBuilder; +import com.exactpro.cradle.testevents.TestEventFilter; +import com.exactpro.cradle.testevents.TestEventFilterBuilder; +import com.exactpro.cradle.utils.CradleStorageException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.time.Instant; + +import static com.google.common.collect.Lists.newArrayList; +import static java.util.Collections.emptyList; +import static org.testng.Assert.assertEquals; + +public class BookWithoutPageTest extends BaseCradleCassandraTest { + private static final Logger LOGGER = LoggerFactory.getLogger(BookWithoutPageTest.class); + + @BeforeClass + public void startUp() throws IOException, InterruptedException, CradleStorageException { + super.startUp(false); + generateData(); + } + + @Override + protected void generateData() throws CradleStorageException, IOException { + try { + storage.addBook(new BookToAdd(bookId.getName())); + } catch (CradleStorageException | IOException e) { + LOGGER.error("Error while generating data:", e); + throw e; + } + } + + @Test + public void testGetAllPages() throws CradleStorageException { + try { + assertEquals(storage.getAllPages(bookId), emptyList()); + } catch (CradleStorageException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test + public void testGetMessages() throws CradleStorageException, IOException { + try { + MessageFilter filter = new MessageFilterBuilder() + .bookId(bookId) + .sessionAlias("test-session-alias") + .direction(Direction.FIRST) + .timestampFrom().isGreaterThan(Instant.MIN) + .timestampTo().isLessThan(Instant.MAX) + .build(); + assertEquals(newArrayList(storage.getMessages(filter)), emptyList()); + } catch (CradleStorageException | IOException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test + public void testGetMessageBatches() throws CradleStorageException, IOException { + try { + MessageFilter filter = new MessageFilterBuilder() + .bookId(bookId) + .sessionAlias("test-session-alias") + .direction(Direction.FIRST) + .timestampFrom().isGreaterThan(Instant.MIN) + .timestampTo().isLessThan(Instant.MAX) + .build(); + assertEquals(newArrayList(storage.getMessageBatches(filter)), emptyList()); + } catch (CradleStorageException | IOException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test + public void testGetGroupedMessageBatches() throws CradleStorageException, IOException { + try { + GroupedMessageFilter filter = new GroupedMessageFilterBuilder() + .bookId(bookId) + .groupName("test-group") + .timestampFrom().isGreaterThan(Instant.MIN) + .timestampTo().isLessThan(Instant.MAX) + .build(); + assertEquals(newArrayList(storage.getGroupedMessageBatches(filter)), emptyList()); + } catch (CradleStorageException | IOException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test + public void testGetLastSequence() throws CradleStorageException, IOException { + try { + assertEquals(storage.getLastSequence("test-session-alias", Direction.FIRST, bookId), -1L); + } catch (CradleStorageException | IOException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test + public void testGetFirstSequence() throws CradleStorageException, IOException { + try { + assertEquals(storage.getFirstSequence("test-session-alias", Direction.FIRST, bookId), -1L); + } catch (CradleStorageException | IOException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test + public void testGetSessionAliases1() throws CradleStorageException, IOException { + try { + assertEquals(storage.getSessionAliases(bookId), emptyList()); + } catch (CradleStorageException | IOException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test + public void testGetSessionAliases2() throws CradleStorageException { + try { + assertEquals(newArrayList(storage.getSessionAliases(bookId, new Interval(Instant.MIN, Instant.MAX))), + emptyList()); + } catch (CradleStorageException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test + public void testGetGroups() throws CradleStorageException, IOException { + try { + assertEquals(storage.getGroups(bookId), emptyList()); + } catch (CradleStorageException | IOException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test + public void testGetTestEvents() throws CradleStorageException, IOException { + try { + TestEventFilter filter = new TestEventFilterBuilder() + .bookId(bookId) + .scope("test-scope") + .timestampFrom().isGreaterThan(Instant.MIN) + .timestampTo().isLessThan(Instant.MAX) + .build(); + assertEquals(newArrayList(storage.getTestEvents(filter)), emptyList()); + } catch (CradleStorageException | IOException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test + public void testGetScopes1() throws CradleStorageException, IOException { + try { + assertEquals(storage.getScopes(bookId), emptyList()); + } catch (CradleStorageException | IOException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test + public void testGetScopes2() throws CradleStorageException { + try { + assertEquals(newArrayList(storage.getScopes(bookId, new Interval(Instant.MIN, Instant.MAX))), + emptyList()); + } catch (CradleStorageException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test + public void testGetSessionGroups() throws CradleStorageException { + try { + assertEquals(newArrayList(storage.getSessionGroups(bookId, new Interval(Instant.MIN, Instant.MAX))), + emptyList()); + } catch (CradleStorageException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test + public void testGetPages() throws CradleStorageException { + try { + assertEquals(newArrayList(storage.getPages(bookId, new Interval(Instant.MIN, Instant.MAX))), + emptyList()); + } catch (CradleStorageException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } +} diff --git a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/CassandraCradleHelper.java b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/CassandraCradleHelper.java index c418cdfeb..641ee60d4 100644 --- a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/CassandraCradleHelper.java +++ b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/CassandraCradleHelper.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,19 +33,20 @@ public class CassandraCradleHelper { - private static final Logger logger = LoggerFactory.getLogger(CassandraCradleHelper.class); + private static final Logger LOGGER = LoggerFactory.getLogger(CassandraCradleHelper.class); - private static String CASSANDRA_IMAGE = "cassandra:3.11.13"; - public static String LOCAL_DATACENTER_NAME = "datacenter1"; - public static String KEYSPACE_NAME = "test_keyspace"; - public static int TIMEOUT = 5000; - public static int RESULT_PAGE_SIZE = 5; - public static int PERSISTENCE_INTERVAL = 0; + private static final String CASSANDRA_IMAGE = "cassandra:3.11.13"; + public static final String LOCAL_DATACENTER_NAME = "datacenter1"; + public static final long BOOK_REFRESH_INTERVAL_MILLIS = 60_000; + public static final String KEYSPACE_NAME = "test_keyspace"; + public static final int TIMEOUT = 5000; + public static final int RESULT_PAGE_SIZE = 5; + public static final int PERSISTENCE_INTERVAL = 0; private CqlSession session; private CassandraConnectionSettings connectionSettings; private CassandraCradleStorage storage; - private CassandraStorageSettings storageSettings; + protected CassandraStorageSettings storageSettings; private static CassandraContainer cassandra; @@ -84,7 +85,7 @@ private void setUpEmbeddedCassandra() { cassandra.getFirstMappedPort(), LOCAL_DATACENTER_NAME); } catch (Exception e) { - logger.error(e.getMessage(), e); + LOGGER.error(e.getMessage(), e); throw e; } } @@ -98,11 +99,13 @@ private void setUpCradle() { storageSettings.setResultPageSize(RESULT_PAGE_SIZE); storageSettings.setKeyspace(KEYSPACE_NAME); storageSettings.setCounterPersistenceInterval(PERSISTENCE_INTERVAL); + storageSettings.setBookRefreshIntervalMillis(BOOK_REFRESH_INTERVAL_MILLIS); + CassandraCradleManager manager = new CassandraCradleManager(connectionSettings, storageSettings, true); storage = (CassandraCradleStorage) manager.getStorage(); } catch (CradleStorageException | IOException e) { - logger.error(e.getMessage(), e); + LOGGER.error(e.getMessage(), e); } } diff --git a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/messages/GroupedMessageIteratorProviderTest.java b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/messages/GroupedMessageIteratorProviderTest.java index b38bbd818..25385ddab 100644 --- a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/messages/GroupedMessageIteratorProviderTest.java +++ b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/messages/GroupedMessageIteratorProviderTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,9 @@ package com.exactpro.cradle.cassandra.integration.messages; -import com.exactpro.cradle.*; +import com.exactpro.cradle.CoreStorageSettings; +import com.exactpro.cradle.Direction; +import com.exactpro.cradle.Order; import com.exactpro.cradle.cassandra.dao.messages.GroupedMessageIteratorProvider; import com.exactpro.cradle.cassandra.integration.BaseCradleCassandraTest; import com.exactpro.cradle.cassandra.integration.CassandraCradleHelper; @@ -26,9 +28,10 @@ import com.exactpro.cradle.cassandra.resultset.CassandraCradleResultSet; import com.exactpro.cradle.cassandra.retries.SelectQueryExecutor; import com.exactpro.cradle.filters.FilterForGreater; -import com.exactpro.cradle.messages.*; +import com.exactpro.cradle.messages.GroupedMessageBatchToStore; +import com.exactpro.cradle.messages.GroupedMessageFilter; +import com.exactpro.cradle.messages.StoredGroupedMessageBatch; import com.exactpro.cradle.utils.CradleStorageException; -import org.assertj.core.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.BeforeClass; @@ -42,6 +45,9 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; + public class GroupedMessageIteratorProviderTest extends BaseCradleCassandraTest { private static final Logger logger = LoggerFactory.getLogger(GroupedMessageIteratorProviderTest.class); @@ -51,10 +57,9 @@ public class GroupedMessageIteratorProviderTest extends BaseCradleCassandraTest private final long storeActionRejectionThreshold = new CoreStorageSettings().calculateStoreActionRejectionThreshold(); - private List data; private List storedData; private CassandraOperators operators; - private ExecutorService composingService = Executors.newSingleThreadExecutor(); + private final ExecutorService composingService = Executors.newFixedThreadPool(3); @BeforeClass public void startUp () throws IOException, InterruptedException, CradleStorageException { @@ -64,7 +69,7 @@ public void startUp () throws IOException, InterruptedException, CradleStorageEx generateData(); } - private void setUpOperators() throws IOException, InterruptedException { + private void setUpOperators() { CassandraDataMapper dataMapper = new CassandraDataMapperBuilder(session).build(); operators = new CassandraOperators(dataMapper, CassandraCradleHelper.getInstance().getStorageSettings()); } @@ -84,7 +89,7 @@ protected void generateData () { b3.addMessage(generateMessage(FIRST_SESSION_ALIAS, Direction.FIRST, 25, 5L)); b3.addMessage(generateMessage(SECOND_SESSION_ALIAS, Direction.SECOND, 25, 6L)); - data = List.of(b1, b2, b3); + List data = List.of(b1, b2, b3); storedData = List.of( MessageTestUtils.groupedMessageBatchToStored(pages.get(0).getId(), null, b1), MessageTestUtils.groupedMessageBatchToStored(pages.get(1).getId(), null, b2), @@ -128,7 +133,7 @@ public void getAllGroupedMessagesTest () throws ExecutionException, InterruptedE Iterable actual = rsFuture.get().asIterable(); List expected = storedData; - Assertions.assertThat(actual) + assertThat(actual) .usingElementComparatorIgnoringFields("recDate") .isEqualTo(expected); } catch (InterruptedException | ExecutionException e) { @@ -150,7 +155,7 @@ public void getFirstTwoGroupedMessagesTest () throws ExecutionException, Interru Iterable actual = rsFuture.get().asIterable(); List expected = storedData.subList(0, 2); - Assertions.assertThat(actual) + assertThat(actual) .usingElementComparatorIgnoringFields("recDate") .isEqualTo(expected); } catch (InterruptedException | ExecutionException e) { @@ -172,7 +177,7 @@ public void getGroupedMessagesAfterSecondPageTest () throws ExecutionException, Iterable actual = rsFuture.get().asIterable(); List expected = storedData.subList(1, 3); - Assertions.assertThat(actual) + assertThat(actual) .usingElementComparatorIgnoringFields("recDate") .isEqualTo(expected); } catch (InterruptedException | ExecutionException e) { @@ -183,21 +188,8 @@ public void getGroupedMessagesAfterSecondPageTest () throws ExecutionException, @Test(description = "Tries to get grouped messages by filter which has negative limit, should end with exception") public void tryToGetGroupedMessagesWithNegativeLimitTest () { - - try { - GroupedMessageFilter groupedMessageFilter = new GroupedMessageFilter(bookId, GROUP_NAME); - groupedMessageFilter.setLimit(-1); - GroupedMessageIteratorProvider iteratorProvider = createIteratorProvider(groupedMessageFilter); - - CompletableFuture> rsFuture = iteratorProvider.nextIterator() - .thenApplyAsync(r -> new CassandraCradleResultSet<>(r, iteratorProvider), composingService); - - Iterable actual = rsFuture.get().asIterable(); - - Assertions.fail("Exception wasn't thrown while getting messages with negative limit"); - } catch (Exception e) { - // Test passed - } + Throwable throwable = catchThrowable(() -> new GroupedMessageFilter(bookId, GROUP_NAME).setLimit(-1)); + assertThat(throwable).hasMessage("Invalid limit value: -1. limit must be greater than 0"); } @Test(description = "Gets grouped messages from iterator provider starting with second page and limit 1") @@ -214,7 +206,7 @@ public void getGroupedMessagesAfterSecondPageWithLimitTest () throws ExecutionEx Iterable actual = rsFuture.get().asIterable(); List expected = storedData.subList(1, 2); - Assertions.assertThat(actual) + assertThat(actual) .usingElementComparatorIgnoringFields("recDate") .isEqualTo(expected); } catch (InterruptedException | ExecutionException e) { @@ -237,7 +229,7 @@ public void getGroupedMessagesFromSecondPage () throws ExecutionException, Inter Iterable actual = rsFuture.get().asIterable(); List expected = storedData.subList(1, 2); - Assertions.assertThat(actual) + assertThat(actual) .usingElementComparatorIgnoringFields("recDate") .isEqualTo(expected); } catch (InterruptedException | ExecutionException e) { @@ -260,7 +252,7 @@ public void getGroupedMessagesFromEmptyPage () throws ExecutionException, Interr Iterable actual = rsFuture.get().asIterable(); List expected = Collections.emptyList(); - Assertions.assertThat(actual) + assertThat(actual) .usingElementComparatorIgnoringFields("recDate") .isEqualTo(expected); } catch (InterruptedException | ExecutionException e) { diff --git a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/messages/MessageBatchIteratorProviderTest.java b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/messages/MessageBatchIteratorProviderTest.java index ed05455a2..f3458949e 100644 --- a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/messages/MessageBatchIteratorProviderTest.java +++ b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/messages/MessageBatchIteratorProviderTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -51,10 +51,9 @@ public class MessageBatchIteratorProviderTest extends BaseCradleCassandraTest { private final long storeActionRejectionThreshold = new CoreStorageSettings().calculateStoreActionRejectionThreshold(); - private List data; private Map> storedData; private CassandraOperators operators; - private ExecutorService composingService = Executors.newSingleThreadExecutor(); + private final ExecutorService composingService = Executors.newFixedThreadPool(3); @BeforeClass public void startUp () throws IOException, InterruptedException, CradleStorageException { @@ -68,11 +67,6 @@ private static class StoredMessageKey { private final String sessionAlias; private final Direction direction; - public StoredMessageKey (StoredMessage message) { - this.sessionAlias = message.getSessionAlias(); - this.direction = message.getDirection(); - } - public StoredMessageKey (String sessionAlias, Direction direction) { this.sessionAlias = sessionAlias; this.direction = direction; @@ -100,7 +94,7 @@ public int hashCode() { } } - private void setUpOperators() throws IOException, InterruptedException { + private void setUpOperators() { CassandraDataMapper dataMapper = new CassandraDataMapperBuilder(session).build(); operators = new CassandraOperators(dataMapper, CassandraCradleHelper.getInstance().getStorageSettings()); } @@ -130,7 +124,7 @@ protected void generateData () { b3.addMessage(generateMessage(FIRST_SESSION_ALIAS, Direction.FIRST, 27, 11L)); b3.addMessage(generateMessage(SECOND_SESSION_ALIAS, Direction.SECOND, 28, 12L)); - data = List.of(b1, b2, b3); + List data = List.of(b1, b2, b3); storedData = new HashMap<>(); BookInfo bookInfo = storage.refreshBook(bookId.getName()); for (GroupedMessageBatchToStore groupedBatch : data) { diff --git a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/messages/MessageIteratorProviderTest.java b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/messages/MessageIteratorProviderTest.java index 2745287c7..21ba3448f 100644 --- a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/messages/MessageIteratorProviderTest.java +++ b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/messages/MessageIteratorProviderTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -49,10 +49,9 @@ public class MessageIteratorProviderTest extends BaseCradleCassandraTest { private final long storeActionRejectionThreshold = new CoreStorageSettings().calculateStoreActionRejectionThreshold(); - private List data; private Map> storedData; private CassandraOperators operators; - private ExecutorService composingService = Executors.newSingleThreadExecutor(); + private final ExecutorService composingService = Executors.newFixedThreadPool(3); @BeforeClass public void startUp () throws IOException, InterruptedException, CradleStorageException { @@ -98,7 +97,7 @@ public int hashCode() { } } - private void setUpOperators() throws IOException, InterruptedException { + private void setUpOperators() { CassandraDataMapper dataMapper = new CassandraDataMapperBuilder(session).build(); operators = new CassandraOperators(dataMapper, CassandraCradleHelper.getInstance().getStorageSettings()); } @@ -122,7 +121,7 @@ protected void generateData () { b3.addMessage(generateMessage(FIRST_SESSION_ALIAS, Direction.FIRST, 25, 5L)); b3.addMessage(generateMessage(SECOND_SESSION_ALIAS, Direction.SECOND, 25, 6L)); - data = List.of(b1, b2, b3); + List data = List.of(b1, b2, b3); storedData = new HashMap<>(); BookInfo bookInfo = storage.refreshBook(bookId.getName()); for (GroupedMessageBatchToStore batch : data) { diff --git a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/pages/PagesApiRemoveTest.java b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/pages/PagesApiRemoveTest.java new file mode 100644 index 000000000..327531741 --- /dev/null +++ b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/pages/PagesApiRemoveTest.java @@ -0,0 +1,195 @@ +/* + * Copyright 2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.cradle.cassandra.integration.pages; + +import com.exactpro.cradle.BookId; +import com.exactpro.cradle.BookToAdd; +import com.exactpro.cradle.PageId; +import com.exactpro.cradle.PageInfo; +import com.exactpro.cradle.PageToAdd; +import com.exactpro.cradle.cassandra.integration.BaseCradleCassandraTest; +import com.exactpro.cradle.cassandra.integration.CassandraCradleHelper; +import com.exactpro.cradle.utils.CradleStorageException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static java.time.temporal.ChronoUnit.HOURS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; + +public class PagesApiRemoveTest extends BaseCradleCassandraTest { + private static final Logger LOGGER = LoggerFactory.getLogger(PagesApiRemoveTest.class); + private static final BookId BOOK_ID = new BookId(PagesApiRemoveTest.class.getSimpleName() + "Book"); + private static final Instant BOOK_START = Instant.now().minus(10, ChronoUnit.DAYS); + private static final List PAGE_IDS = List.of( + new PageId(BOOK_ID, Instant.now().minus(5, ChronoUnit.DAYS), "page1"), + new PageId(BOOK_ID, Instant.now().minus(4, ChronoUnit.DAYS), "page2"), + new PageId(BOOK_ID, Instant.now(), "page3"), + new PageId(BOOK_ID, Instant.now().plus(1, HOURS), "page4"), + new PageId(BOOK_ID, Instant.now().plus(1, ChronoUnit.DAYS), "page5") + ); + + @BeforeClass + public void startUp() throws IOException, InterruptedException, CradleStorageException { + this.session = CassandraCradleHelper.getInstance().getSession(); + this.storage = CassandraCradleHelper.getInstance().getStorage(); + generateData(); + } + + @Override + protected void generateData() throws CradleStorageException, IOException { + try { + storage.addBook(new BookToAdd(BOOK_ID.getName(), BOOK_START)); + storage.addPages(BOOK_ID, PAGE_IDS.stream() + .map(id -> new PageToAdd(id.getName(), id.getStart(), null)) + .collect(Collectors.toList())); + } catch (CradleStorageException | IOException e) { + LOGGER.error("Error while generating data:", e); + throw e; + } + } + + @Test(dataProvider = "pageIds") + public void testRemoveRandomPage(PageId pageId) throws CradleStorageException, IOException { + try { + storage.removePage(pageId); + + var result = storage.getAllPages(BOOK_ID); + var filteredPage = result.stream() + .filter(pageInfo -> pageInfo.getId().equals(pageId)) + .collect(Collectors.toList()); + + Instant now = Instant.now(); + if (pageId.getStart().isAfter(now)) { + assertThat(filteredPage.size()).isEqualTo(0); + } else { + assertThat(filteredPage.size()).isEqualTo(1); + filteredPage.forEach(pageInfo -> assertThat(pageInfo.getRemoved()).isBefore(now)); + } + } catch (IOException | CradleStorageException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test + public void testRemoveLastPage() throws CradleStorageException, IOException { + try { + // Preparation + BookId testBookId = new BookId("testRemoveLastPage"); + PageId pageId1 = new PageId(testBookId, Instant.now(), "page1"); + PageId pageId2 = new PageId(testBookId, Instant.now().plus(1, HOURS), "page2"); + storage.addBook(new BookToAdd(testBookId.getName(), BOOK_START)); + storage.addPage(testBookId, pageId1.getName(), pageId1.getStart(), null); + storage.addPage(testBookId, pageId2.getName(), pageId2.getStart(), null); + + var result = storage.getAllPages(testBookId); + assertEquals(result.size(), 2); + var pages = toMap(result); + + PageInfo page1 = pages.get(pageId1); + PageInfo page2 = pages.get(pageId2); + assertNotNull(page1); + assertNotNull(page2); + assertEquals(page1.getEnded(), pageId2.getStart()); + assertNull(page2.getEnded()); + + // test + storage.removePage(pageId2); + + result = storage.getAllPages(testBookId); + assertEquals(result.size(), 1); + pages = toMap(result); + + page1 = pages.get(pageId1); + assertNotNull(page1); + + assertNull(page1.getEnded()); + + } catch (IOException | CradleStorageException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test + public void testRemovePenultimatePage() throws CradleStorageException, IOException { + try { + // Preparation + BookId testBookId = new BookId("testRemovePenultimatePage"); + PageId pageId1 = new PageId(testBookId, Instant.now(), "page1"); + PageId pageId2 = new PageId(testBookId, Instant.now().plus(1, HOURS), "page2"); + PageId pageId3 = new PageId(testBookId, Instant.now().plus(2, HOURS), "page3"); + storage.addBook(new BookToAdd(testBookId.getName(), BOOK_START)); + storage.addPage(testBookId, pageId1.getName(), pageId1.getStart(), null); + storage.addPage(testBookId, pageId2.getName(), pageId2.getStart(), null); + storage.addPage(testBookId, pageId3.getName(), pageId3.getStart(), null); + + var result = storage.getAllPages(testBookId); + assertEquals(result.size(), 3); + var pages = toMap(result); + + PageInfo page1 = pages.get(pageId1); + PageInfo page2 = pages.get(pageId2); + PageInfo page3 = pages.get(pageId3); + assertNotNull(page1); + assertNotNull(page2); + assertNotNull(page3); + assertEquals(page1.getEnded(), pageId2.getStart()); + assertEquals(page2.getEnded(), pageId3.getStart()); + assertNull(page3.getEnded()); + + // test + storage.removePage(pageId2); + + result = storage.getAllPages(testBookId); + assertEquals(result.size(), 2); + pages = toMap(result); + + page1 = pages.get(pageId1); + page3 = pages.get(pageId3); + assertNotNull(page1); + assertNotNull(page3); + assertEquals(page1.getEnded(), pageId2.getStart()); + assertNull(page3.getEnded()); + + } catch (IOException | CradleStorageException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @DataProvider(name = "pageIds") + public static PageId[] cacheSize() { + List ids = new ArrayList<>(PAGE_IDS); + Collections.shuffle(ids); + return ids.toArray(new PageId[0]); + } +} diff --git a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/pages/PagesApiTest.java b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/pages/PagesApiTest.java index 5ca7a132c..cc709dc3d 100644 --- a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/pages/PagesApiTest.java +++ b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/pages/PagesApiTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Exactpro (Exactpro Systems Limited) + * Copyright 2023-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,9 +16,13 @@ package com.exactpro.cradle.cassandra.integration.pages; +import com.exactpro.cradle.BookId; +import com.exactpro.cradle.BookToAdd; +import com.exactpro.cradle.PageId; +import com.exactpro.cradle.PageInfo; import com.exactpro.cradle.cassandra.integration.BaseCradleCassandraTest; +import com.exactpro.cradle.counters.Interval; import com.exactpro.cradle.utils.CradleStorageException; -import org.assertj.core.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.BeforeClass; @@ -27,10 +31,26 @@ import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.Spliterator; import java.util.stream.Collectors; +import static com.exactpro.cradle.CoreStorageSettings.PAGE_ACTION_REJECTION_THRESHOLD_FACTOR; +import static com.exactpro.cradle.cassandra.integration.CassandraCradleHelper.BOOK_REFRESH_INTERVAL_MILLIS; +import static java.time.temporal.ChronoUnit.DAYS; +import static java.time.temporal.ChronoUnit.HOURS; +import static java.time.temporal.ChronoUnit.MILLIS; +import static java.time.temporal.ChronoUnit.NANOS; +import static java.util.Spliterators.spliteratorUnknownSize; +import static java.util.stream.StreamSupport.stream; +import static org.assertj.core.api.Assertions.assertThat; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; + public class PagesApiTest extends BaseCradleCassandraTest { - private static final Logger logger = LoggerFactory.getLogger(PagesApiTest.class); + private static final Logger LOGGER = LoggerFactory.getLogger(PagesApiTest.class); @BeforeClass public void startUp() throws IOException, InterruptedException, CradleStorageException { @@ -51,7 +71,7 @@ protected void generateData() throws CradleStorageException, IOException { ); } } catch (CradleStorageException | IOException e) { - logger.error("Error while generating data:", e); + LOGGER.error("Error while generating data:", e); throw e; } } @@ -61,17 +81,146 @@ public void testNonNullPages() throws CradleStorageException { try { var result = storage.getAllPages(bookId); var autoPagesNonNull = result.stream() - .filter(pageInfo -> pageInfo.getId().getName().startsWith("autoPageNotNull-")) + .filter(pageInfo -> pageInfo.getName().startsWith("autoPageNotNull-")) + .collect(Collectors.toList()); + assertThat(autoPagesNonNull.size()).isEqualTo(4); + autoPagesNonNull.forEach(pageInfo -> { + assertThat(pageInfo.getComment()).isNotNull(); + assertThat(pageInfo.getUpdated()).isNotNull(); + assertThat(pageInfo.getRemoved()).isNotNull(); + }); + } catch (CradleStorageException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test(description = "Gets pages from the max interval (MIN, MAX), filters them using name and their default values are not null") + public void testPagesByMaxInterval() throws CradleStorageException { + try { + var result = storage.getPages(bookId, new Interval(Instant.MIN, Instant.MAX)); + var autoPagesNonNull = stream(spliteratorUnknownSize(result, Spliterator.ORDERED), false) + .filter(pageInfo -> pageInfo.getName().startsWith("autoPageNotNull-")) .collect(Collectors.toList()); - Assertions.assertThat(autoPagesNonNull.size()).isEqualTo(4); + assertThat(autoPagesNonNull.size()).isEqualTo(4); autoPagesNonNull.forEach(pageInfo -> { - Assertions.assertThat(pageInfo.getComment()).isNotNull(); - Assertions.assertThat(pageInfo.getUpdated()).isNotNull(); - Assertions.assertThat(pageInfo.getRemoved()).isNotNull(); + assertThat(pageInfo.getComment()).isNotNull(); + assertThat(pageInfo.getUpdated()).isNotNull(); + assertThat(pageInfo.getRemoved()).isNotNull(); }); } catch (CradleStorageException e) { - logger.error(e.getMessage(), e); + LOGGER.error(e.getMessage(), e); throw e; } } + + @Test + public void testFillGapInPenultimatePage() throws CradleStorageException, IOException { + try { + // Preparation + BookId testBookId = new BookId("testFillGapInPenultimatePage"); + Instant testBookStart = Instant.now().minus(10, ChronoUnit.DAYS); + PageId pageId1 = new PageId(testBookId, Instant.now(), "page1"); + PageId pageId2 = new PageId(testBookId, Instant.now().plus(1, HOURS), "page2"); + PageId pageId3 = new PageId(testBookId, Instant.now().plus(2, HOURS), "page3"); + storage.addBook(new BookToAdd(testBookId.getName(), testBookStart)); + storage.addPage(testBookId, pageId1.getName(), pageId1.getStart(), null); + storage.addPage(testBookId, pageId2.getName(), pageId2.getStart(), null); + storage.addPage(testBookId, pageId3.getName(), pageId3.getStart(), null); + storage.removePage(pageId2); + + var result = storage.getAllPages(testBookId); + assertEquals(result.size(), 2); + var pages = toMap(result); + + PageInfo page1 = pages.get(pageId1); + PageInfo page3 = pages.get(pageId3); + assertNotNull(page1); + assertNotNull(page3); + assertEquals(page1.getEnded(), pageId2.getStart()); + assertNull(page3.getEnded()); + + // test + PageId pageId4 = new PageId(testBookId, pageId2.getStart(), "page4"); + storage.addPage(testBookId, pageId4.getName(), pageId4.getStart(), null); + + result = storage.getAllPages(testBookId); + assertEquals(result.size(), 3); + pages = toMap(result); + + page1 = pages.get(pageId1); + PageInfo page4 = pages.get(pageId4); + page3 = pages.get(pageId3); + assertNotNull(page1); + assertNotNull(pageId4); + assertNotNull(page3); + assertEquals(page1.getEnded(), pageId2.getStart()); + assertEquals(page4.getEnded(), pageId3.getStart()); + assertNull(page3.getEnded()); + + } catch (IOException | CradleStorageException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test(description = "Try to add the second page before page action reject threshold", + expectedExceptions = CradleStorageException.class, + expectedExceptionsMessageRegExp = "You can only create pages which start more than.*" + ) + public void testAddTheSecondPageBeforeThreshold() throws CradleStorageException, IOException { + try { + // Preparation + BookId testBookId = new BookId("testAddPagesToThePast"); + Instant testBookStart = Instant.now().minus(10, ChronoUnit.DAYS); + // Adding page before book start time looks strange but the current behavior doesn't affect anybody + PageId pageId1 = new PageId(testBookId, testBookStart.minus(10, DAYS), "page1"); + storage.addBook(new BookToAdd(testBookId.getName(), testBookStart)); + storage.addPage(testBookId, pageId1.getName(), pageId1.getStart(), null); + assertEquals(storage.getAllPages(testBookId).size(), 1); + + // Test + PageId pageId2 = new PageId(testBookId, + Instant.now().plus(BOOK_REFRESH_INTERVAL_MILLIS * PAGE_ACTION_REJECTION_THRESHOLD_FACTOR, + MILLIS), + "page2"); + storage.addPage(testBookId, pageId2.getName(), pageId2.getStart(), null); + } catch (IOException | CradleStorageException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test(description = "Try to rename page before page action reject threshold", + expectedExceptions = CradleStorageException.class, + expectedExceptionsMessageRegExp = "You can only rename pages which start more than.*" + ) + public void testRenamePageBeforeThreshold() throws CradleStorageException, IOException { + try { + // Preparation + BookId testBookId = new BookId("testRenamePageBeforeThreshold"); + Instant testBookStart = Instant.now().minus(10, ChronoUnit.DAYS); + PageId pageId1 = new PageId(testBookId, Instant.now().plus(BOOK_REFRESH_INTERVAL_MILLIS * PAGE_ACTION_REJECTION_THRESHOLD_FACTOR, + MILLIS), "page1"); + storage.addBook(new BookToAdd(testBookId.getName(), testBookStart)); + storage.addPage(testBookId, pageId1.getName(), pageId1.getStart(), null); + assertEquals(storage.getAllPages(testBookId).size(), 1); + + // Test + storage.updatePageName(testBookId, pageId1.getName(), pageId1.getName() + "-new"); + } catch (IOException | CradleStorageException e) { + LOGGER.error(e.getMessage(), e); + throw e; + } + } + + @Test(description = "Try to add page with start timestamp between already existed page", + expectedExceptions = CradleStorageException.class, + expectedExceptionsMessageRegExp = "Can't add new page in book.*" + ) + public void testAddPageInTheMiddleOfExist() throws CradleStorageException, IOException { + List allPages = new ArrayList<>(storage.getAllPages(bookId)); + PageInfo pageInfo = allPages.get(allPages.size() - 2); + storage.addPage(bookId, "invalid-page", pageInfo.getStarted().plus(1, NANOS), null); + } } diff --git a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/testevents/TestEventIteratorProviderTest.java b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/testevents/TestEventIteratorProviderTest.java index d99324f68..26d7477aa 100644 --- a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/testevents/TestEventIteratorProviderTest.java +++ b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/testevents/TestEventIteratorProviderTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,20 +16,23 @@ package com.exactpro.cradle.cassandra.integration.testevents; import com.exactpro.cradle.BookInfo; -import com.exactpro.cradle.CoreStorageSettings; import com.exactpro.cradle.PageId; -import com.exactpro.cradle.cassandra.dao.testevents.TestEventIteratorProvider; -import com.exactpro.cradle.cassandra.integration.BaseCradleCassandraTest; -import com.exactpro.cradle.cassandra.integration.CassandraCradleHelper; import com.exactpro.cradle.cassandra.EventBatchDurationWorker; import com.exactpro.cradle.cassandra.dao.CassandraDataMapper; import com.exactpro.cradle.cassandra.dao.CassandraDataMapperBuilder; import com.exactpro.cradle.cassandra.dao.CassandraOperators; +import com.exactpro.cradle.cassandra.dao.testevents.TestEventIteratorProvider; +import com.exactpro.cradle.cassandra.integration.BaseCradleCassandraTest; +import com.exactpro.cradle.cassandra.integration.CassandraCradleHelper; import com.exactpro.cradle.cassandra.resultset.CassandraCradleResultSet; import com.exactpro.cradle.cassandra.retries.PageSizeAdjustingPolicy; import com.exactpro.cradle.cassandra.retries.SelectQueryExecutor; import com.exactpro.cradle.filters.FilterForGreater; -import com.exactpro.cradle.testevents.*; +import com.exactpro.cradle.testevents.StoredTestEvent; +import com.exactpro.cradle.testevents.StoredTestEventBatch; +import com.exactpro.cradle.testevents.StoredTestEventSingle; +import com.exactpro.cradle.testevents.TestEventFilter; +import com.exactpro.cradle.testevents.TestEventToStore; import com.exactpro.cradle.utils.CradleStorageException; import org.assertj.core.api.Assertions; import org.assertj.core.util.Lists; @@ -40,18 +43,19 @@ import org.testng.annotations.Test; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_RESULT_PAGE_SIZE; - public class TestEventIteratorProviderTest extends BaseCradleCassandraTest { private static final Logger logger = LoggerFactory.getLogger(TestEventIteratorProviderTest.class); @@ -65,7 +69,7 @@ public class TestEventIteratorProviderTest extends BaseCradleCassandraTest { private final List data = new ArrayList<>(); private Map> storedData; private CassandraOperators operators; - private final ExecutorService composingService = Executors.newSingleThreadExecutor(); + private final ExecutorService composingService = Executors.newFixedThreadPool(3); private EventBatchDurationWorker eventBatchDurationWorker; @BeforeClass @@ -162,7 +166,7 @@ private TestEventIteratorProvider createIteratorProvider(TestEventFilter filter, } } - private void setUpOperators() throws IOException, InterruptedException { + private void setUpOperators() { CassandraDataMapper dataMapper = new CassandraDataMapperBuilder(session).build(); operators = new CassandraOperators(dataMapper, CassandraCradleHelper.getInstance().getStorageSettings()); } diff --git a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/resultset/PagesInIntervalIteratorProviderTest.java b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/resultset/PagesInIntervalIteratorProviderTest.java index c2c5ef9a4..0abc19da7 100644 --- a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/resultset/PagesInIntervalIteratorProviderTest.java +++ b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/resultset/PagesInIntervalIteratorProviderTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Exactpro (Exactpro Systems Limited) + * Copyright 2023-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,26 +33,27 @@ public class PagesInIntervalIteratorProviderTest { @Test public void defaultPageVsIncludedInterval() { - PageInfo pageInfo = new PageInfo(new PageId(bookId, "test-page"), Instant.EPOCH, null, null); + PageInfo pageInfo = new PageInfo(new PageId(bookId, Instant.EPOCH, "test-page"), null, null); assertTrue(checkInterval(pageInfo, pageInfo.getStarted(), Instant.now()), "in.st = p.st, in.en = p.en"); assertTrue(checkInterval(pageInfo, pageInfo.getStarted().plusNanos(1), Instant.now()), "in.st = p.st + 1, in.en = p.en"); - assertTrue(checkInterval(pageInfo, pageInfo.getStarted(), pageInfo.getStarted()), "in.st = p.st, in.en = p.st"); - assertTrue(checkInterval(pageInfo, pageInfo.getStarted().minusNanos(1), pageInfo.getStarted()), "in.st = p.st - 1, in.en = p.st"); + assertTrue(checkInterval(pageInfo, pageInfo.getStarted(), pageInfo.getId().getStart()), "in.st = p.st, in.en = p.st"); + assertTrue(checkInterval(pageInfo, pageInfo.getStarted().minusNanos(1), pageInfo.getId().getStart()), "in.st = p.st - 1, in.en = p.st"); assertTrue(checkInterval(pageInfo, Instant.now(), Instant.now()), "in.st = p.en, in.en = p.en"); assertFalse(checkInterval(pageInfo, pageInfo.getStarted().minusNanos(1), pageInfo.getStarted().minusNanos(1)), "in.st = p.st - 1, in.en = p.st - 1"); } @Test public void onePageVsIntervals() { - PageInfo pageInfo = new PageInfo(new PageId(bookId, "test-page"), Instant.now(), Instant.now().plusSeconds(1), null); + Instant now = Instant.now(); + PageInfo pageInfo = new PageInfo(new PageId(bookId, now, "test-page"), now.plusSeconds(1), null); assertTrue(checkInterval(pageInfo, pageInfo.getStarted(), pageInfo.getEnded()), "in.st = p.st, in.en = p.en"); assertTrue(checkInterval(pageInfo, pageInfo.getStarted().plusNanos(1), pageInfo.getEnded()), "in.st = p.st + 1, in.en = p.en"); assertTrue(checkInterval(pageInfo, pageInfo.getStarted(), pageInfo.getEnded().minusNanos(1)), "in.st = p.st, in.en = p.en - 1"); assertTrue(checkInterval(pageInfo, pageInfo.getStarted().plusNanos(1), pageInfo.getEnded().minusNanos(1)), "in.st = p.st + 1, in.en = p.en - 1"); - assertTrue(checkInterval(pageInfo, pageInfo.getStarted(), pageInfo.getStarted()), "in.st = p.st, in.en = p.st"); - assertTrue(checkInterval(pageInfo, pageInfo.getStarted().minusNanos(1), pageInfo.getStarted()), "in.st = p.st - 1, in.en = p.st"); + assertTrue(checkInterval(pageInfo, pageInfo.getStarted(), pageInfo.getId().getStart()), "in.st = p.st, in.en = p.st"); + assertTrue(checkInterval(pageInfo, pageInfo.getStarted().minusNanos(1), pageInfo.getId().getStart()), "in.st = p.st - 1, in.en = p.st"); assertTrue(checkInterval(pageInfo, pageInfo.getEnded(), pageInfo.getEnded()), "in.st = p.en, in.en = p.en"); assertTrue(checkInterval(pageInfo, pageInfo.getEnded(), pageInfo.getEnded().plusNanos(1)), "in.st = p.en, in.en = p.en + 1"); assertFalse(checkInterval(pageInfo, pageInfo.getStarted().minusNanos(1), pageInfo.getStarted().minusNanos(1)), "in.st = p.st - 1, in.en = p.st - 1"); diff --git a/cradle-core/build.gradle b/cradle-core/build.gradle index e84ff9f96..e1bf27cce 100644 --- a/cradle-core/build.gradle +++ b/cradle-core/build.gradle @@ -8,6 +8,8 @@ dependencies { implementation 'com.google.guava:guava' implementation 'org.lz4:lz4-java:1.8.0' + implementation 'com.github.ben-manes.caffeine:caffeine:3.1.8' + testImplementation 'org.apache.logging.log4j:log4j-slf4j2-impl' testImplementation 'org.apache.logging.log4j:log4j-core' testImplementation 'org.testng:testng:7.9.0' diff --git a/cradle-core/src/main/java/com/exactpro/cradle/BookCache.java b/cradle-core/src/main/java/com/exactpro/cradle/BookCache.java index 942ed6cd2..2ac4bba4b 100644 --- a/cradle-core/src/main/java/com/exactpro/cradle/BookCache.java +++ b/cradle-core/src/main/java/com/exactpro/cradle/BookCache.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,9 +33,5 @@ public interface BookCache { Collection loadPageInfo(BookId bookId, boolean loadRemoved) throws CradleStorageException; - BookInfo loadBook(BookId bookId) throws CradleStorageException; - - void updateCachedBook(BookInfo bookInfo); - Collection getCachedBooks(); } diff --git a/cradle-core/src/main/java/com/exactpro/cradle/BookInfo.java b/cradle-core/src/main/java/com/exactpro/cradle/BookInfo.java index f7bdc594b..aa53bcc24 100644 --- a/cradle-core/src/main/java/com/exactpro/cradle/BookInfo.java +++ b/cradle-core/src/main/java/com/exactpro/cradle/BookInfo.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,170 +16,572 @@ package com.exactpro.cradle; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.*; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; import java.util.Map.Entry; -import java.util.concurrent.ConcurrentHashMap; +import java.util.NavigableMap; +import java.util.Objects; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; -import com.exactpro.cradle.utils.CradleStorageException; +import static com.exactpro.cradle.BookInfoMetrics.CacheName.HOT; +import static com.exactpro.cradle.BookInfoMetrics.CacheName.RANDOM; +import static com.exactpro.cradle.BookInfoMetrics.RequestMethod.FIND; +import static com.exactpro.cradle.BookInfoMetrics.RequestMethod.GET; +import static com.exactpro.cradle.BookInfoMetrics.RequestMethod.ITERATE; +import static com.exactpro.cradle.BookInfoMetrics.RequestMethod.NEXT; +import static com.exactpro.cradle.BookInfoMetrics.RequestMethod.PREVIOUS; +import static com.exactpro.cradle.BookInfoMetrics.RequestMethod.REFRESH; +import static java.util.Collections.emptyIterator; +import static java.util.Collections.unmodifiableMap; +import static java.util.Collections.unmodifiableNavigableMap; /** * Information about a book */ -public class BookInfo -{ - private final BookId id; - private final String fullName, - desc; - private final Instant created; - private final Map pages; - private final TreeMap orderedPages; - - public BookInfo(BookId id, String fullName, String desc, Instant created, Collection pages) throws CradleStorageException - { - this.id = id; - this.fullName = fullName; - this.desc = desc; - this.created = created; - this.pages = new ConcurrentHashMap<>(); - this.orderedPages = new TreeMap<>(); - - if (pages == null) - return; - - PageInfo notEndedPage = null; - for (PageInfo p : pages) - { - this.pages.put(p.getId(), p); - this.orderedPages.put(p.getStarted(), p); - if (p.getEnded() == null) - { - if (notEndedPage != null) - throw new CradleStorageException("Inconsistent state of book '"+id+"': " - + "page '"+ notEndedPage.getId().getName()+"' is not ended, " - + "but '"+p.getId().getName()+"' is not ended as well"); - notEndedPage = p; - } - } - } - - - public BookId getId() - { - return id; - } - - public String getFullName() - { - return fullName; - } - - public String getDesc() - { - return desc; - } - - public Instant getCreated() - { - return created; - } - - public Collection getPages() - { - return Collections.unmodifiableCollection(orderedPages.values()); - } - - public PageInfo getFirstPage() - { - return orderedPages.size() > 0 ? orderedPages.firstEntry().getValue() : null; - } - - public PageInfo getLastPage() - { - return orderedPages.size() > 0 ? orderedPages.lastEntry().getValue() : null; - } - - public PageInfo getPage(PageId pageId) - { - return pages.get(pageId); - } - - public PageInfo findPage(Instant timestamp) - { - Entry result = orderedPages.floorEntry(timestamp); - return result != null ? result.getValue() : null; - } - - public PageInfo getNextPage(Instant startTimestamp) - { - Entry result = orderedPages.ceilingEntry(startTimestamp.plus(1, ChronoUnit.NANOS)); - return result != null ? result.getValue() : null; - } - - public PageInfo getPreviousPage(Instant startTimestamp) - { - Entry result = orderedPages.floorEntry(startTimestamp.minus(1, ChronoUnit.NANOS)); - return result != null ? result.getValue() : null; - } - - - void removePage(PageId pageId) - { - PageInfo pageInfo = pages.remove(pageId); - if (pageInfo != null) - orderedPages.remove(pageInfo.getStarted()); - } - - void addPage(PageInfo page) - { - pages.put(page.getId(), page); - orderedPages.put(page.getStarted(), page); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - BookInfo bookInfo = (BookInfo) o; - - if (!Objects.equals(this.getId(), bookInfo.getId())) { - return false; - } - if (!Objects.equals(this.getCreated(), bookInfo.getCreated())) { - return false; - } - - if (!Objects.equals(this.getDesc(), bookInfo.getDesc())) { - return false; - } - - if (!Objects.equals(this.getFullName(), bookInfo.getFullName())) { - return false; - } - - /* - As getPages returns new unmodifiable list of ordered pages - this should work without any problems - */ - List pages = new ArrayList<> (this.getPages()); - List otherPages = new ArrayList<> (bookInfo.getPages()); - - if (pages.size() != otherPages.size()) { - return false; - } - - for (int i = 0; i < pages.size(); i ++) { - if (!pages.get(i).equals(otherPages.get(i))) { - return false; - } - } - - return true; - } - - @Override - public int hashCode() { - return Objects.hash(getId(), getFullName(), getDesc(), getCreated(), getPages(), orderedPages); - } +public class BookInfo { + private static final Logger LOGGER = LoggerFactory.getLogger(BookInfo.class); + private static final BookInfoMetrics METRICS = new BookInfoMetrics(); + private static final int HOT_CACHE_SIZE = 2; + private static final int SECONDS_IN_DAY = 60 * 60 * 24; + private static final int MILLISECONDS_IN_DAY = SECONDS_IN_DAY * 1_000; + private static final long MAX_EPOCH_DAY = getEpochDay(Instant.MAX); + private static final IPageInterval EMPTY_PAGE_INTERVAL = new EmptyPageInterval(); + + private static final BookId EMPTY_BOOK_ID = new BookId(""); + + static { + METRICS.setPageCacheSize(EMPTY_BOOK_ID, HOT, HOT_CACHE_SIZE); + } + + private final BookId id; + private final String fullName, + desc; + private final Instant created; + + private final LoadingCache hotCache; + private final LoadingCache randomAccessCache; + + // AtomicInitializer call initialize method again if previous value is null + private final AtomicReference firstPage = new AtomicReference<>(); + private final PagesLoader pagesLoader; + private final Function firstPageLoader; + private final Function lastPageLoader; + + /** + * This class provides access to pages related to the {@code id} book id. + * Two cache ara used internally to hold pages relate to day interval + * * Hot cache holds pages for the current and previous day. It can be refreshed by the {@link #refresh()} method + * * Random access cache holds pages for days in the past. Book info invalidates this cache with the {@code raCacheInvalidateInterval} interval + * @param raCacheSize - random access cache size. How many days of pages should be cached for the past. + * @param raCacheInvalidateInterval - invalidate interval in millisecond for random access cache. + */ + public BookInfo(BookId id, + String fullName, + String desc, + Instant created, + int raCacheSize, + long raCacheInvalidateInterval, + PagesLoader pagesLoader, + Function firstPageLoader, + Function lastPageLoader) { + this.id = id; + this.fullName = fullName; + this.desc = desc; + this.created = created; + this.pagesLoader = pagesLoader; + this.firstPageLoader = firstPageLoader; + this.lastPageLoader = lastPageLoader; + + this.hotCache = Caffeine.newBuilder() + .maximumSize(HOT_CACHE_SIZE) + .removalListener((epochDay, pageInterval, cause) -> METRICS.incInvalidate(id, HOT, cause)) + .build(epochDay -> createPageInterval(HOT, epochDay)); + + this.randomAccessCache = Caffeine.newBuilder() + .maximumSize(raCacheSize) + .expireAfterWrite(raCacheInvalidateInterval, TimeUnit.MILLISECONDS) + .removalListener((epochDay, pageInterval, cause) -> METRICS.incInvalidate(id, RANDOM, cause)) + .build(epochDay -> createPageInterval(RANDOM, epochDay)); + + METRICS.setPageCacheSize(id, RANDOM, raCacheSize); + } + + public BookId getId() { + return id; + } + + public String getFullName() { + return fullName; + } + + public String getDesc() { + return desc; + } + + public Instant getCreated() { + return created; + } + + /** + * Requests pages from {@link PagesLoader} with book id and null start/end + * + * @return all ordered pages related to book id + */ + public Collection getPages() { + return pagesLoader.load(id, null, null); + } + + public @Nullable PageInfo getFirstPage() { + PageInfo result = firstPage.get(); + + if (result == null) { + result = firstPageLoader.apply(id); + if (!firstPage.compareAndSet(null, result)) { + // another thread has initialized the reference + result = firstPage.get(); + } + } + + return result; + } + + public @Nullable PageInfo getLastPage() { + return lastPageLoader.apply(id); + } + + public PageInfo getPage(PageId pageId) { + IPageInterval pageInterval = getPageInterval(pageId.getStart()); + return pageInterval.get(pageId); + } + + /** + * Requests page iterator using cache. Iterator lazy loads pages into cache when requested day isn't in there. + * + * @param leftBoundTimestamp inclusive minimal timestamp. Start time of first page is used if passed value is null + * @param rightBoundTimestamp inclusive maximum timestamp. Start time of last page is used if passed value is null + */ + public Iterator getPages(@Nullable Instant leftBoundTimestamp, + @Nullable Instant rightBoundTimestamp, + @Nonnull Order order) { + if (leftBoundTimestamp != null && + rightBoundTimestamp != null && + leftBoundTimestamp.isAfter(rightBoundTimestamp)) { + LOGGER.warn("Left bound '{}' should be <= right bound '{}'", leftBoundTimestamp, rightBoundTimestamp); + return emptyIterator(); + } + + if (getFirstPage() == null) { + return emptyIterator(); + } + + Instant leftTimestamp = calculateBound(leftBoundTimestamp, this::getFirstPage); + if (leftTimestamp == null) { + return emptyIterator(); + } + Instant rightTimestamp = calculateBound(rightBoundTimestamp, this::getLastPage); + if (rightTimestamp == null) { + return emptyIterator(); + } + + if (leftTimestamp.isAfter(rightTimestamp)) { + LOGGER.warn("Left calculated bound '{}' should be <= right calculated bound '{}'", leftTimestamp, rightTimestamp); + return emptyIterator(); + } + + long leftBoundEpochDay = getEpochDay(leftTimestamp); + long rightBoundEpochDay = getEpochDay(rightTimestamp); + + Stream.Builder builder = Stream.builder(); + switch (order) { + case DIRECT: + for (long i = leftBoundEpochDay; i <= rightBoundEpochDay; i++) { + builder.add(i); + } + break; + case REVERSE: + for (long i = rightBoundEpochDay; i >= leftBoundEpochDay; i--) { + builder.add(i); + } + break; + default: + throw new IllegalStateException("Unexpected value: " + order); + } + return builder.build() + .flatMap(epochDay -> getPageInterval(epochDay).stream(leftTimestamp, rightTimestamp, order)) + .iterator(); + } + + /** + * Searches for a page that contains the specified timestamp. + * Page contains the timestamp if it is inside the following interval `[start, end)` where: + * - start - page start timestamp included + * - end - page end timestamp excluded + */ + public PageInfo findPage(Instant timestamp) { + long epochDay = getEpochDay(timestamp); + if (epochDay < 0) { + return null; + } + if (epochDay >= MAX_EPOCH_DAY) { + PageInfo lastPage = getLastPage(); + if (lastPage != null && lastPage.getEnded() == null) { + return lastPage; + } + return null; + } + // Search in the page interval related to the timestamp + IPageInterval pageInterval = getPageInterval(epochDay); + PageInfo pageInfo = pageInterval.find(timestamp); + if (pageInfo != null) { + return pageInfo; + } + + // Check first page + PageInfo firstPage = getFirstPage(); + if (firstPage == null) { + return null; + } + if (firstPage.getStarted().isAfter(timestamp)) { + return null; + } + + // Check last page + PageInfo lastPage = getLastPage(); + if (lastPage == null) { + return null; + } + if (lastPage.getEnded() != null && !timestamp.isBefore(lastPage.getEnded())) { + return null; + } + + // Search in page intervals before the timestamp + long firstEpochDay = getEpochDay(firstPage.getStarted()); + if (epochDay < firstEpochDay) { + return null; + } + if (epochDay > firstEpochDay) { + for (long previousEpochDay = epochDay - 1; previousEpochDay >= firstEpochDay; previousEpochDay--) { + IPageInterval previousPageInterval = getPageInterval(previousEpochDay); + if (previousPageInterval.size() == 0) { + // page interval without new pages + continue; + } + // sP1 ... | ... eP1 ^ ... gap ... sP2 ... | + return previousPageInterval.find(timestamp); + } + throw new IllegalStateException( + "First page is before requested timestamp, but neither cache contains appropriate page, requested: " + + timestamp + ", first page: " + firstPage.getStarted() + ); + } + // | ... sP1 ... eP1 ^ ... gap ... sP2 ... | + // sP1 ... eP1 ^ ... | ... gap ... sP2 ... | + return null; + } + + public PageInfo getNextPage(Instant startTimestamp) { + long epochDate = getEpochDay(startTimestamp); + IPageInterval pageInterval = getPageInterval(epochDate); + PageInfo currentInterval = pageInterval.next(startTimestamp); + if (currentInterval != null) { + return currentInterval; + } + pageInterval = getPageInterval(epochDate + 1); + return pageInterval.next(startTimestamp); + } + + public PageInfo getPreviousPage(Instant startTimestamp) { + long epochDate = getEpochDay(startTimestamp); + IPageInterval pageInterval = getPageInterval(epochDate); + PageInfo currentInterval = pageInterval.previous(startTimestamp); + if (currentInterval != null) { + return currentInterval; + } + pageInterval = getPageInterval(epochDate - 1); + return pageInterval.previous(startTimestamp); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BookInfo bookInfo = (BookInfo) o; + return Objects.equals(id, bookInfo.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public String toString() { + return "BookInfo{" + + "id=" + id + + ", fullName='" + fullName + '\'' + + ", desc='" + desc + '\'' + + ", created=" + created + + '}'; + } + + /** + * Refreshes: first page and hot cache + * Invalidates: random access cache + */ + void refresh() { + firstPage.set(null); + METRICS.incRequest(id, HOT, REFRESH); + hotCache.invalidateAll(); + + getFirstPage(); + long currentEpochDay = currentEpochDay(); + for (int shift = HOT_CACHE_SIZE - 1; shift >= 0; shift--) { + this.hotCache.get(currentEpochDay - shift); + } + } + + void invalidate(Instant timestamp) { + long epochDay = getEpochDay(timestamp); + invalidate(epochDay); + } + + void invalidate(Iterable timestamps) { + StreamSupport.stream(timestamps.spliterator(), false) + .map(BookInfo::getEpochDay) + .distinct() + .forEach(this::invalidate); + } + + private @Nullable Instant calculateBound(@Nullable Instant origin, Supplier supplier) { + if (origin != null) { + PageInfo pageInfo = findPage(origin); + if (pageInfo != null) { + return pageInfo.getStarted(); + } + return origin; + } + PageInfo pageInfo = supplier.get(); + if (pageInfo == null) { + return null; + } + return pageInfo.getStarted(); + } + + private void invalidate(long epochDay) { + hotCache.invalidate(epochDay); + randomAccessCache.invalidate(epochDay); + } + + private IPageInterval getPageInterval(long epochDate) { + long currentEpochDay = currentEpochDay(); + long diff = currentEpochDay - epochDate; + LoadingCache cache = 0 <= diff && diff < 2 ? hotCache : randomAccessCache; + IPageInterval pageInterval = cache.get(epochDate); + return pageInterval != null + ? pageInterval + : EMPTY_PAGE_INTERVAL; + } + + private IPageInterval getPageInterval(Instant timestamp) { + long epochDate = getEpochDay(timestamp); + return getPageInterval(epochDate); + } + + private @Nullable IPageInterval createPageInterval(BookInfoMetrics.CacheName cacheName, Long epochDay) { + Instant start = toInstant(epochDay); + Instant end = start.plus(1, ChronoUnit.DAYS).minus(1, ChronoUnit.NANOS); + Collection loaded = pagesLoader.load(id, start, end); + if (loaded.isEmpty()) { + return null; + } + // We shouldn't register `load` when day page interval (interval in the next) is empty because don't register `request` for empty interval. + // `EmptyPageInterval` handles `requests` for empty interval and doesn't register `request` + // `loads` and `requests` are required for calculating hit / miss rate + METRICS.incLoads(id, cacheName, loaded.size()); + return create(id, start, cacheName, loaded); + } + + private static long currentEpochDay() { + return System.currentTimeMillis() / MILLISECONDS_IN_DAY; + } + + private static long getEpochDay(Instant instant) { + return instant.getEpochSecond() / SECONDS_IN_DAY; + } + + private static Instant toInstant(long epochDay) { + return Instant.ofEpochSecond(epochDay * SECONDS_IN_DAY); + } + + private interface IPageInterval { + int size(); + + PageInfo get(PageId pageId); + + PageInfo find(Instant timestamp); + + PageInfo next(Instant startTimestamp); + + PageInfo previous(Instant startTimestamp); + + Stream stream(Instant leftBoundTimestamp, Instant rightBoundTimestamp, Order order); + } + + private static final class EmptyPageInterval implements IPageInterval { + + @Override + public int size() { + return 0; + } + + @Override + public PageInfo get(PageId pageId) { + return null; + } + + @Override + public PageInfo find(Instant timestamp) { + return null; + } + + @Override + public PageInfo next(Instant startTimestamp) { + return null; + } + + @Override + public PageInfo previous(Instant startTimestamp) { + return null; + } + + @Override + public Stream stream(Instant leftBoundTimestamp, Instant rightBoundTimestamp, Order order) { + return Stream.empty(); + } + } + + private static class PageInterval implements IPageInterval { + private final BookId bookId; + private final BookInfoMetrics.CacheName cacheName; + private final Map pageById; + private final NavigableMap pageByInstant; + + private PageInterval(BookId bookId, BookInfoMetrics.CacheName cacheName, Map pageById, NavigableMap pageByInstant) { + this.bookId = bookId; + this.cacheName = cacheName; + this.pageById = pageById; + this.pageByInstant = pageByInstant; + } + + @Override + public int size() { + return pageById.size(); + } + + @Override + public PageInfo get(PageId pageId) { + METRICS.incRequest(bookId, cacheName, GET); + return pageById.get(pageId); + } + + @Override + public PageInfo find(Instant timestamp) { + METRICS.incRequest(bookId, cacheName, FIND); + Entry result = pageByInstant.floorEntry(timestamp); + if (result == null) { + return null; + } + PageInfo pageInfo = result.getValue(); + if (pageInfo.getEnded() == null) { + return pageInfo; + } + if (!timestamp.isBefore(pageInfo.getEnded())) { // the page's end timestamp is excluded from the page's interval + return null; + } + return pageInfo; + } + + @Override + public PageInfo next(Instant startTimestamp) { + METRICS.incRequest(bookId, cacheName, NEXT); + Entry result = pageByInstant.higherEntry(startTimestamp); + return result != null ? result.getValue() : null; + } + + @Override + public PageInfo previous(Instant startTimestamp) { + METRICS.incRequest(bookId, cacheName, PREVIOUS); + Entry result = pageByInstant.lowerEntry(startTimestamp); + return result != null ? result.getValue() : null; + } + + @Override + public Stream stream(@Nonnull Instant leftBoundTimestamp, + @Nonnull Instant rightBoundTimestamp, + @Nonnull Order order) { + METRICS.incRequest(bookId, cacheName, ITERATE); + Instant start = pageByInstant.floorKey(leftBoundTimestamp); + NavigableMap subMap = pageByInstant.subMap( + start == null ? leftBoundTimestamp : start, true, + rightBoundTimestamp, true + ); + + Predicate predicate = pageInfo -> !(pageInfo.getStarted().isAfter(rightBoundTimestamp) || + (pageInfo.getEnded() != null && !leftBoundTimestamp.isBefore(pageInfo.getEnded()))); + + switch (order) { + case DIRECT: + return subMap.values().stream().filter(predicate); + case REVERSE: + return subMap.descendingMap().values().stream().filter(predicate); + default: + throw new IllegalArgumentException("Unsupported order: " + order); + } + } + } + + private static PageInterval create(BookId id, Instant start, BookInfoMetrics.CacheName cacheName, Collection pages) { + Map pageById = new HashMap<>(); + TreeMap pageByInstant = new TreeMap<>(); + for (PageInfo page : pages) { + PageInfo previous = pageById.put(page.getId(), page); + if (previous != null) { + throw new IllegalStateException( + "Page with '" + page.getId() + "' id is duplicated, previous: " + + previous + ", current: " + page + ); + } + previous = pageByInstant.put(page.getStarted(), page); + if (previous != null) { + throw new IllegalStateException( + "Page with '" + page.getStarted() + "' start time is duplicated, previous: " + + previous + ", current: " + page + ); + } + } + LOGGER.debug("Loaded {} pages for the book {}, {}", pages.size(), id, start); + return new PageInterval( + id, cacheName, + unmodifiableMap(pageById), + unmodifiableNavigableMap(pageByInstant) + ); + } } \ No newline at end of file diff --git a/cradle-core/src/main/java/com/exactpro/cradle/BookInfoMetrics.java b/cradle-core/src/main/java/com/exactpro/cradle/BookInfoMetrics.java new file mode 100644 index 000000000..d70c44d80 --- /dev/null +++ b/cradle-core/src/main/java/com/exactpro/cradle/BookInfoMetrics.java @@ -0,0 +1,190 @@ +/* + * Copyright 2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.cradle; + +import com.github.benmanes.caffeine.cache.RemovalCause; +import io.prometheus.client.Counter; +import io.prometheus.client.Gauge; +import io.prometheus.client.Summary; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +public class BookInfoMetrics { + private static final String REQUEST_METHOD_LABEL = "method"; + private static final String BOOK_LABEL = "book"; + private static final String CACHE_NAME_LABEL = "cache"; + private static final String INVALIDATE_CAUSE_LABEL = "cause"; + private static final Gauge PAGE_CACHE_SIZE_GAUGE = Gauge.build() + .name("cradle_page_cache_size") + .help("Size of page cache") + .labelNames(BOOK_LABEL, CACHE_NAME_LABEL) + .register(); + + private static final Map PAGE_CACHE_SIZE_MAP = new ConcurrentHashMap<>(); + private static final Counter PAGE_REQUEST_COUNTER = Counter.build() + .name("cradle_page_cache_page_request_total") + .help("Page requests number from cache") + .labelNames(BOOK_LABEL, CACHE_NAME_LABEL, REQUEST_METHOD_LABEL) + .register(); + private static final Map PAGE_REQUEST_MAP = new ConcurrentHashMap<>(); + private static final Counter INVALIDATE_CACHE_COUNTER = Counter.build() + .name("cradle_page_cache_invalidate_total") + .help("Cache invalidates") + .labelNames(BOOK_LABEL, CACHE_NAME_LABEL, INVALIDATE_CAUSE_LABEL) + .register(); + private static final Map INVALIDATE_CACHE_MAP = new ConcurrentHashMap<>(); + + private static final Summary PAGE_LOADS_COUNTER = Summary.build() + .name("cradle_page_cache_page_loads_total") + .help("Page loads number to cache") + .labelNames(BOOK_LABEL, CACHE_NAME_LABEL) + .register(); + + private static final Map PAGE_LOADS_MAP = new ConcurrentHashMap<>(); + + public void setPageCacheSize(BookId bookIdId, CacheName cacheName, int value) { + if (cacheName == null) { + return; + } + PAGE_CACHE_SIZE_MAP.computeIfAbsent( + new LoadsKey(bookIdId, cacheName), key -> PAGE_CACHE_SIZE_GAUGE.labels(key.toLabels()) + ).set(value); + } + + public void incRequest(BookId bookId, CacheName cacheName, RequestMethod method) { + if (cacheName == null) { + return; + } + + PAGE_REQUEST_MAP.computeIfAbsent( + new PageRequestKey(bookId, cacheName, method), key -> PAGE_REQUEST_COUNTER.labels(key.toLabels()) + ).inc(); + } + + public void incInvalidate(BookId bookId, CacheName cacheName, RemovalCause cause) { + INVALIDATE_CACHE_MAP.computeIfAbsent( + new InvalidateKey(bookId, cacheName, cause), key -> INVALIDATE_CACHE_COUNTER.labels(key.toLabels()) + ).inc(); + } + + public void incLoads(BookId bookIdId, CacheName cacheName, int value) { + LoadsKey loadsKey = new LoadsKey(bookIdId, cacheName); + PAGE_LOADS_MAP.computeIfAbsent( + loadsKey, key -> PAGE_LOADS_COUNTER.labels(key.toLabels()) + ).observe(value); + } + + public enum RequestMethod { + GET, + NEXT, + PREVIOUS, + FIND, + ITERATE, + REFRESH, + } + + public enum CacheName { + HOT, + RANDOM + } + + private static class LoadsKey { + private final BookId bookIdId; + private final CacheName cacheName; + + private LoadsKey(BookId bookIdId, CacheName cacheName) { + this.bookIdId = bookIdId; + this.cacheName = cacheName; + } + + private String[] toLabels() { + return new String[]{bookIdId.getName(), cacheName.name()}; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LoadsKey that = (LoadsKey) o; + return Objects.equals(bookIdId, that.bookIdId) && cacheName == that.cacheName; + } + + @Override + public int hashCode() { + return Objects.hash(bookIdId, cacheName); + } + } + + private static class InvalidateKey { + private final BookId bookId; + private final CacheName cacheName; + private final RemovalCause cause; + + private InvalidateKey(BookId bookId, CacheName cacheName, RemovalCause cause) { + this.bookId = bookId; + this.cacheName = cacheName; + this.cause = cause; + } + + private String[] toLabels() { + return new String[]{bookId.getName(), cacheName.name(), cause.name()}; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + InvalidateKey that = (InvalidateKey) o; + return Objects.equals(bookId, that.bookId) && cacheName == that.cacheName && cause == that.cause; + } + + @Override + public int hashCode() { + return Objects.hash(bookId, cacheName, cause); + } + } + + private static class PageRequestKey { + private final BookId bookId; + private final CacheName cacheName; + private final RequestMethod method; + + private PageRequestKey(BookId bookId, CacheName cacheName, RequestMethod method) { + this.bookId = bookId; + this.cacheName = cacheName; + this.method = method; + } + + private String[] toLabels() { + return new String[]{bookId.getName(), cacheName.name(), method.name()}; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PageRequestKey that = (PageRequestKey) o; + return Objects.equals(bookId, that.bookId) && method == that.method && cacheName == that.cacheName; + } + + @Override + public int hashCode() { + return Objects.hash(bookId, method, cacheName); + } + } +} diff --git a/cradle-core/src/main/java/com/exactpro/cradle/BookManager.java b/cradle-core/src/main/java/com/exactpro/cradle/BookManager.java index 5a4e7ab30..316516cca 100644 --- a/cradle-core/src/main/java/com/exactpro/cradle/BookManager.java +++ b/cradle-core/src/main/java/com/exactpro/cradle/BookManager.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import com.exactpro.cradle.utils.CradleStorageException; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -68,19 +69,17 @@ private static class Refresher implements Runnable { public void run() { logger.debug("Refreshing books"); try { - List cachedBookInfos = new ArrayList<>(bookCache.getCachedBooks()); - List oldBookInfos = new ArrayList<>(cachedBookInfos); + List bookIds = bookCache.getCachedBooks().stream() + .map(BookInfo::getId) + .collect(Collectors.toList()); - for (BookInfo oldBookInfo : oldBookInfos) { + for (BookId bookId : bookIds) { try { - BookInfo newBookInfo = bookCache.loadBook(oldBookInfo.getId()); - if (!oldBookInfo.equals(newBookInfo)) { - logger.info("Refreshing book {}", newBookInfo.getId().getName()); - bookCache.updateCachedBook(newBookInfo); - } - + BookInfo bookInfo = bookCache.getBook(bookId); + logger.debug("Refreshing book {}", bookInfo.getId().getName()); + bookInfo.refresh(); } catch (CradleStorageException e) { - logger.error("Refresher could not get new book info for {}: {}", oldBookInfo.getId().getName(), e.getMessage()); + logger.error("Refresher could not get new book info for {}: {}", bookId.getName(), e.getMessage()); } } } catch (Exception e) { diff --git a/cradle-core/src/main/java/com/exactpro/cradle/CoreStorageSettings.java b/cradle-core/src/main/java/com/exactpro/cradle/CoreStorageSettings.java index f0b9b30a9..92d2f77cd 100644 --- a/cradle-core/src/main/java/com/exactpro/cradle/CoreStorageSettings.java +++ b/cradle-core/src/main/java/com/exactpro/cradle/CoreStorageSettings.java @@ -22,6 +22,7 @@ @SuppressWarnings("unused") public class CoreStorageSettings { public static final long DEFAULT_BOOK_REFRESH_INTERVAL_MILLIS = 60000; + public static final int PAGE_ACTION_REJECTION_THRESHOLD_FACTOR = 2; public static final boolean DEFAULT_STORE_INDIVIDUAL_MESSAGE_SESSIONS = true; private long bookRefreshIntervalMillis = DEFAULT_BOOK_REFRESH_INTERVAL_MILLIS; @@ -41,7 +42,7 @@ public void setBookRefreshIntervalMillis(long bookRefreshIntervalMillis) { } public long calculatePageActionRejectionThreshold() { - return this.bookRefreshIntervalMillis * 2; + return this.bookRefreshIntervalMillis * PAGE_ACTION_REJECTION_THRESHOLD_FACTOR; } public long calculateStoreActionRejectionThreshold() { diff --git a/cradle-core/src/main/java/com/exactpro/cradle/CradleStorage.java b/cradle-core/src/main/java/com/exactpro/cradle/CradleStorage.java index 5bfffb4a8..b7ef08071 100644 --- a/cradle-core/src/main/java/com/exactpro/cradle/CradleStorage.java +++ b/cradle-core/src/main/java/com/exactpro/cradle/CradleStorage.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +33,6 @@ import com.exactpro.cradle.messages.StoredMessageBatch; import com.exactpro.cradle.messages.StoredMessageId; import com.exactpro.cradle.resultset.CradleResultSet; -import com.exactpro.cradle.resultset.EmptyResultSet; import com.exactpro.cradle.testevents.StoredTestEvent; import com.exactpro.cradle.testevents.StoredTestEventId; import com.exactpro.cradle.testevents.TestEventBatchToStore; @@ -56,16 +55,21 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; +import java.util.stream.Collectors; + +import static com.exactpro.cradle.Order.DIRECT; +import static com.exactpro.cradle.Order.REVERSE; +import static com.exactpro.cradle.resultset.EmptyResultSet.emptyResultSet; /** * Storage which holds information about all data sent or received and test events. @@ -390,15 +394,14 @@ public BookInfo addBook(BookToAdd book) throws CradleStorageException, IOExcepti BookId id = new BookId(book.getName()); logger.info("Adding book '{}' to storage", id); - if (checkBook(id)) + if (checkBook(id)) { throw new CradleStorageException("Book '" + id + "' is already present in storage"); + } doAddBook(book, id); - BookInfo newBook = new BookInfo(id, book.getFullName(), book.getDesc(), book.getCreated(), null); - getBookCache().updateCachedBook(newBook); + BookInfo newBook = getBookCache().getBook(id); logger.info("Book '{}' has been added to storage", id); - return newBook; } @@ -412,6 +415,10 @@ public Collection listBooks() { return doListBooks(); } + public BookInfo getBook(BookId bookId) throws CradleStorageException { + return getBookCache().getBook(bookId); + } + /** * @return collection of books currently available in storage */ @@ -448,11 +455,17 @@ public BookInfo addPage(BookId bookId, String pageName, Instant pageStart, Strin public BookInfo addPages(BookId bookId, List pages) throws CradleStorageException, IOException { logger.info("Adding pages {} to book '{}'", pages, bookId); - BookInfo book = refreshPages(bookId); - if (pages == null || pages.isEmpty()) + BookInfo book = getBookCache().getBook(bookId); + if (pages == null || pages.isEmpty()) { return book; + } + + Set timestamps = pages.stream() + .map(PageToAdd::getStart) + .collect(Collectors.toSet()); + book.invalidate(timestamps); - List toAdd = checkPages(pages, book); + List toAdd = checkAndConvertPages(pages, book); PageInfo bookLastPage = book.getLastPage(); PageInfo endedPage = null; @@ -465,9 +478,9 @@ public BookInfo addPages(BookId bookId, List pages) throws CradleStor if (bookLastPage != null && bookLastPage.getEnded() == null && lastPageToAdd != null - && lastPageToAdd.getStarted().isAfter(bookLastPage.getStarted())) { + && lastPageToAdd.getStarted().isAfter(bookLastPage.getId().getStart())) { - endedPage = PageInfo.ended(bookLastPage, toAdd.get(0).getStarted()); + endedPage = PageInfo.ended(bookLastPage, toAdd.get(0).getId().getStart()); } try { @@ -478,10 +491,10 @@ public BookInfo addPages(BookId bookId, List pages) throws CradleStor throw e; } - if (endedPage != null) - book.addPage(endedPage); //Replacing last page with ended one, i.e. updating last page info - for (PageInfo newPage : toAdd) - book.addPage(newPage); + if (endedPage != null) { + book.invalidate(endedPage.getId().getStart()); + } + book.invalidate(timestamps); return book; } @@ -493,14 +506,11 @@ public BookInfo addPages(BookId bookId, List pages) throws CradleStor * @param bookId ID of the book whose pages to refresh * @return refreshed book information * @throws CradleStorageException if given bookId is unknown - * @throws IOException if page data reading failed */ - public BookInfo refreshPages(BookId bookId) throws CradleStorageException, IOException { + public BookInfo refreshPages(BookId bookId) throws CradleStorageException { logger.info("Refreshing pages of book '{}'", bookId); BookInfo book = getBookCache().getBook(bookId); - Collection pages = doLoadPages(bookId); - book = new BookInfo(book.getId(), book.getFullName(), book.getDesc(), book.getCreated(), pages); - getBookCache().updateCachedBook(book); + book.refresh(); return book; } @@ -530,13 +540,7 @@ public Collection getAllPages(BookId bookId) throws CradleStorageExcep public BookInfo refreshBook(String name) throws CradleStorageException { logger.info("Refreshing book {} from storage", name); - BookInfo bookInfo = getBookCache().loadBook(new BookId(name)); - - if (bookInfo != null) { - getBookCache().updateCachedBook(bookInfo); - } - - return bookInfo; + return refreshPages(new BookId(name)); } /** @@ -551,14 +555,15 @@ public BookInfo removePage(PageId pageId) throws CradleStorageException, IOExcep logger.info("Removing page '{}'", pageId); BookId bookId = pageId.getBookId(); - BookInfo book = refreshPages(bookId); + BookInfo book = getBookCache().getBook(bookId); + book.invalidate(pageId.getStart()); - String pageName = pageId.getName(); PageInfo page = book.getPage(pageId); - if (page == null) - throw new CradleStorageException("Page '" + pageName + "' is not present in book '" + bookId + "'"); + if (page == null) { // TODO: Should we check page existing ? + throw new CradleStorageException("Page '" + pageId.getStart() + "' is not present in book '" + bookId + "'"); + } doRemovePage(page); - book.removePage(pageId); + book.invalidate(pageId.getStart()); logger.info("Page '{}' has been removed", pageId); return book; } @@ -647,7 +652,7 @@ List> paginateBatch(GroupedMessageBat boolean singlePageBatch = true; for (var message : batch.getMessages()) { Instant ts = message.getTimestamp(); - if (!lastPage.isValidFor(ts)) { + if (lastPage.isNotValidFor(ts)) { lastPage = findPage(bookId, ts); singlePageBatch = false; } @@ -737,7 +742,7 @@ TestEventToStore alignEventTimestampsToPage(TestEventToStore event, PageInfo pag Map idMappings = new HashMap<>(); batch.getTestEvents().forEach((e) -> { - if (!page.isValidFor(e.getStartTimestamp())) { + if (page.isNotValidFor(e.getStartTimestamp())) { StoredTestEventId id = e.getId(); idMappings.put(id, new StoredTestEventId(id.getBookId(), id.getScope(), page.getEnded().minusNanos(1), id.getId())); } @@ -919,7 +924,7 @@ protected final CompletableFuture getMessageBatchAsync(Store public final CradleResultSet getMessages(MessageFilter filter) throws IOException, CradleStorageException { logger.debug("Filtering messages by {}", filter); if (!checkFilter(filter)) - return new EmptyResultSet<>(); + return emptyResultSet(); BookInfo book = getBookCache().getBook(filter.getBookId()); CradleResultSet result = doGetMessages(filter, book); @@ -937,7 +942,7 @@ public final CradleResultSet getMessages(MessageFilter filter) th public final CompletableFuture> getMessagesAsync(MessageFilter filter) throws CradleStorageException { logger.debug("Asynchronously getting messages filtered by {}", filter); if (!checkFilter(filter)) - return CompletableFuture.completedFuture(new EmptyResultSet<>()); + return CompletableFuture.completedFuture(emptyResultSet()); BookInfo book = getBookCache().getBook(filter.getBookId()); CompletableFuture> result = doGetMessagesAsync(filter, book); @@ -962,7 +967,7 @@ public final CompletableFuture> getMessagesAsync( public final CradleResultSet getMessageBatches(MessageFilter filter) throws IOException, CradleStorageException { logger.debug("Filtering message batches by {}", filter); if (!checkFilter(filter)) - return new EmptyResultSet<>(); + return emptyResultSet(); BookInfo book = getBookCache().getBook(filter.getBookId()); CradleResultSet result = doGetMessageBatches(filter, book); @@ -982,7 +987,9 @@ public final CradleResultSet getMessageBatches(MessageFilter public final CradleResultSet getGroupedMessageBatches(GroupedMessageFilter filter) throws CradleStorageException, IOException { logger.debug("Filtering grouped message batches by {}", filter); - checkAbstractFilter(filter); + if (!checkFilter(filter)) { + return emptyResultSet(); + } BookInfo book = getBookCache().getBook(filter.getBookId()); CradleResultSet result = doGetGroupedMessageBatches(filter, book); @@ -1001,7 +1008,7 @@ public final CradleResultSet getGroupedMessageBatches public final CompletableFuture> getMessageBatchesAsync(MessageFilter filter) throws CradleStorageException { logger.debug("Asynchronously getting message batches filtered by {}", filter); if (!checkFilter(filter)) - return CompletableFuture.completedFuture(new EmptyResultSet<>()); + return CompletableFuture.completedFuture(emptyResultSet()); BookInfo book = getBookCache().getBook(filter.getBookId()); CompletableFuture> result = doGetMessageBatchesAsync(filter, book); @@ -1024,7 +1031,9 @@ public final CompletableFuture> getMessageBa */ public final CompletableFuture> getGroupedMessageBatchesAsync(GroupedMessageFilter filter) throws CradleStorageException { logger.debug("Asynchronously getting grouped message batches filtered by {}", filter); - checkAbstractFilter(filter); + if (!checkFilter(filter)) { + return CompletableFuture.completedFuture(emptyResultSet()); + } BookInfo book = getBookCache().getBook(filter.getBookId()); CompletableFuture> result = doGetGroupedMessageBatchesAsync(filter, book); @@ -1153,7 +1162,7 @@ public final CompletableFuture getTestEventAsync(StoredTestEven public final CradleResultSet getTestEvents(TestEventFilter filter) throws CradleStorageException, IOException { logger.debug("Filtering test events by {}", filter); if (!checkFilter(filter)) - return new EmptyResultSet<>(); + return emptyResultSet(); BookInfo book = getBookCache().getBook(filter.getBookId()); CradleResultSet result = doGetTestEvents(filter, book); @@ -1171,7 +1180,7 @@ public final CradleResultSet getTestEvents(TestEventFilter filt public final CompletableFuture> getTestEventsAsync(TestEventFilter filter) throws CradleStorageException { logger.debug("Asynchronously getting test events filtered by {}", filter); if (!checkFilter(filter)) - return CompletableFuture.completedFuture(new EmptyResultSet<>()); + return CompletableFuture.completedFuture(emptyResultSet()); BookInfo book = getBookCache().getBook(filter.getBookId()); CompletableFuture> result = doGetTestEventsAsync(filter, book); @@ -1434,11 +1443,11 @@ public CradleResultSet getSessionGroups(BookId bookId, Interval interval * @throws CradleStorageException Page was edited but cache wasn't refreshed, try to refresh pages */ public PageInfo updatePageComment(BookId bookId, String pageName, String comment) throws CradleStorageException { - getBookCache().getBook(bookId); PageInfo updatedPageInfo = doUpdatePageComment(bookId, pageName, comment); try { - updatePage(new PageId(bookId, pageName), updatedPageInfo); + BookInfo bookInfo = getBookCache().getBook(bookId); + bookInfo.invalidate(updatedPageInfo.getStarted()); } catch (Exception e) { logger.error("Page was edited but cache wasn't refreshed, try to refresh pages"); throw e; @@ -1457,11 +1466,11 @@ public PageInfo updatePageComment(BookId bookId, String pageName, String comment * @throws CradleStorageException Page was edited but cache wasn't refreshed, try to refresh pages */ public PageInfo updatePageName(BookId bookId, String pageName, String newPageName) throws CradleStorageException { - getBookCache().getBook(bookId); PageInfo updatedPageInfo = doUpdatePageName(bookId, pageName, newPageName); try { - updatePage(new PageId(bookId, pageName), updatedPageInfo); + BookInfo bookInfo = getBookCache().getBook(bookId); + bookInfo.invalidate(updatedPageInfo.getStarted()); } catch (Exception e) { logger.error("Page was edited but cache wasn't refreshed, try to refresh pages"); throw e; @@ -1520,13 +1529,6 @@ public CompletableFuture> getScopesAsync(BookId bookId, return doGetScopesAsync(bookId, interval); } - private void updatePage(PageId pageId, PageInfo updatedPageInfo) throws CradleStorageException { - BookInfo bookInfo = getBookCache().getBook(pageId.getBookId()); - - bookInfo.removePage(pageId); - bookInfo.addPage(updatedPageInfo); - } - public final void updateEventStatus(StoredTestEvent event, boolean success) throws IOException { logger.debug("Updating status of event {}", event.getId()); doUpdateEventStatus(event, success); @@ -1546,24 +1548,25 @@ public final CompletableFuture updateEventStatusAsync(StoredTestEvent even } private Instant checkCollisionAndGetPageEnd(BookInfo book, PageToAdd page, Instant defaultPageEnd) throws CradleStorageException { - PageInfo pageInBookBeforeStart = book.findPage(page.getStart()); + Iterator reverseIterator = book.getPages(null, page.getStart(), REVERSE); + PageInfo pageBefore = reverseIterator.hasNext() ? reverseIterator.next() : null; - if (pageInBookBeforeStart != null - && pageInBookBeforeStart.getEnded() != null - && pageInBookBeforeStart.getEnded().isAfter(page.getStart())) { + if (pageBefore != null + && pageBefore.getEnded() != null + && pageBefore.getEnded().isAfter(page.getStart())) { throw new CradleStorageException(String.format("Can't add new page in book %s, it collided with current page %s %s-%s", book.getId().getName(), - pageInBookBeforeStart.getId().getName(), - pageInBookBeforeStart.getStarted(), - pageInBookBeforeStart.getEnded())); + pageBefore.getName(), + pageBefore.getStarted(), + pageBefore.getEnded())); } - PageInfo pageInBookAfterStart = book.getNextPage(page.getStart()); - - return pageInBookAfterStart == null ? defaultPageEnd : pageInBookAfterStart.getStarted(); + Iterator directIterator = book.getPages(page.getStart(), null, DIRECT); + PageInfo pageAfter = directIterator.hasNext() ? directIterator.next() : null; + return pageAfter == null || Objects.equals(pageAfter, pageBefore) ? defaultPageEnd : pageAfter.getStarted(); } - private List checkPages(List pages, BookInfo book) throws CradleStorageException { + private List checkAndConvertPages(List pages, BookInfo book) throws CradleStorageException { PageInfo lastPage = book.getLastPage(); if (lastPage != null) //If book has any pages, i.e. may have some data { @@ -1577,7 +1580,6 @@ private List checkPages(List pages, BookInfo book) throws C nowPlusThreshold)); } - Set names = new HashSet<>(); PageToAdd prevPage = null; BookId bookId = book.getId(); List result = new ArrayList<>(pages.size()); @@ -1585,20 +1587,15 @@ private List checkPages(List pages, BookInfo book) throws C BookPagesNamesChecker.validatePageName(page.getName()); String name = page.getName(); - if (names.contains(name)) - throw new CradleStorageException("Duplicated page name: '" + page.getName() + "'"); - names.add(name); - - if (book.getPage(new PageId(bookId, name)) != null) - throw new CradleStorageException("Page '" + name + "' is already present in book '" + bookId + "'"); + if (book.getPage(new PageId(bookId, page.getStart(), name)) != null) + throw new CradleStorageException("Page '" + name + ":"+page.getStart()+"' is already present in book '" + bookId + "'"); if (prevPage != null) { if (!page.getStart().isAfter(prevPage.getStart())) { throw new CradleStorageException("Unordered pages: page '" + name + "' should start after page '" + prevPage.getName() + "'"); } - result.add(new PageInfo(new PageId(bookId, prevPage.getName()), - prevPage.getStart(), + result.add(new PageInfo(new PageId(bookId, prevPage.getStart(), prevPage.getName()), checkCollisionAndGetPageEnd(book, prevPage, page.getStart()), prevPage.getComment())); } @@ -1606,8 +1603,7 @@ private List checkPages(List pages, BookInfo book) throws C } if (prevPage != null) { - result.add(new PageInfo(new PageId(bookId, prevPage.getName()), - prevPage.getStart(), + result.add(new PageInfo(new PageId(bookId, prevPage.getStart(), prevPage.getName()), checkCollisionAndGetPageEnd(book, prevPage, null), prevPage.getComment())); } @@ -1622,15 +1618,23 @@ private boolean checkFilter(MessageFilter filter) throws CradleStorageException return true; } + private boolean checkFilter(GroupedMessageFilter filter) throws CradleStorageException { + checkAbstractFilter(filter); + + //TODO: add more checks + return true; + } + private void checkAbstractFilter(AbstractFilter filter) throws CradleStorageException { BookInfo book = getBookCache().getBook(filter.getBookId()); - if (filter.getPageId() != null) + if (filter.getPageId() != null) { checkPage(filter.getPageId(), book.getId()); + } } private boolean checkFilter(TestEventFilter filter) throws CradleStorageException { - BookInfo book = getBookCache().getBook(filter.getBookId()); checkAbstractFilter(filter); + BookInfo book = getBookCache().getBook(filter.getBookId()); if (filter.getParentId() != null && !book.getId().equals(filter.getParentId().getBookId())) throw new CradleStorageException("Requested book (" + book.getId() + ") doesn't match book of requested parent (" + filter.getParentId() + ")"); @@ -1650,10 +1654,12 @@ private boolean checkFilter(TestEventFilter filter) throws CradleStorageExceptio Instant pageStarted = page.getStarted(), pageEnded = page.getEnded(); - if (timeFrom != null && pageEnded != null && timeFrom.isAfter(pageEnded)) + if (timeFrom != null && pageEnded != null && timeFrom.isAfter(pageEnded)) { return false; - if (timeTo != null && timeTo.isBefore(pageStarted)) + } + if (timeTo != null && timeTo.isBefore(pageStarted)) { return false; + } } return true; @@ -1666,21 +1672,25 @@ public boolean checkBook(BookId bookId) { public PageInfo findPage(BookId bookId, Instant timestamp) throws CradleStorageException { BookInfo book = getBookCache().getBook(bookId); PageInfo page = book.findPage(timestamp); - if (page == null || (page.getEnded() != null && !timestamp.isBefore(page.getEnded()))) //If page.getEnded().equals(timestamp), timestamp is outside of page + if (page == null || (page.getEnded() != null && !timestamp.isBefore(page.getEnded()))) { //If page.getEnded().equals(timestamp), timestamp is outside of page throw new PageNotFoundException(String.format("Book '%s' has no page for timestamp %s", bookId, timestamp)); + } return page; } public void checkPage(PageId pageId, BookId bookFromId) throws CradleStorageException { - BookInfo book = getBookCache().getBook(bookFromId); - if (!bookFromId.equals(pageId.getBookId())) + if (!bookFromId.equals(pageId.getBookId())) { throw new CradleStorageException("Requested book (" + bookFromId + ") doesn't match book of requested page (" + pageId.getBookId() + ")"); - if (book.getPage(pageId) == null) + } + BookInfo book = getBookCache().getBook(bookFromId); + if (book.getPage(pageId) == null) { throw new CradleStorageException("Page '" + pageId + "' is unknown"); + } } public void checkPage(PageId pageId) throws CradleStorageException { - if (getBookCache().getBook(pageId.getBookId()).getPage(pageId) == null) + if (getBookCache().getBook(pageId.getBookId()).getPage(pageId) == null) { throw new CradleStorageException("Page '" + pageId + "' is unknown"); + } } } \ No newline at end of file diff --git a/cradle-core/src/main/java/com/exactpro/cradle/DummyCradleStorage.java b/cradle-core/src/main/java/com/exactpro/cradle/DummyCradleStorage.java index 98af55f5d..87f7b93f0 100644 --- a/cradle-core/src/main/java/com/exactpro/cradle/DummyCradleStorage.java +++ b/cradle-core/src/main/java/com/exactpro/cradle/DummyCradleStorage.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -72,16 +72,6 @@ public Collection loadPageInfo(BookId bookId, boolean loadRemoved) { return null; } - @Override - public BookInfo loadBook(BookId bookId) { - return null; - } - - @Override - public void updateCachedBook(BookInfo bookInfo) { - books.put(bookInfo.getId(), bookInfo); - } - @Override public Collection getCachedBooks() { return null; diff --git a/cradle-core/src/main/java/com/exactpro/cradle/PageId.java b/cradle-core/src/main/java/com/exactpro/cradle/PageId.java index 79e7c87d6..0d8467d37 100644 --- a/cradle-core/src/main/java/com/exactpro/cradle/PageId.java +++ b/cradle-core/src/main/java/com/exactpro/cradle/PageId.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,15 @@ package com.exactpro.cradle; +import java.time.Instant; import java.util.Objects; import com.exactpro.cradle.utils.EscapeUtils; +import javax.annotation.Nonnull; + +import static java.util.Objects.requireNonNull; + /** * Identifier of the page within a book */ @@ -27,21 +32,30 @@ public class PageId { public static final String DELIMITER = EscapeUtils.DELIMITER_STR; - private final BookId bookId; - private final String name; - - public PageId(BookId bookId, String pageName) + private final @Nonnull BookId bookId; + private final @Nonnull Instant start; + private final @Nonnull String name; + + public PageId(BookId bookId, Instant start, String pageName) { - this.bookId = bookId; - this.name = pageName; + this.bookId = requireNonNull(bookId, "Book id can't be null"); + this.start = requireNonNull(start, "Start timestamp can't be null"); + this.name = requireNonNull(pageName, "Page name can't be null"); } + @Nonnull public BookId getBookId() { return bookId; } - + + @Nonnull + public Instant getStart() { + return start; + } + + @Nonnull public String getName() { return name; @@ -51,7 +65,7 @@ public String getName() @Override public int hashCode() { - return Objects.hash(bookId, name); + return Objects.hash(bookId, start, name); } @Override @@ -64,13 +78,17 @@ public boolean equals(Object obj) if (getClass() != obj.getClass()) return false; PageId other = (PageId) obj; - return Objects.equals(bookId, other.bookId) && Objects.equals(name, other.name); + return Objects.equals(bookId, other.bookId) && + Objects.equals(start, other.start) && + Objects.equals(name, other.name); } @Override public String toString() { - return EscapeUtils.escape(bookId.toString())+DELIMITER+EscapeUtils.escape(name); + return EscapeUtils.escape(bookId.toString())+DELIMITER+ + EscapeUtils.escape(start.toString())+DELIMITER+ + EscapeUtils.escape(name); } } diff --git a/cradle-core/src/main/java/com/exactpro/cradle/PageInfo.java b/cradle-core/src/main/java/com/exactpro/cradle/PageInfo.java index 240f82469..d3c760fc5 100644 --- a/cradle-core/src/main/java/com/exactpro/cradle/PageInfo.java +++ b/cradle-core/src/main/java/com/exactpro/cradle/PageInfo.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,26 +25,23 @@ public class PageInfo { private final PageId id; - private final Instant started, - ended; + private final Instant ended; private final String comment; private final Instant updated; private final Instant removed; - public PageInfo(PageId id, Instant started, Instant ended, String comment, Instant updated, Instant removed) + public PageInfo(PageId id, Instant ended, String comment, Instant updated, Instant removed) { this.id = id; - this.started = started; this.ended = ended; this.comment = comment; this.updated = updated; this.removed = removed; } - public PageInfo(PageId id, Instant started, Instant ended, String comment) + public PageInfo(PageId id, Instant ended, String comment) { this.id = id; - this.started = started; this.ended = ended; this.comment = comment; this.updated = null; @@ -56,17 +53,17 @@ public PageId getId() { return id; } - + public Instant getStarted() { - return started; + return id.getStart(); } - + public Instant getEnded() { return ended; } - + public String getComment() { return comment; @@ -80,14 +77,29 @@ public Instant getRemoved() { return removed; } + + public String getBookName() { + return id.getBookId().getName(); + } + + public String getName() { + return id.getName(); + } + public static PageInfo ended(PageInfo page, Instant endTimestamp) { - return page == null ? null : new PageInfo(page.getId(), page.getStarted(), endTimestamp, page.getComment(), page.getUpdated(), page.getRemoved()); + return page == null ? null : new PageInfo(page.getId(), endTimestamp, page.getComment(), page.getUpdated(), page.getRemoved()); + } + + public boolean isNotValidFor(Instant timestamp) { + return (getStarted() != null && getStarted().isAfter(timestamp)) || + (ended != null && !ended.isAfter(timestamp)); } + // Backward compatibility + @SuppressWarnings("unused") public boolean isValidFor(Instant timestamp) { - return (started == null || !started.isAfter(timestamp)) && - (ended == null || ended.isAfter(timestamp)); + return !isNotValidFor(timestamp); } @Override @@ -96,7 +108,6 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; PageInfo pageInfo = (PageInfo) o; return getId().equals(pageInfo.getId()) - && getStarted().equals(pageInfo.getStarted()) && Objects.equals(getEnded(), pageInfo.getEnded()) && Objects.equals(getComment(), pageInfo.getComment()) && Objects.equals(getUpdated(), pageInfo.getUpdated()) @@ -107,7 +118,6 @@ && getStarted().equals(pageInfo.getStarted()) public String toString() { return "PageInfo{" + "id=" + id + - ", started=" + started + ", ended=" + ended + ", comment='" + comment + '\'' + ", updated=" + updated + diff --git a/cradle-core/src/main/java/com/exactpro/cradle/PagesLoader.java b/cradle-core/src/main/java/com/exactpro/cradle/PagesLoader.java new file mode 100644 index 000000000..be4cfaa81 --- /dev/null +++ b/cradle-core/src/main/java/com/exactpro/cradle/PagesLoader.java @@ -0,0 +1,30 @@ +/* + * Copyright 2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.cradle; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.time.Instant; +import java.util.Collection; + +@FunctionalInterface +public interface PagesLoader { + /** + * @return ordered page info collection from start to end date time. Both borders are included + */ + @Nonnull + Collection load(@Nonnull BookId bookId, @Nullable Instant start, @Nullable Instant end); +} diff --git a/cradle-core/src/main/java/com/exactpro/cradle/filters/AbstractFilter.java b/cradle-core/src/main/java/com/exactpro/cradle/filters/AbstractFilter.java index 5c630e7da..c8d72e58b 100644 --- a/cradle-core/src/main/java/com/exactpro/cradle/filters/AbstractFilter.java +++ b/cradle-core/src/main/java/com/exactpro/cradle/filters/AbstractFilter.java @@ -52,7 +52,7 @@ protected AbstractFilter(AbstractFilter copyFrom) { this.from = copyFrom.getFrom(); this.to = copyFrom.getTo(); if(copyFrom.getLimit() < 0){ - throw new IllegalArgumentException("Invalid limit value: " + copyFrom.getLimit() + ". limit must be greater than 0 )"); + throw new IllegalArgumentException("Invalid limit value: " + copyFrom.getLimit() + ". limit must be greater than 0"); } this.limit = copyFrom.getLimit(); this.order = copyFrom.getOrder(); @@ -94,7 +94,7 @@ public int getLimit() { public void setLimit(int limit) { if(limit < 0){ - throw new IllegalArgumentException("Invalid limit value: " + limit + ". limit must be greater than 0 )"); + throw new IllegalArgumentException("Invalid limit value: " + limit + ". limit must be greater than 0"); } this.limit = limit; } diff --git a/cradle-core/src/main/java/com/exactpro/cradle/resultset/EmptyResultSet.java b/cradle-core/src/main/java/com/exactpro/cradle/resultset/EmptyResultSet.java index b48fb5e79..d349bf2cd 100644 --- a/cradle-core/src/main/java/com/exactpro/cradle/resultset/EmptyResultSet.java +++ b/cradle-core/src/main/java/com/exactpro/cradle/resultset/EmptyResultSet.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,8 +16,11 @@ package com.exactpro.cradle.resultset; -public class EmptyResultSet implements CradleResultSet -{ +public class EmptyResultSet implements CradleResultSet { + public static final EmptyResultSet EMPTY_RESULT_SET = new EmptyResultSet<>(); + + private EmptyResultSet() { } + @Override public boolean hasNext() { @@ -29,4 +32,9 @@ public T next() { return null; } + + @SuppressWarnings("unchecked") + public static EmptyResultSet emptyResultSet() { + return (EmptyResultSet) EMPTY_RESULT_SET; + } } diff --git a/cradle-core/src/main/java/com/exactpro/cradle/utils/TestEventUtils.java b/cradle-core/src/main/java/com/exactpro/cradle/utils/TestEventUtils.java index a21efc72e..77e6663d6 100644 --- a/cradle-core/src/main/java/com/exactpro/cradle/utils/TestEventUtils.java +++ b/cradle-core/src/main/java/com/exactpro/cradle/utils/TestEventUtils.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -63,7 +63,7 @@ public static void validateTestEvent(TestEvent event, BookInfo bookInfo, long st PageInfo pageInfo = bookInfo.findPage(event.getParentId().getStartTimestamp()); if (pageInfo == null) { throw new CradleStorageException( - String.format("Test event's parent event's startTimestamp is %s , could not find corresponding page in book %s", + String.format("Test event's parent event's startTimestamp is %s, could not find corresponding page in book %s", event.getParentId().getStartTimestamp(), bookInfo.getId())); } diff --git a/cradle-core/src/test/java/com/exactpro/cradle/BookInfoTest.java b/cradle-core/src/test/java/com/exactpro/cradle/BookInfoTest.java new file mode 100644 index 000000000..6eabd9aec --- /dev/null +++ b/cradle-core/src/test/java/com/exactpro/cradle/BookInfoTest.java @@ -0,0 +1,645 @@ +/* + * Copyright 2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.cradle; + +import com.google.common.collect.Lists; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import javax.annotation.Nullable; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Random; + +import static com.exactpro.cradle.Order.DIRECT; +import static com.exactpro.cradle.Order.REVERSE; +import static com.google.common.collect.Lists.newArrayList; +import static java.time.temporal.ChronoUnit.DAYS; +import static java.time.temporal.ChronoUnit.HOURS; +import static java.time.temporal.ChronoUnit.NANOS; +import static java.util.Collections.emptyList; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertSame; + +public class BookInfoTest { + private static final Logger LOGGER = LoggerFactory.getLogger(BookInfoTest.class); + public static Random RANDOM = new Random(); + public static final BookId BOOK_ID = new BookId("test-book"); + private static final List PAGES; + + static { + List pages = new ArrayList<>(); + Instant start = Instant.now().minus(7, ChronoUnit.DAYS); + Instant end = Instant.now(); + Instant current = start; + Instant previous; + do { + previous = current; + current = current.plus(1, ChronoUnit.HOURS); + pages.add(createPageInfo(previous, current)); + } while (current.isBefore(end)); + pages.add(createPageInfo(current, null)); + + // Add random gap + int index = RANDOM.nextInt(pages.size() - 2) + 1; + PageInfo pageInfo = pages.remove(index); + + PAGES = Collections.unmodifiableList(pages); + LOGGER.info("Pages - min: {}, gap: {}:{}, max: {}, size: {}", + PAGES.get(0).getStarted(), + index, pageInfo.getStarted(), + PAGES.get(PAGES.size() - 1).getStarted(), + PAGES.size() + ); + } + + @Test(dataProvider = "cacheSize") + public void lazyPageAddTest(int cacheSize) { + List operateSource = new ArrayList<>(); + BookInfo bookInfo = createBookInfo(operateSource, cacheSize); + + assertNull(bookInfo.getFirstPage()); + assertNull(bookInfo.getLastPage()); + assertEquals(bookInfo.getPages(), emptyList()); + + for (int addIndex = 0; addIndex < PAGES.size(); addIndex++) { + PageInfo newPage = PAGES.get(addIndex); + operateSource.add(newPage); + bookInfo.invalidate(newPage.getStarted()); + + assertSame(bookInfo.getFirstPage(), PAGES.get(0), "iteration - " + addIndex); + assertSame(bookInfo.getLastPage(), newPage, "iteration - " + addIndex); + assertEquals(bookInfo.getPages(), PAGES.subList(0, addIndex + 1), "iteration - " + addIndex); + + for (int checkIndex = addIndex; checkIndex >= 0; checkIndex--) { + PageInfo source = PAGES.get(checkIndex); + assertSame(bookInfo.getPage(source.getId()), source, "iteration - " + addIndex + '.' + checkIndex); + assertSame(bookInfo.findPage(source.getId().getStart()), source, "iteration - " + addIndex + '.' + checkIndex); + + if (source.getEnded() == null) { + assertSame(bookInfo.findPage(Instant.MAX), source, "iteration - " + addIndex + '.' + checkIndex); + } else { + assertSame(bookInfo.findPage(source.getEnded().minus(1, NANOS)), source, "iteration - " + addIndex + '.' + checkIndex); + } + + if (checkIndex > 0) { + assertSame(bookInfo.getPreviousPage(source.getId().getStart()), PAGES.get(checkIndex - 1), "iteration - " + addIndex + '.' + checkIndex); + } else { + assertNull(bookInfo.getPreviousPage(source.getId().getStart()), "iteration - " + addIndex + '.' + checkIndex + ", timestamp: " + source.getId().getStart()); + } + + if (checkIndex < addIndex) { + assertSame(bookInfo.getNextPage(source.getId().getStart()), PAGES.get(checkIndex + 1), "iteration - " + addIndex + '.' + checkIndex); + } else { + assertNull(bookInfo.getNextPage(source.getId().getStart()), "iteration - " + addIndex + '.' + checkIndex + ", timestamp: " + source.getId().getStart()); + } + } + } + } + + @Test(dataProvider = "orders") + public void emptyBookTest(Order order) { + BookInfo bookInfo = createBookInfo(emptyList(), 1); + + assertNull(bookInfo.getFirstPage()); + assertNull(bookInfo.getLastPage()); + assertEquals(newArrayList(bookInfo.getPages()), emptyList()); + assertEquals(newArrayList(bookInfo.getPages(Instant.MIN, Instant.MAX, order)), emptyList()); + assertNull(bookInfo.findPage(Instant.now())); + assertNull(bookInfo.getPage(PAGES.get(0).getId())); + } + + @Test(dataProvider = "orders") + public void getAllPagesTest(Order order) { + List operateSource = new ArrayList<>(PAGES); + BookInfo bookInfo = createBookInfo(operateSource, 1); + + Iterator iterator = bookInfo.getPages(null, null, order); + assertEquals(newArrayList(iterator), optionalReverse(order, operateSource)); + } + + @Test(dataProvider = "orderToPages") + public void getPagesTest(Order order, List pageTimestamps) { + Instant time1 = pageTimestamps.get(0); + Instant time2 = pageTimestamps.get(1); + Instant time3 = pageTimestamps.get(2); + Instant time4 = pageTimestamps.get(3); + List operateSource = List.of( + createPageInfo(time1, time2), + createPageInfo(time2, time3), + createPageInfo(time3, time4) + ); + BookInfo bookInfo = createBookInfo(operateSource, 1); + + assertEquals( + newArrayList(bookInfo.getPages(time2.minus(1, NANOS), time3.minus(1, NANOS), order)), + optionalReverse(order, operateSource.subList(0, 2)), + "Pages where start (-1) to end (-1) timestamps" + ); + assertEquals( + newArrayList(bookInfo.getPages(time2, time3.minus(1, NANOS), order)), + optionalReverse(order, operateSource.subList(1, 2)), + "Pages where start (0) to end (-1) timestamps" + ); + assertEquals( + newArrayList(bookInfo.getPages(time2.plus(1, NANOS), time3.minus(1, NANOS), order)), + optionalReverse(order, operateSource.subList(1, 2)), + "Pages where start (+1) to end (-1) timestamps" + ); + + assertEquals( + newArrayList(bookInfo.getPages(time2.minus(1, NANOS), time3, order)), + optionalReverse(order, operateSource.subList(0, 3)), + "Pages where start (-1) to end (0) timestamps" + ); + assertEquals( + newArrayList(bookInfo.getPages(time2, time3, order)), + optionalReverse(order, operateSource.subList(1, 3)), + "Pages where start (0) to end (0) timestamps" + ); + assertEquals( + newArrayList(bookInfo.getPages(time2.plus(1, NANOS), time3, order)), + optionalReverse(order, operateSource.subList(1, 3)), + "Pages where start (+1) to end (0) timestamps" + ); + + assertEquals( + newArrayList(bookInfo.getPages(time2.minus(1, NANOS), time3.plus(1, NANOS), order)), + optionalReverse(order, operateSource.subList(0, 3)), + "Pages where start (-1) to end (+1) timestamps" + ); + assertEquals( + newArrayList(bookInfo.getPages(time2, time3.plus(1, NANOS), order)), + optionalReverse(order, operateSource.subList(1, 3)), + "Pages where start (0) to end (-1) timestamps" + ); + assertEquals( + newArrayList(bookInfo.getPages(time2.plus(1, NANOS), time3.plus(1, NANOS), order)), + optionalReverse(order, operateSource.subList(1, 3)), + "Pages where start (+1) to end (+1) timestamps" + ); + } + + @Test(dataProvider = "orderToPages") + public void getPagesWhenBookHasGapTest(Order order, List pageTimestamps) { + Instant time1 = pageTimestamps.get(0); + Instant time2 = pageTimestamps.get(1); + Instant time3 = pageTimestamps.get(2); + Instant time4 = pageTimestamps.get(3); + List operateSource = List.of( + createPageInfo(time1, time2), + createPageInfo(time3, time4) + ); + BookInfo bookInfo = createBookInfo(operateSource, 1); + + assertEquals( + newArrayList(bookInfo.getPages(time2.minus(1, NANOS), time3.minus(1, NANOS), order)), + optionalReverse(order, operateSource.subList(0, 1)), + "Pages where start (-1) to end (-1) timestamps" + ); + assertEquals( + newArrayList(bookInfo.getPages(time2, time3.minus(1, NANOS), order)), + emptyList(), + "Pages where start (0) to end (-1) timestamps" + ); + assertEquals( + newArrayList(bookInfo.getPages(time2.plus(1, NANOS), time3.minus(1, NANOS), order)), + emptyList(), + "Pages where start (+1) to end (-1) timestamps" + ); + + assertEquals( + newArrayList(bookInfo.getPages(time2.minus(1, NANOS), time3, order)), + optionalReverse(order, operateSource.subList(0, 2)), + "Pages where start (-1) to end (0) timestamps" + ); + assertEquals( + newArrayList(bookInfo.getPages(time2, time3, order)), + optionalReverse(order, operateSource.subList(1, 2)), + "Pages where start (0) to end (0) timestamps" + ); + assertEquals( + newArrayList(bookInfo.getPages(time2.plus(1, NANOS), time3, order)), + optionalReverse(order, operateSource.subList(1, 2)), + "Pages where start (+1) to end (0) timestamps" + ); + + assertEquals( + newArrayList(bookInfo.getPages(time2.minus(1, NANOS), time3.plus(1, NANOS), order)), + optionalReverse(order, operateSource.subList(0, 2)), + "Pages where start (-1) to end (+1) timestamps" + ); + assertEquals( + newArrayList(bookInfo.getPages(time2, time3.plus(1, NANOS), order)), + optionalReverse(order, operateSource.subList(1, 2)), + "Pages where start (0) to end (-1) timestamps" + ); + assertEquals( + newArrayList(bookInfo.getPages(time2.plus(1, NANOS), time3.plus(1, NANOS), order)), + optionalReverse(order, operateSource.subList(1, 2)), + "Pages where start (+1) to end (+1) timestamps" + ); + } + + @Test(dataProvider = "orders") + public void getPagesWithEmptyResult(Order order) { + PageInfo pageInfo = PAGES.get(0); + List operateSource = List.of(pageInfo); + BookInfo bookInfo = createBookInfo(operateSource, 1); + + assertEquals( + newArrayList(bookInfo.getPages(null, pageInfo.getStarted().minus(1, NANOS), order)), + emptyList(), + "End timestamp before first page start" + ); + assertEquals( + newArrayList(bookInfo.getPages(pageInfo.getEnded().plus(1, NANOS), null, order)), + emptyList(), + "Start timestamp after last page end" + ); + assertEquals( + newArrayList(bookInfo.getPages(pageInfo.getEnded(), pageInfo.getStarted(), order)), + emptyList(), + "Start > end timestamp after last page end" + ); + } + + @Test(dataProvider = "orders") + // ... ps | | ps ps ... + public void getPagesByPointTest(Order order) { + Instant time1 = Instant.parse("2024-02-13T12:00:00Z"); + Instant time2 = Instant.parse("2024-02-15T12:00:00Z"); + Instant time3 = Instant.parse("2024-02-15T18:00:00Z"); + List operateSource = List.of( + createPageInfo(time1, time2), + createPageInfo(time2, time3), + createPageInfo(time3, null) + ); + BookInfo bookInfo = createBookInfo(operateSource, 1); + + assertEquals( + newArrayList(bookInfo.getPages(Instant.MIN, Instant.MIN, order)), + emptyList(), + "Point with min timestamp" + ); + assertEquals( + newArrayList(bookInfo.getPages(time1.minus(1, NANOS), time1.minus(1, NANOS), order)), + emptyList(), + "Point before first page start" + ); + assertEquals( + newArrayList(bookInfo.getPages(time1, time1, order)), + operateSource.subList(0, 1), + "Point equals first page start" + ); + assertEquals( + newArrayList(bookInfo.getPages(time1.plus(1, NANOS), time1.plus(1, NANOS), order)), + operateSource.subList(0, 1), + "Point after first page start" + ); + assertEquals( + newArrayList(bookInfo.getPages(time1.plus(1, DAYS), time1.plus(1, DAYS), order)), + operateSource.subList(0, 1), + "Point in the middle of first page" + ); + assertEquals( + newArrayList(bookInfo.getPages(time2.minus(1, NANOS), time2.minus(1, NANOS), order)), + operateSource.subList(0, 1), + "Point before first page end" + ); + assertEquals( + newArrayList(bookInfo.getPages(time2, time2, order)), + operateSource.subList(1, 2), + "Point equals second page start" + ); + assertEquals( + newArrayList(bookInfo.getPages(time3, time3, order)), + operateSource.subList(2, 3), + "Point equals third page start" + ); + assertEquals( + newArrayList(bookInfo.getPages(Instant.MAX, Instant.MAX, order)), + operateSource.subList(2, 3), + "Point with max timestamp" + ); + } + + @Test + public void findPageTest() { + Instant time1 = Instant.parse("2024-02-13T12:00:00Z"); + Instant time2 = Instant.parse("2024-02-15T12:00:00Z"); + Instant time3 = Instant.parse("2024-02-15T18:00:00Z"); + List operateSource = List.of( + createPageInfo(time1, time2), + createPageInfo(time2, time3), + createPageInfo(time3, null) + ); + BookInfo bookInfo = createBookInfo(operateSource, 1); + + assertNull( + bookInfo.findPage(Instant.MIN), + "Min timestamp" + ); + assertNull( + bookInfo.findPage(time1.minus(1, NANOS)), + "Timestamp before first page start" + ); + assertEquals( + bookInfo.findPage(time1), + operateSource.get(0), + "Timestamp equals first page start" + ); + assertEquals( + bookInfo.findPage(time1.plus(1, NANOS)), + operateSource.get(0), + "Timestamp after first page start" + ); + assertEquals( + bookInfo.findPage(time1.plus(1, DAYS)), + operateSource.get(0), + "Timestamp in the middle of first page" + ); + assertEquals( + bookInfo.findPage(time2.minus(1, NANOS)), + operateSource.get(0), + "Timestamp before first page end" + ); + assertEquals( + bookInfo.findPage(time2), + operateSource.get(1), + "Timestamp equals second page start" + ); + assertEquals( + bookInfo.findPage(time3), + operateSource.get(2), + "Timestamp equals third page start" + ); + assertEquals( + bookInfo.findPage(Instant.MAX), + operateSource.get(2), + "Max timestamp" + ); + } + + @Test(description = "Missed page is between to pages where - sP1 ... | ... eP1 ... gap ... sP2 ... | ") + public void findPageWhenBookHasGapTest1() { + Instant time1 = Instant.parse("2024-02-14T12:00:00Z"); + Instant time2 = Instant.parse("2024-02-15T12:00:00Z"); + // Gap [2024-02-15T12:00:00Z - 2024-02-15T18:00:00Z) + Instant time3 = Instant.parse("2024-02-15T18:00:00Z"); + List operateSource = List.of( + createPageInfo(time1, time2), + createPageInfo(time3, null) + ); + BookInfo bookInfo = createBookInfo(operateSource, 1); + + assertEquals( + bookInfo.findPage(time2.minus(1, NANOS)), + operateSource.get(0), + "Timestamp before first page end" + ); + assertNull( + bookInfo.findPage(time2), + "Timestamp equals missed page start" + ); + assertNull( + bookInfo.findPage(time2.plus(1, HOURS)), + "Timestamp equals missed page start" + ); + assertEquals( + bookInfo.findPage(time3), + operateSource.get(1), + "Timestamp equals second page start" + ); + } + + @Test(description = "Missed page is between to pages where - | ... sP1 ... eP1 ... gap ... sP2 ... | ") + public void findPageWhenBookHasGapTest2() { + Instant time1 = Instant.parse("2024-02-15T11:00:00Z"); + Instant time2 = Instant.parse("2024-02-15T12:00:00Z"); + // Gap [2024-02-15T12:00:00Z - 2024-02-15T18:00:00Z) + Instant time3 = Instant.parse("2024-02-15T18:00:00Z"); + List operateSource = List.of( + createPageInfo(time1, time2), + createPageInfo(time3, null) + ); + BookInfo bookInfo = createBookInfo(operateSource, 1); + + assertEquals( + bookInfo.findPage(time2.minus(1, NANOS)), + operateSource.get(0), + "Timestamp before first page end" + ); + assertNull( + bookInfo.findPage(time2), + "Timestamp equals missed page start" + ); + assertNull( + bookInfo.findPage(time2.plus(1, HOURS)), + "Timestamp equals missed page start" + ); + assertEquals( + bookInfo.findPage(time3), + operateSource.get(1), + "Timestamp equals second page start" + ); + } + + @Test(description = "Missed page is between to pages where - sP1 ... eP1 ... | ... gap ... sP2 ... | ") + public void findPageWhenBookHasGapTest3() { + Instant time1 = Instant.parse("2024-02-13T11:00:00Z"); + Instant time2 = Instant.parse("2024-02-13T12:00:00Z"); + // Gap [2024-02-13T12:00:00Z - 2024-02-15T18:00:00Z) + Instant time3 = Instant.parse("2024-02-15T18:00:00Z"); + List operateSource = List.of( + createPageInfo(time1, time2), + createPageInfo(time3, null) + ); + BookInfo bookInfo = createBookInfo(operateSource, 1); + + assertEquals( + bookInfo.findPage(time2.minus(1, NANOS)), + operateSource.get(0), + "Timestamp before first page end" + ); + assertNull( + bookInfo.findPage(time2), + "Timestamp equals missed page start" + ); + assertNull( + bookInfo.findPage(time2.plus(1, DAYS)), + "Timestamp equals missed page start" + ); + assertEquals( + bookInfo.findPage(time3), + operateSource.get(1), + "Timestamp equals second page start" + ); + } + + @Test + public void removePageTest() { + List operateSource = new ArrayList<>(PAGES); + BookInfo bookInfo = createBookInfo(operateSource, 1); + + assertEquals(bookInfo.getPages(), operateSource); + int iteration = 0; + while (!operateSource.isEmpty()) { + iteration++; + int index = RANDOM.nextInt(operateSource.size()); + PageInfo pageForRemove = operateSource.get(index); + + assertSame(bookInfo.getPage(pageForRemove.getId()), pageForRemove, "iteration - " + iteration); + assertSame(bookInfo.findPage(pageForRemove.getId().getStart()), pageForRemove, "iteration - " + iteration); + assertEquals(bookInfo.getPages(), operateSource, "iteration - " + iteration); + + operateSource.remove(index); + bookInfo.invalidate(pageForRemove.getStarted()); + + assertNull(bookInfo.getPage(pageForRemove.getId()), "iteration - " + iteration); + + for (PageInfo page : operateSource) { + assertSame(bookInfo.getPage(page.getId()), page, "iteration - " + iteration); + assertSame(bookInfo.findPage(page.getId().getStart()), page, "iteration - " + iteration); + } + } + } + + @Test + public void addPageTest() { + ArrayList pages = new ArrayList<>(); + BookInfo bookInfo = createBookInfo(pages, 1); + + assertNull(bookInfo.getFirstPage()); + assertNull(bookInfo.getLastPage()); + + for (int i = 0; i < PAGES.size(); i++) { + PageInfo page = PAGES.get(i); + pages.add(page); + bookInfo.invalidate(page.getStarted()); + + assertSame(bookInfo.getFirstPage(), pages.get(0), "iteration - " + i); + assertSame(bookInfo.getLastPage(), pages.get(pages.size() - 1), "iteration - " + i); + assertSame(bookInfo.getPage(page.getId()), page, "iteration - " + i); + assertSame(bookInfo.findPage(page.getId().getStart()), page, "iteration - " + i); + } + } + + private static List optionalReverse(Order order, List origin) { + if (order == DIRECT) { + return origin; + } + return Lists.reverse(origin); + } + + private static BookInfo createBookInfo(List pages, int cacheSize) { + return new BookInfo( + BOOK_ID, + "test-full-name", + "test-description", + Instant.EPOCH, + cacheSize, + Long.MAX_VALUE, + new TestPagesLoader(pages), + new TestPageLoader(pages, true), + new TestPageLoader(pages, false)); + } + + private static PageInfo createPageInfo(Instant start, @Nullable Instant end) { + return new PageInfo(new PageId(BOOK_ID, start, start.toString()), end, "test-comment"); + } + + @DataProvider(name = "orders") + public Order[] orders() { + return Order.values(); + } + + @DataProvider(name = "orderToPages") + public Object[][] orderToPages() { + // ... sP1 ... eP1 [sP2 ... eP2) sP3 ... eP3 ... + List case1 = List.of( + Instant.parse("2024-02-13T01:01:00Z"), + Instant.parse("2024-02-13T12:00:00Z"), + Instant.parse("2024-02-13T13:00:00Z"), + Instant.parse("2024-02-13T23:00:00Z") + ); + + // ... sP1 ... | ... eP1 [sP2 ... eP2) sP3 ... eP3 ... + List case2 = List.of( + Instant.parse("2024-02-12T01:02:00Z"), + Instant.parse("2024-02-13T12:00:00Z"), + Instant.parse("2024-02-15T13:00:00Z"), + Instant.parse("2024-02-15T23:00:00Z") + ); + // ... sP1 ... eP1 [sP2 ... eP2) sP3 ... | ... eP3 ... + List case3 = List.of( + Instant.parse("2024-02-13T01:03:00Z"), + Instant.parse("2024-02-13T12:00:00Z"), + Instant.parse("2024-02-13T13:00:00Z"), + Instant.parse("2024-02-14T23:00:00Z") + ); + + // ... sP1 ... | ... | ... eP1 [sP2 ... eP2) sP3 ... eP3 ... + List case4 = List.of( + Instant.parse("2024-02-11T01:04:00Z"), + Instant.parse("2024-02-13T12:00:00Z"), + Instant.parse("2024-02-13T13:00:00Z"), + Instant.parse("2024-02-13T23:00:00Z") + ); + // ... sP1 ... eP1 [sP2 ... eP2) sP3 ... | ... | ... eP3 ... + List case5 = List.of( + Instant.parse("2024-02-13T01:05:00Z"), + Instant.parse("2024-02-13T12:00:00Z"), + Instant.parse("2024-02-13T13:00:00Z"), + Instant.parse("2024-02-15T23:00:00Z") + ); + + // ... sP1 ... eP1 [sP2 ... | ... eP2) sP3 ... eP3 ... + List case6 = List.of( + Instant.parse("2024-02-13T01:06:00Z"), + Instant.parse("2024-02-13T12:00:00Z"), + Instant.parse("2024-02-14T13:00:00Z"), + Instant.parse("2024-02-14T23:00:00Z") + ); + + return new Object[][]{ + {DIRECT, case1}, + {DIRECT, case2}, + {DIRECT, case3}, + {DIRECT, case4}, + {DIRECT, case5}, + {DIRECT, case6}, + {REVERSE, case1}, + {REVERSE, case2}, + {REVERSE, case3}, + {REVERSE, case4}, + {REVERSE, case5}, + {REVERSE, case6}, + }; + } + + @DataProvider(name = "cacheSize") + public static Integer[] cacheSize() { + return new Integer[]{1, 5, 10}; + } +} diff --git a/cradle-core/src/test/java/com/exactpro/cradle/CradleStorageBookPageNamingTest.java b/cradle-core/src/test/java/com/exactpro/cradle/CradleStorageBookPageNamingTest.java index e151963a7..dc6e1f4a9 100644 --- a/cradle-core/src/test/java/com/exactpro/cradle/CradleStorageBookPageNamingTest.java +++ b/cradle-core/src/test/java/com/exactpro/cradle/CradleStorageBookPageNamingTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2022-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2022-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ import java.io.IOException; import java.time.Instant; +import java.time.temporal.ChronoUnit; public class CradleStorageBookPageNamingTest { @@ -33,7 +34,7 @@ public class CradleStorageBookPageNamingTest { @BeforeMethod public void prepare() throws CradleStorageException, IOException { - storage = new DummyCradleStorage(); + storage = new InMemoryCradleStorage(); storage.init(false); storage.addBook(new BookToAdd(BOOK_ID.getName(), START_TIMESTAMP)); } @@ -102,10 +103,11 @@ public void invalidPageName5() throws IOException, CradleStorageException @Test public void validPageName() throws IOException, CradleStorageException { - storage.addPage(BOOK_ID, "pag-~e", Instant.now(), "comment"); - storage.addPage(BOOK_ID, "pa`ge 2_", Instant.now(), "comment"); - storage.addPage(BOOK_ID, "'page=_3", Instant.now(), "comment"); - storage.addPage(BOOK_ID, "4\"pa++ge", Instant.now(), "comment"); - storage.addPage(BOOK_ID, "pag%%e1", Instant.now(), "comment"); + Instant now = Instant.now(); + storage.addPage(BOOK_ID, "pag-~e", now, "comment"); + storage.addPage(BOOK_ID, "pa`ge 2_", now.plus(1, ChronoUnit.HOURS), "comment"); + storage.addPage(BOOK_ID, "'page=_3", now.plus(2, ChronoUnit.HOURS), "comment"); + storage.addPage(BOOK_ID, "4\"pa++ge", now.plus(3, ChronoUnit.HOURS), "comment"); + storage.addPage(BOOK_ID, "pag%%e1", now.plus(4, ChronoUnit.HOURS), "comment"); } } diff --git a/cradle-core/src/test/java/com/exactpro/cradle/CradleStorageTest.java b/cradle-core/src/test/java/com/exactpro/cradle/CradleStorageTest.java index 86215b70c..7c4146c95 100644 --- a/cradle-core/src/test/java/com/exactpro/cradle/CradleStorageTest.java +++ b/cradle-core/src/test/java/com/exactpro/cradle/CradleStorageTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,6 +33,7 @@ import java.io.IOException; import java.time.Instant; +import java.util.List; import static org.testng.Assert.assertEquals; @@ -55,25 +56,21 @@ public class CradleStorageTest { @BeforeMethod public void prepare() throws CradleStorageException, IOException { - storage = new DummyCradleStorage(); + storage = new InMemoryCradleStorage(); storage.init(false); storage.addBook(new BookToAdd(BOOK, BOOK_START)); BookId bookId = new BookId(BOOK); - BookInfo bookInfo = storage.getBookCache().getBook(bookId); - bookInfo.addPage(createPage(bookId, "page1", PAGE1_START, PAGE2_START)); - bookInfo.addPage(createPage(bookId, "page2", PAGE2_START, PAGE3_START)); - bookInfo.addPage(createPage(bookId, "page3", PAGE3_START, null)); + storage.addPages(bookId, List.of( + new PageToAdd("page1", PAGE1_START, null), + new PageToAdd("page2", PAGE2_START, null), + new PageToAdd("page3", PAGE3_START, null) + )); builder = new MessageToStoreBuilder(); } - private PageInfo createPage(BookId bookId, String name, Instant start, Instant end) { - PageId pageId = new PageId(bookId, name); - return new PageInfo(pageId, start, end, null, null, null); - } - private MessageToStore createMessage(BookId book, String sessionAlias, Direction direction, int sequence, Instant timestamp) throws CradleStorageException { return builder.bookId(book) .sessionAlias(sessionAlias) diff --git a/cradle-core/src/test/java/com/exactpro/cradle/InMemoryCradleStorage.java b/cradle-core/src/test/java/com/exactpro/cradle/InMemoryCradleStorage.java new file mode 100644 index 000000000..b4dd2fa21 --- /dev/null +++ b/cradle-core/src/test/java/com/exactpro/cradle/InMemoryCradleStorage.java @@ -0,0 +1,431 @@ +/* + * Copyright 2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.cradle; + +import com.exactpro.cradle.counters.Counter; +import com.exactpro.cradle.counters.CounterSample; +import com.exactpro.cradle.counters.Interval; +import com.exactpro.cradle.intervals.IntervalsWorker; +import com.exactpro.cradle.messages.GroupedMessageBatchToStore; +import com.exactpro.cradle.messages.GroupedMessageFilter; +import com.exactpro.cradle.messages.MessageBatchToStore; +import com.exactpro.cradle.messages.MessageFilter; +import com.exactpro.cradle.messages.StoredGroupedMessageBatch; +import com.exactpro.cradle.messages.StoredMessage; +import com.exactpro.cradle.messages.StoredMessageBatch; +import com.exactpro.cradle.messages.StoredMessageId; +import com.exactpro.cradle.resultset.CradleResultSet; +import com.exactpro.cradle.testevents.StoredTestEvent; +import com.exactpro.cradle.testevents.StoredTestEventId; +import com.exactpro.cradle.testevents.TestEventFilter; +import com.exactpro.cradle.testevents.TestEventToStore; +import com.exactpro.cradle.utils.CradleStorageException; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * In memory implementation of CradleStorage that holds book and pages in memory + */ +public class InMemoryCradleStorage extends CradleStorage { + + static class InMemoryBookCache implements BookCache { + + private final Map inMemoryStorage; + private final Map cache; + + InMemoryBookCache(Map inMemoryStorage) { + this.inMemoryStorage = inMemoryStorage; + this.cache = new ConcurrentHashMap<>(); + } + + @Override + public BookInfo getBook(BookId bookId) throws CradleStorageException { + BookInfo bookInfo = cache.computeIfAbsent(bookId, (key) -> + { + InMemoryBook inMemoryBook = inMemoryStorage.get(bookId); + if (inMemoryBook == null) { + return null; + } + + return inMemoryBook.bookInfo; + }); + if (bookInfo == null) { + throw new CradleStorageException(String.format("Book %s is unknown", bookId.getName())); + } + return bookInfo; + } + + @Override + public boolean checkBook(BookId bookId) { + return cache.containsKey(bookId); + } + + @Override + public Collection loadPageInfo(BookId bookId, boolean loadRemoved) { + InMemoryBook inMemoryBook = inMemoryStorage.get(bookId); + if (inMemoryBook == null) { + return Collections.emptyList(); + } + + return inMemoryBook.pages.stream() + .filter(pageInfo -> loadRemoved || pageInfo.getRemoved() == null) + .collect(Collectors.toList()); + } + + @Override + public Collection getCachedBooks() { + return Collections.unmodifiableCollection(cache.values()); + } + } + + static class InMemoryBook { + private final List pages = new ArrayList<>(); + private final BookInfo bookInfo; + + private InMemoryBook(BookToAdd bookToAdd) { + this.bookInfo = new BookInfo(new BookId(bookToAdd.getName()), + bookToAdd.getFullName(), + bookToAdd.getDesc(), + bookToAdd.getCreated(), + 1, + Long.MAX_VALUE, + new TestPagesLoader(pages), + new TestPageLoader(pages, true), + new TestPageLoader(pages, false)); + } + } + + private final Map inMemoryStorage = new HashMap<>(); + + private final InMemoryBookCache inMemoryBookCache; + + @Override + protected BookCache getBookCache() { + return inMemoryBookCache; + } + + public InMemoryCradleStorage() throws CradleStorageException { + super(); + inMemoryBookCache = new InMemoryBookCache(inMemoryStorage); + } + + + @Override + protected void doInit(boolean prepareStorage) { + } + + @Override + protected void doDispose() { + } + + @Override + protected Collection doGetAllPages(BookId bookId) { + return null; + } + + @Override + protected Collection doListBooks() { + return null; + } + + @Override + protected void doAddBook(BookToAdd newBook, BookId newBookId) { + inMemoryStorage.compute(newBookId, ((bookId, previous) -> { + if (previous != null) { + throw new IllegalStateException("Book '" + bookId + "' is already exist"); + } + return new InMemoryBook(newBook); + })); + } + + @Override + protected void doAddPages(BookId bookId, List pages, PageInfo lastPage) { + try { + List inMemoryPages = inMemoryStorage.get(bookId).pages; + if (lastPage != null) { + inMemoryPages.set(inMemoryPages.size() - 1, lastPage); + } + inMemoryPages.addAll(pages); + inMemoryBookCache.getBook(bookId).refresh(); + } catch (CradleStorageException e) { + throw new RuntimeException(e); + } + } + + @Override + protected Collection doLoadPages(BookId bookId) { + return null; + } + + @Override + protected void doRemovePage(PageInfo page) { + } + + @Override + protected void doStoreMessageBatch(MessageBatchToStore batch, PageInfo page) { + } + + @Override + protected void doStoreGroupedMessageBatch(GroupedMessageBatchToStore batch, PageInfo page) { + + } + + @Override + protected CompletableFuture doStoreMessageBatchAsync(MessageBatchToStore batch, + PageInfo page) { + return CompletableFuture.completedFuture(null); + } + + @Override + protected CompletableFuture doStoreGroupedMessageBatchAsync(GroupedMessageBatchToStore batch, PageInfo page) { + return null; + } + + @Override + protected void doStoreTestEvent(TestEventToStore event, PageInfo page) { + } + + @Override + protected CompletableFuture doStoreTestEventAsync(TestEventToStore event, PageInfo page) { + return CompletableFuture.completedFuture(null); + } + + @Override + protected void doUpdateParentTestEvents(TestEventToStore event) { + } + + @Override + protected CompletableFuture doUpdateParentTestEventsAsync(TestEventToStore event) { + return CompletableFuture.completedFuture(null); + } + + @Override + protected void doUpdateEventStatus(StoredTestEvent event, boolean success) { + } + + @Override + protected CompletableFuture doUpdateEventStatusAsync(StoredTestEvent event, boolean success) { + return CompletableFuture.completedFuture(null); + } + + + @Override + protected StoredMessage doGetMessage(StoredMessageId id, PageId pageId) { + return null; + } + + @Override + protected CompletableFuture doGetMessageAsync(StoredMessageId id, PageId pageId) { + return CompletableFuture.completedFuture(null); + } + + @Override + protected StoredMessageBatch doGetMessageBatch(StoredMessageId id, PageId pageId) { + return null; + } + + @Override + protected CompletableFuture doGetMessageBatchAsync(StoredMessageId id, PageId pageId) { + return CompletableFuture.completedFuture(null); + } + + @Override + protected CradleResultSet doGetMessages(MessageFilter filter, BookInfo book) { + return null; + } + + @Override + protected CompletableFuture> doGetMessagesAsync(MessageFilter filter, + BookInfo book) { + return CompletableFuture.completedFuture(null); + } + + @Override + protected CradleResultSet doGetMessageBatches(MessageFilter filter, BookInfo book) { + return null; + } + + @Override + protected CradleResultSet doGetGroupedMessageBatches(GroupedMessageFilter filter, + BookInfo book) { + return null; + } + + @Override + protected CompletableFuture> doGetMessageBatchesAsync(MessageFilter filter, + BookInfo book) { + return CompletableFuture.completedFuture(null); + } + + @Override + protected CompletableFuture> doGetGroupedMessageBatchesAsync( + GroupedMessageFilter filter, BookInfo book) { + return null; + } + + + @Override + protected long doGetLastSequence(String sessionAlias, Direction direction, BookId bookId) { + return 0; + } + + @Override + protected long doGetFirstSequence(String sessionAlias, Direction direction, BookId bookId) { + return 0; + } + + @Override + protected Collection doGetSessionAliases(BookId bookId) { + return null; + } + + @Override + protected Collection doGetGroups(BookId bookId) { + return null; + } + + + @Override + protected StoredTestEvent doGetTestEvent(StoredTestEventId id, PageId pageId) { + return null; + } + + @Override + protected CompletableFuture doGetTestEventAsync(StoredTestEventId ids, PageId pageId) { + return CompletableFuture.completedFuture(null); + } + + @Override + protected CradleResultSet doGetTestEvents(TestEventFilter filter, BookInfo book) { + return null; + } + + @Override + protected CompletableFuture> doGetTestEventsAsync(TestEventFilter filter, BookInfo book) { + return CompletableFuture.completedFuture(null); + } + + + @Override + protected Collection doGetScopes(BookId bookId) { + return null; + } + + @Override + protected CradleResultSet doGetScopes(BookId bookId, Interval interval) { + return null; + } + + @Override + protected CompletableFuture> doGetScopesAsync(BookId bookId, Interval interval) { + return null; + } + + @Override + protected CompletableFuture> doGetMessageCountersAsync(BookId bookId, String sessionAlias, Direction direction, FrameType frameType, Interval interval) { + return null; + } + + @Override + protected CradleResultSet doGetMessageCounters(BookId bookId, String sessionAlias, Direction direction, FrameType frameType, Interval interval) { + return null; + } + + @Override + protected CompletableFuture> doGetCountersAsync(BookId bookId, EntityType entityType, FrameType frameType, Interval interval) { + return null; + } + + @Override + protected CradleResultSet doGetCounters(BookId bookId, EntityType entityType, FrameType frameType, Interval interval) { + return null; + } + + @Override + protected CompletableFuture doGetMessageCountAsync(BookId bookId, String sessionAlias, Direction direction, Interval interval) { + return null; + } + + @Override + protected Counter doGetMessageCount(BookId bookId, String sessionAlias, Direction direction, Interval interval) { + return null; + } + + @Override + protected CompletableFuture doGetCountAsync(BookId bookId, EntityType entityType, Interval interval) { + return null; + } + + @Override + protected Counter doGetCount(BookId bookId, EntityType entityType, Interval interval) { + return null; + } + + @Override + protected CompletableFuture> doGetSessionAliasesAsync(BookId bookId, Interval interval) { + return null; + } + + @Override + protected CradleResultSet doGetSessionAliases(BookId bookId, Interval interval) { + return null; + } + + @Override + protected CompletableFuture> doGetSessionGroupsAsync(BookId bookId, Interval interval) { + return null; + } + + @Override + protected CradleResultSet doGetSessionGroups(BookId bookId, Interval interval) { + return null; + } + + @Override + protected PageInfo doUpdatePageComment(BookId bookId, String pageName, String comment) { + return null; + } + + @Override + protected PageInfo doUpdatePageName(BookId bookId, String pageName, String newPageName) { + return null; + } + + @Override + protected Iterator doGetPages(BookId bookId, Interval interval) { + return null; + } + + @Override + protected CompletableFuture> doGetPagesAsync(BookId bookId, Interval interval) { + return null; + } + + @Override + public IntervalsWorker getIntervalsWorker() { + return null; + } +} \ No newline at end of file diff --git a/cradle-core/src/test/java/com/exactpro/cradle/TestPageLoader.java b/cradle-core/src/test/java/com/exactpro/cradle/TestPageLoader.java new file mode 100644 index 000000000..5e197c982 --- /dev/null +++ b/cradle-core/src/test/java/com/exactpro/cradle/TestPageLoader.java @@ -0,0 +1,37 @@ +/* + * Copyright 2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.cradle; + +import java.util.List; +import java.util.function.Function; + +public class TestPageLoader implements Function { + + private final List pages; + private final boolean first; + + public TestPageLoader(List pages, boolean first) { + this.pages = pages; + this.first = first; + } + + @Override + public PageInfo apply(BookId bookId) { + return first + ? pages.isEmpty() ? null : pages.get(0) + : pages.isEmpty() ? null : pages.get(pages.size() - 1); + } +} diff --git a/cradle-core/src/test/java/com/exactpro/cradle/TestPagesLoader.java b/cradle-core/src/test/java/com/exactpro/cradle/TestPagesLoader.java new file mode 100644 index 000000000..35d1d9f9c --- /dev/null +++ b/cradle-core/src/test/java/com/exactpro/cradle/TestPagesLoader.java @@ -0,0 +1,42 @@ +/* + * Copyright 2024 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.cradle; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.time.Instant; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +public class TestPagesLoader implements PagesLoader { + + private final List pages; + + public TestPagesLoader(List pages) { + this.pages = pages; + } + + @Nonnull + @Override + public Collection load(@Nonnull BookId bookId, @Nullable Instant start, @Nullable Instant end) { + return pages.stream() + .filter(page -> (start == null || !start.isAfter(page.getId().getStart())) + && (end == null || !end.isBefore(page.getStarted())) + ) + .collect(Collectors.toList()); + } +} diff --git a/cradle-core/src/test/java/com/exactpro/cradle/TestUtils.java b/cradle-core/src/test/java/com/exactpro/cradle/TestUtils.java index 6deae526f..818e713d8 100644 --- a/cradle-core/src/test/java/com/exactpro/cradle/TestUtils.java +++ b/cradle-core/src/test/java/com/exactpro/cradle/TestUtils.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,16 +16,20 @@ package com.exactpro.cradle; -import org.testng.Assert; - import com.exactpro.cradle.utils.CradleStorageException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.testng.Assert.assertTrue; public class TestUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(TestUtils.class); public static void handleException(CradleStorageException e, String errorMessage) throws CradleStorageException { + LOGGER.error(e.getMessage(), e); String msg = e.getMessage(); - Assert.assertTrue(msg.contains(errorMessage), "error '"+msg+"' contains '"+errorMessage+"'"); + assertTrue(msg.contains(errorMessage), "error '" + msg + "' contains '" + errorMessage + "'"); throw e; } diff --git a/cradle-core/src/test/java/com/exactpro/cradle/testevents/EventBatchTest.java b/cradle-core/src/test/java/com/exactpro/cradle/testevents/EventBatchTest.java index 2b654e3d6..f573b83e1 100644 --- a/cradle-core/src/test/java/com/exactpro/cradle/testevents/EventBatchTest.java +++ b/cradle-core/src/test/java/com/exactpro/cradle/testevents/EventBatchTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2021-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2021-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,8 @@ import com.exactpro.cradle.Direction; import com.exactpro.cradle.PageId; import com.exactpro.cradle.PageInfo; +import com.exactpro.cradle.TestPageLoader; +import com.exactpro.cradle.TestPagesLoader; import com.exactpro.cradle.TestUtils; import com.exactpro.cradle.messages.StoredMessageId; import com.exactpro.cradle.serialization.EventsSizeCalculator; @@ -37,6 +39,7 @@ import java.time.Instant; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Set; import java.util.UUID; import java.util.stream.Stream; @@ -49,6 +52,7 @@ import static com.exactpro.cradle.testevents.EventSingleTest.START_TIMESTAMP; import static com.exactpro.cradle.testevents.EventSingleTest.batchParentId; import static com.exactpro.cradle.testevents.EventSingleTest.validEvent; +import static java.time.temporal.ChronoUnit.NANOS; public class EventBatchTest { private final int MAX_SIZE = 1024; @@ -69,7 +73,7 @@ public void prepareBatch() throws CradleStorageException { } @DataProvider(name = "batch invalid events") - public Object[][] batchInvalidEvents() throws CradleStorageException { + public Object[][] batchInvalidEvents() { Object[][] batchEvents = new Object[][] { {validEvent().parentId(null), //No parent ID @@ -202,7 +206,7 @@ public void duplicateIds() throws CradleStorageException { batch.addTestEvent(eventBuilder.id(eventId).name(DUMMY_NAME).parentId(batch.getParentId()).build()); } - @Test(expectedExceptions = {CradleStorageException.class}, expectedExceptionsMessageRegExp = ".* '.*\\:XXX' .* stored in this batch .*") + @Test(expectedExceptions = {CradleStorageException.class}, expectedExceptionsMessageRegExp = ".* '.*:XXX' .* stored in this batch .*") public void externalReferences() throws CradleStorageException { StoredTestEventId parentId = new StoredTestEventId(BOOK, SCOPE, START_TIMESTAMP, "1"); batch.addTestEvent(eventBuilder.id(parentId) @@ -283,16 +287,7 @@ public void childIsNotRoot() throws CradleStorageException { public void batchEventValidation(TestEventSingleToStoreBuilder builder, String errorMessage) throws CradleStorageException { try { var singleEvent = builder.build(); - BookInfo bookInfo = new BookInfo( - BOOK, - null, - null, - START_TIMESTAMP, - Collections.singleton(new PageInfo( - new PageId(null, null), - START_TIMESTAMP, - START_TIMESTAMP, - null))); + BookInfo bookInfo = createBookInfo(); TestEventUtils.validateTestEvent(singleEvent, bookInfo, storeActionRejectionThreshold); batch.addTestEvent(singleEvent); Assertions.fail("Invalid message passed validation"); @@ -301,6 +296,23 @@ public void batchEventValidation(TestEventSingleToStoreBuilder builder, String e } } + private static BookInfo createBookInfo() { + List pages = List.of(new PageInfo( + new PageId(BOOK, START_TIMESTAMP, "test-page"), + START_TIMESTAMP.plus(1, NANOS), + null) + ); + return new BookInfo( + BOOK, + null, + null, + START_TIMESTAMP, + 1, + Long.MAX_VALUE, + new TestPagesLoader(pages), + new TestPageLoader(pages, true), new TestPageLoader(pages, false)); + } + @Test public void batchEventMessagesAreIndependent() throws CradleStorageException { TestEventSingleToStore event = validEvent().success(true).message(new StoredMessageId(BOOK, "Session1", Direction.FIRST, START_TIMESTAMP, 1)).build(); diff --git a/cradle-core/src/test/java/com/exactpro/cradle/testevents/EventSingleTest.java b/cradle-core/src/test/java/com/exactpro/cradle/testevents/EventSingleTest.java index 7b7558a4b..7da71fab5 100644 --- a/cradle-core/src/test/java/com/exactpro/cradle/testevents/EventSingleTest.java +++ b/cradle-core/src/test/java/com/exactpro/cradle/testevents/EventSingleTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2024 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,8 @@ import com.exactpro.cradle.Direction; import com.exactpro.cradle.PageId; import com.exactpro.cradle.PageInfo; +import com.exactpro.cradle.TestPageLoader; +import com.exactpro.cradle.TestPagesLoader; import com.exactpro.cradle.TestUtils; import com.exactpro.cradle.messages.StoredMessageId; import com.exactpro.cradle.utils.CradleStorageException; @@ -35,6 +37,7 @@ import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Set; public class EventSingleTest { @@ -52,7 +55,7 @@ public class EventSingleTest { private final TestEventSingleToStoreBuilder eventBuilder = new TestEventSingleToStoreBuilder(storeActionRejectionThreshold); @DataProvider(name = "invalid events") - public Object[][] invalidEvents() throws CradleStorageException { + public Object[][] invalidEvents() { return new Object[][] { {new TestEventSingleToStoreBuilder(storeActionRejectionThreshold), //Empty event @@ -113,16 +116,7 @@ public void eventFields() throws CradleStorageException { expectedExceptions = {CradleStorageException.class}) public void eventValidation(TestEventSingleToStoreBuilder builder, String errorMessage) throws CradleStorageException { try { - BookInfo bookInfo = new BookInfo( - BOOK, - null, - null, - START_TIMESTAMP, - Collections.singleton(new PageInfo( - new PageId(null, null), - START_TIMESTAMP, - START_TIMESTAMP, - null))); + BookInfo bookInfo = createBookInfo(); TestEventUtils.validateTestEvent(builder.build(), bookInfo, storeActionRejectionThreshold); Assertions.fail("Invalid message passed validation"); } catch (CradleStorageException e) { @@ -130,6 +124,23 @@ public void eventValidation(TestEventSingleToStoreBuilder builder, String errorM } } + private static BookInfo createBookInfo() { + List pages = List.of(new PageInfo( + new PageId(BOOK, START_TIMESTAMP, "test-page"), + START_TIMESTAMP, + null) + ); + return new BookInfo( + BOOK, + null, + null, + START_TIMESTAMP, + 1, + Long.MAX_VALUE, + new TestPagesLoader(pages), + new TestPageLoader(pages, true), new TestPageLoader(pages, false)); + } + @Test public void passedEvent() throws CradleStorageException { TestEventSingleToStore event = eventBuilder diff --git a/gradle.properties b/gradle.properties index 2d0fef2e0..46e157892 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ -release_version=5.1.5 +release_version=5.2.0 description='Cradle API' vcs_url=https://github.com/th2-net/cradleapi diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 1dcfc8730..241db45c8 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ # -# Copyright 2023 Exactpro (Exactpro Systems Limited) +# Copyright 2024 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,4 +19,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip