[TH2-5204] Improve book info cache (#259)

Co-authored-by: Oleg Smelov <[email protected]>
th2-net · Jul 1, 2024 · bc6f219 · bc6f219
1 parent b6c168a
commit bc6f219
Show file tree

Hide file tree

Showing 18 changed files with 711 additions and 189 deletions.
diff --git a/.github/workflows/build-dev-release.yml b/.github/workflows/build-dev-release.yml
@@ -13,4 +13,5 @@ jobs:
       sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }}
       sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}
       sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }}
-      sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
+      sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
+      nvd-api-key: ${{ secrets.NVD_APIKEY }}
diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml
@@ -13,4 +13,5 @@ jobs:
       sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }}
       sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}
       sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }}
-      sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
+      sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
+      nvd-api-key: ${{ secrets.NVD_APIKEY }}
diff --git a/.github/workflows/build-sanpshot.yml b/.github/workflows/build-sanpshot.yml
@@ -18,4 +18,5 @@ jobs:
       sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }}
       sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}
       sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }}
-      sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
+      sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
+      nvd-api-key: ${{ secrets.NVD_APIKEY }}
diff --git a/README.md b/README.md
@@ -209,6 +209,14 @@ Test events have mandatory parameters that are verified when storing an event. T
 
 ## Release notes
 
+### 5.4.1
+* Page interval in page cache hols all pages covered the interval.<br>
+  Fixed the problem - components reload page interval and last page each time when page starts in day before and ends day after requested period.
+* Corrected default settings:
+  * counterPersistenceInterval: `1000` -> `15000` ms
+  * compressionType: `ZLIB` -> `LZ4`
+  * composingServiceThreads: `5` -> `1`
+
 ### 5.4.0
 * Using internal executor instead of ForkJoinPool.commonPool() to process intermediate tasks 
 

diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraCradleStorage.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraCradleStorage.java
@@ -1028,9 +1028,9 @@ protected PageInfo doUpdatePageComment(BookId bookId, String pageName, String co
 		if (pageNameEntity == null)
 			throw new CradleStorageException(String.format("Page \"%s\" not found in book \"%s\"", pageName, bookId.getName()));
 
-		PageEntity pageEntity = pageOperator.get(bookId.getName(),
+		PageEntity pageEntity = pageOperator.getAllAfter(bookId.getName(),
 					pageNameEntity.getStartDate(),
-					pageNameEntity.getStartTime().minusNanos(1),
+					pageNameEntity.getStartTime(),
 					readAttrs).one();
 
 		if (pageEntity == null || !pageEntity.getName().equals(pageNameEntity.getName()))
@@ -1073,9 +1073,9 @@ protected PageInfo doUpdatePageName(BookId bookId, String oldPageName, String ne
 		if (pageNameEntity == null)
 			throw new CradleStorageException(String.format("Page \"%s\" not found in book \"%s\"", oldPageName, bookId.getName()));
 
-		PageEntity pageEntity = pageOperator.get(bookId.getName(),
+		PageEntity pageEntity = pageOperator.getAllAfter(bookId.getName(),
 				pageNameEntity.getStartDate(),
-				pageNameEntity.getStartTime().minusNanos(1),
+				pageNameEntity.getStartTime(),
 				readAttrs).one();
 
 		if (pageEntity == null || !pageEntity.getName().equals(pageNameEntity.getName()))

diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraStorageSettings.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraStorageSettings.java
@@ -51,10 +51,10 @@ public class CassandraStorageSettings extends CoreStorageSettings {
     public static final int DEFAULT_GROUPS_CACHE_SIZE = 10_000;
     public static final int DEFAULT_EVENT_BATCH_DURATION_CACHE_SIZE = 5_000;
     public static final int DEFAULT_PAGE_GROUPS_CACHE_SIZE = 10_000;
-    public static final int DEFAULT_COUNTER_PERSISTENCE_INTERVAL_MS = 1000;
+    public static final int DEFAULT_COUNTER_PERSISTENCE_INTERVAL_MS = 15000;
     public static final long DEFAULT_EVENT_BATCH_DURATION_MILLIS = 5_000;
     public static final long DEFAULT_TIMEOUT = 5000;
-    public static final CompressionType DEFAULT_COMPRESSION_TYPE = CompressionType.ZLIB;
+    public static final CompressionType DEFAULT_COMPRESSION_TYPE = CompressionType.LZ4;
 
     //we need to use Instant.EPOCH instead of Instant.MIN.
     //when cassandra driver tries to convert Instant.MIN to milliseconds using toEpochMilli() it causes long overflow.

diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/ReadThroughBookCache.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/ReadThroughBookCache.java
@@ -32,6 +32,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Function;
@@ -89,24 +90,42 @@ public Collection<PageInfo> loadPageInfo(BookId bookId, boolean loadRemoved) {
         return result;
     }
 
-    public Collection<PageInfo> loadPageInfo(BookId bookId, Instant start, Instant end, boolean loadRemoved) {
-        Collection<PageInfo> result = new ArrayList<>();
+    /**
+     * Executes request to Cassandra and filter results by parameters.
+     * This method can take much time when cradle contains a lot of pages.
+     * @param bookId - book id
+     * @param start - start timestamp (inclusive)
+     * @param end - end timestamp (exclusive)
+     * @param loadRemoved - if true, the method add pages marked as removed into result collection
+     * @return collection with pages which covered interval [start, end) where start is inclusive and end is exclusive.
+     */
+    Collection<PageInfo> loadPageInfo(BookId bookId, Instant start, Instant end, boolean loadRemoved) {
+        List<PageInfo> result = new ArrayList<>();
         LocalDate startDate = start != null ? toLocalDate(start) : LocalDate.MIN;
         LocalTime startTime = start != null ? toLocalTime(start) : LocalTime.MIN;
         LocalDate endDate = end != null ? toLocalDate(end) : LocalDate.MAX;
         LocalTime endTime = end != null ? toLocalTime(end) : LocalTime.MAX;
-        for (PageEntity pageEntity : operators.getPageOperator().get(
+        for (PageEntity pageEntity : operators.getPageOperator().getAllDescBefore(
                 bookId.getName(),
-                startDate,
-                startTime,
                 endDate,
                 endTime,
                 readAttrs
         )) {
             if (loadRemoved || pageEntity.getRemoved() == null || pageEntity.getRemoved().equals(DEFAULT_PAGE_REMOVE_TIME)) {
+                if (pageEntity.getEndDate() != null
+                        && pageEntity.getEndTime() != null
+                        && (startDate.isAfter(pageEntity.getEndDate())
+                            || startDate.equals(pageEntity.getEndDate())
+                                && startTime.isAfter(pageEntity.getEndTime()))) {
+                    break;
+                }
                 result.add(pageEntity.toPageInfo());
             }
         }
+        if (result.isEmpty()) {
+            return Collections.emptyList();
+        }
+        Collections.reverse(result);
         return result;
     }
 

diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/books/PageOperator.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/books/PageOperator.java
@@ -35,23 +35,41 @@
 public interface PageOperator {
 	@Select
 	PagingIterable<PageEntity> getAll(String book, Function<BoundStatementBuilder, BoundStatementBuilder> attributes);
-
+
+	/**
+	 * Executes request to Cassandra to receive several pages
+	 * @param book - book id
+	 * @param date - start day
+	 * @param time - start time
+	 * @param attributes - request attributes
+	 * @return iterator for all pages which start datetime is greater or equal than requested datetime - [requested datetime ... ]
+	 */
 	@Query( "SELECT * FROM ${qualifiedTableId} " +
 			"WHERE " +
 				FIELD_BOOK +"=:book AND " +
-			    "(" + FIELD_START_DATE + ", " + FIELD_START_TIME + ") > (:startDate, :startTime)")
-	PagingIterable<PageEntity> get(String book, LocalDate startDate, LocalTime startTime,
-								   Function<BoundStatementBuilder, BoundStatementBuilder> attributes);
+			    "(" + FIELD_START_DATE + ", " + FIELD_START_TIME + ") >= (:date, :time)")
+	PagingIterable<PageEntity> getAllAfter(String book, LocalDate date, LocalTime time,
+										   Function<BoundStatementBuilder, BoundStatementBuilder> attributes);
 
+	/**
+	 * Executes request to Cassandra to receive several pages in descending order
+	 *
+	 * @param book       - book id
+	 * @param date       - day part of timestamp filter
+	 * @param time       - time part of timestamp filter
+	 * @param attributes - request attributes
+	 * @return iterator for all pages in descending order which start datetime is less or equal than requested datetime - [... requested datetime]
+	 */
 	@Query("SELECT * FROM ${qualifiedTableId} " +
 			"WHERE " +
 			FIELD_BOOK +"=:book AND " +
-			"(" + FIELD_START_DATE + ", " + FIELD_START_TIME + ") >= (:startDate, :startTime) " +
-			"AND " +
-			"(" + FIELD_START_DATE + ", " + FIELD_START_TIME + ") <= (:endDate, :endTime)")
-	PagingIterable<PageEntity> get(String book, LocalDate startDate, LocalTime startTime,
-								   LocalDate endDate, LocalTime endTime,
-								   Function<BoundStatementBuilder, BoundStatementBuilder> attributes);
+			"(" + FIELD_START_DATE + ", " + FIELD_START_TIME + ") <= (:date, :time) " +
+			"ORDER BY " +
+			FIELD_START_DATE + " DESC, " +
+			FIELD_START_TIME + " DESC")
+	PagingIterable<PageEntity> getAllDescBefore(String book,
+											LocalDate date, LocalTime time,
+											Function<BoundStatementBuilder, BoundStatementBuilder> attributes);
 
 	@Query("SELECT * FROM ${qualifiedTableId} " +
 			"WHERE " +

diff --git a/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/ReadThroughBookCacheTest.java b/cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/ReadThroughBookCacheTest.java
@@ -1,5 +1,5 @@
 /*
- *  Copyright 2023 Exactpro (Exactpro Systems Limited)
+ *  Copyright 2023-2024 Exactpro (Exactpro Systems Limited)
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -34,14 +34,11 @@
 import java.time.LocalTime;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Function;
 
 import static com.exactpro.cradle.cassandra.CassandraStorageSettings.DEFAULT_PAGE_REMOVE_TIME;
-import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.same;
-import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 import static org.testng.Assert.assertEquals;
@@ -110,25 +107,6 @@ public void beforeMethod() {
         doReturn(pageOperator).when(operators).getPageOperator();
         doReturn(bookOperator).when(operators).getBookOperator();
         doReturn(pagingIterable).when(pageOperator).getAll(same(bookId.getName()), same(readAttrs));
-        doAnswer(invocation -> {
-            LocalDate startDate = invocation.getArgument(1);
-            LocalTime startTime = invocation.getArgument(2);
-            LocalDate endDate = invocation.getArgument(3);
-            LocalTime endTime = invocation.getArgument(4);
-
-            List<PageEntity> result = new ArrayList<>();
-            for (PageEntity pageEntity : pagingIterable) {
-                if ((startDate == null || !startDate.isAfter(pageEntity.getStartDate())) &&
-                        (startTime == null || !startTime.isAfter(pageEntity.getStartTime())) &&
-                        (endDate == null || pageEntity.getEndDate() == null || !endDate.isBefore(pageEntity.getEndDate())) &&
-                        (endTime == null || pageEntity.getEndTime() == null || !endTime.isBefore(pageEntity.getEndTime()))) {
-                    result.add(pageEntity);
-                }
-            }
-            PagingIterable<PageEntity> iterable = mock(PagingIterable.class);
-            doReturn(result.iterator()).when(iterable).iterator();
-            return iterable;
-        }).when(pageOperator).get(same(bookId.getName()), any(), any(), any(), any(), same(readAttrs));
         doReturn(bookEntity).when(bookOperator).get(same(bookId.getName()), same(readAttrs));
     }