[Th2-5165] Use cache in BookInfo (#253)

* bookId, page name, page start are unique page identifier * Added book info metrics * [TH2-5165] Removed ThreadSafeProvider class * [TH2-5165] Corrected java doc in BookInfo * [TH2-5165] Corrected after review * [TH2-5165] Corrected after review * [TH2-5165] Fixed problem with Instant MIN/MAX conversion problem * [TH2-5165] Corrected book cache behaviour when book has gap * [TH2-5165] Added testAddPageInTheMiddleOfExist * [TH2-5165] corrected after review * Update cradle-cassandra/src/test/java/com/exactpro/cradle/cassandra/integration/BaseCradleCassandraTest.java Co-authored-by: Oleg Smirnov <[email protected]> * [TH2-5165] corrected after review * [TH2-5165] removed java 17 API * [TH2-5165] corrected book info metrics * [TH2-5165] added testAddTheSecondPageBeforeThreshold test * [TH2-5165] added randomAccessDaysInvalidateInterval option * [TH2-5165] corrected after review * [TH2-5165] revert BOOK_LABEL to book * [TH2-5165] revert logic to skip register load for empty day page interval * [TH2-5165] revert logic to skip register load for empty day page interval * [TH2-5165] merged with dev-version-5 * [TH2-5165] added BookWithoutPageTest * [TH2-5165] corrected after review * [TH2-5165] Updated github workflow --------- Co-authored-by: Oleg Smirnov <[email protected]>
th2-net · Mar 6, 2024 · 40c0478 · 40c0478
1 parent e6e84ec
commit 40c0478
Show file tree

Hide file tree

Showing 70 changed files with 3,708 additions and 1,162 deletions.
diff --git a/...ows/dev-release-java-publish-sonatype.yml → .github/workflows/build-dev-release.yml b/...ows/dev-release-java-publish-sonatype.yml → .github/workflows/build-dev-release.yml
@@ -1,17 +1,14 @@
-name: Build and publish dev-release Java distributions to sonatype.
+name: Build and publish dev release jar to sonatype repository
 
-on:
-  push:
-    tags:
-      - \d+.\d+.\d+-dev
+on: workflow_dispatch
 
 jobs:
   build:
     uses: th2-net/.github/.github/workflows/compound-java.yml@main
     with:
       build-target: 'Sonatype'
-      runsOn: ubuntu-latest
       devRelease: true
+      createTag: true
     secrets:
       sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }}
       sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}

diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml
@@ -0,0 +1,16 @@
+name: Build and publish release jar to sonatype repository
+
+on: workflow_dispatch
+
+jobs:
+  build:
+    uses: th2-net/.github/.github/workflows/compound-java.yml@main
+    with:
+      build-target: 'Sonatype'
+      devRelease: false
+      createTag: true
+    secrets:
+      sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }}
+      sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}
+      sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }}
+      sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
diff --git a/...b/workflows/dev-java-publish-sonatype.yml → .github/workflows/build-sanpshot.yml b/...b/workflows/dev-java-publish-sonatype.yml → .github/workflows/build-sanpshot.yml
@@ -1,19 +1,21 @@
-name: Dev build and publish Java distributions to sonatype snapshot repository
+name: Build and publish jar to sonatype snapshot repository
+
 on:
   push:
     branches-ignore:
-    - master
-    - version-*
+      - master
+      - version-*
+      - dependabot*
+    paths-ignore:
+      - README.md
 
 jobs:
   build-job:
     uses: th2-net/.github/.github/workflows/compound-java-dev.yml@main
     with:
-      multiproject: true
       build-target: 'Sonatype'
-      runsOn: ubuntu-latest
     secrets:
       sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }}
       sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}
       sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }}
-      sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
+      sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -0,0 +1,26 @@
+name: "Run integration tests for cradle API"
+
+on:
+  push:
+    branches:
+      - '*'
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 'zulu' '11'
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: '11'
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+      - name: Build with Gradle
+        run: ./gradlew --info clean integrationTest
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: integration-test-results
+          path: build/reports/tests/integrationTest/
diff --git a/.github/workflows/java-publish-sonatype.yml b/.github/workflows/java-publish-sonatype.yml
diff --git a/README.md b/README.md
@@ -193,6 +193,25 @@ Events in a batch can have a reference only to the parent of the batch or other
 
 Test events have mandatory parameters that are verified when storing an event. These are: id, name (for non-batch events), start timestamp.
 
+## Metrics
+
+* `cradle_page_cache_size`               (type: Gauge,   labels: book, cache)         - Size of page cache.
+* `cradle_page_cache_page_request_total` (type: Counter, labels: book, cache, method) - Page requests number from cache
+* `cradle_page_cache_invalidate_total`   (type: Counter, labels: book, cache, cause)  - Cache invalidates
+* `cradle_page_cache_page_loads_total`   (type: Summary, labels: book, cache)         - Page loads number to cache
+  * `_count` - loaded page day intervals
+  * `_sum` - loaded pages
+
+### Labels:
+  * cache: HOT, RANDOM
+  * method: GET, NEXT, PREVIOUS, FIND, ITERATE, REFRESH
+  * cause: EXPLICIT, REPLACED, COLLECTED, EXPIRED, SIZE
+
+## Release notes
+
+### 5.2.0
+* Added page cache for each book to avoid memory leak
+
 ## Changes
 
 ### 5.1.5

diff --git a/build.gradle b/build.gradle
@@ -55,6 +55,10 @@ subprojects {
     targetCompatibility = JavaVersion.VERSION_11 //Java version to generate classes for.
     compileJava.options.debugOptions.debugLevel = 'source,lines,vars' // Include debug information
 
+    dependencies {
+        implementation 'io.prometheus:simpleclient_dropwizard:0.16.0'
+    }
+
     jar {
         manifest {
             attributes(

diff --git a/cradle-cassandra/build.gradle b/cradle-cassandra/build.gradle
@@ -13,7 +13,6 @@ dependencies {
     implementation "com.datastax.oss:java-driver-query-builder"
     implementation "com.datastax.oss:java-driver-mapper-processor"
     implementation "com.datastax.oss:java-driver-mapper-runtime"
-    implementation 'io.prometheus:simpleclient_dropwizard:0.16.0'
     implementation 'com.google.guava:guava'
 
     // this section is required to bypass failing vulnerability check caused by cassandra driver's transitive dependencies

diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraCradleStorage.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraCradleStorage.java
@@ -1,5 +1,5 @@
 /*
- *  Copyright 2023 Exactpro (Exactpro Systems Limited)
+ *  Copyright 2023-2024 Exactpro (Exactpro Systems Limited)
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -73,7 +73,6 @@
 import com.exactpro.cradle.cassandra.keyspaces.CradleInfoKeyspaceCreator;
 import com.exactpro.cradle.cassandra.metrics.DriverMetrics;
 import com.exactpro.cradle.cassandra.resultset.CassandraCradleResultSet;
-import com.exactpro.cradle.cassandra.resultset.SessionsStatisticsIteratorProvider;
 import com.exactpro.cradle.cassandra.retries.FixedNumberRetryPolicy;
 import com.exactpro.cradle.cassandra.retries.PageSizeAdjustingPolicy;
 import com.exactpro.cradle.cassandra.retries.SelectExecutionPolicy;
@@ -128,6 +127,9 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static com.exactpro.cradle.cassandra.utils.StorageUtils.toLocalDate;
+import static com.exactpro.cradle.cassandra.utils.StorageUtils.toLocalTime;
+
 public class CassandraCradleStorage extends CradleStorage
 {
 	private final Logger LOGGER = LoggerFactory.getLogger(CassandraCradleStorage.class);
@@ -211,7 +213,11 @@ protected void doInit(boolean prepareStorage) throws CradleStorageException
 					.setTimeout(timeout)
 					.setPageSize(resultPageSize);
 			operators = createOperators(connection.getSession(), settings);
-			bookCache = new ReadThroughBookCache(operators, readAttrs, settings.getSchemaVersion());
+			bookCache = new ReadThroughBookCache(operators,
+					readAttrs,
+					settings.getSchemaVersion(),
+					settings.getRandomAccessDaysCacheSize(),
+					settings.getRandomAccessDaysInvalidateInterval());
 			bookManager = new BookManager(getBookCache(), settings.getBookRefreshIntervalMillis());
 			bookManager.start();
 
@@ -294,12 +300,12 @@ protected void doAddPages(BookId bookId, List<PageInfo> pages, PageInfo lastPage
 		String bookName = bookId.getName();
 		for (PageInfo page : pages)
 		{
-			String pageName = page.getId().getName();
+			String pageName = page.getName();
 			try
 			{
 				PageNameEntity nameEntity = new PageNameEntity(bookName, pageName, page.getStarted(), page.getComment(), page.getEnded());
 				if (!pageNameOp.writeNew(nameEntity, writeAttrs).wasApplied())
-					throw new IOException("Query to insert page '"+nameEntity.getName()+"' was not applied. Probably, page already exists");
+					throw new IOException("Query to insert page '"+nameEntity.getName()+"' book '" + bookId.getName() + "' was not applied. Probably, page already exists");
 				PageEntity entity = new PageEntity(bookName, pageName, page.getStarted(), page.getComment(), page.getEnded(), page.getUpdated());
 				pageOp.write(entity, writeAttrs);
 			}
@@ -943,29 +949,6 @@ protected Counter doGetCount(BookId bookId, EntityType entityType, Interval inte
 		}
 	}
 
-	private CompletableFuture<CradleResultSet<String>> doGetSessionsAsync(BookId bookId, Interval interval, SessionRecordType recordType) throws CradleStorageException {
-		String queryInfo = String.format("%s Aliases in book %s from %s to %s",
-				recordType.name(),
-				bookId.getName(),
-				interval.getStart().toString(),
-				interval.getEnd().toString());
-
-		List<FrameInterval> frameIntervals = StorageUtils.sliceInterval(interval);
-
-		SessionsStatisticsIteratorProvider iteratorProvider = new SessionsStatisticsIteratorProvider(
-				queryInfo,
-				operators,
-				getBookCache().getBook(bookId),
-				composingService,
-				selectExecutor,
-				readAttrs,
-				frameIntervals,
-				recordType);
-
-		return iteratorProvider.nextIterator()
-				.thenApplyAsync(it -> new CassandraCradleResultSet<>(it, iteratorProvider));
-	}
-
 	@Override
 	protected CompletableFuture<CradleResultSet<String>> doGetSessionAliasesAsync(BookId bookId, Interval interval) throws CradleStorageException {
 		String queryInfo = String.format("Session Aliases in book %s from pages that fall within %s to %s",
@@ -1311,8 +1294,8 @@ protected void removeTestEventData(PageId pageId) {
 
 	protected void removePageData(PageInfo pageInfo) throws CradleStorageException {
 
-		String book = pageInfo.getId().getBookId().getName();
-		String page = pageInfo.getId().getName();
+		String book = pageInfo.getBookName();
+		String page = pageInfo.getName();
 
 		// remove sessions
 		operators.getPageSessionsOperator().remove(book, page, writeAttrs);
@@ -1371,23 +1354,25 @@ private void removeEntityStatistics(PageId pageId) {
 
 	@Override
 	protected CompletableFuture<Iterator<PageInfo>> doGetPagesAsync (BookId bookId, Interval interval) {
+		Instant start = interval.getStart();
+		Instant end = interval.getEnd();
 		String queryInfo = String.format(
 				"Getting pages for book %s between  %s - %s ",
 				bookId.getName(),
-				interval.getStart(),
-				interval.getEnd());
+				start,
+				end);
 
 		PageEntity startPage = operators.getPageOperator()
 				.getPageForLessOrEqual(
 						bookId.getName(),
-						LocalDate.ofInstant(interval.getStart(), TIMEZONE_OFFSET),
-						LocalTime.ofInstant(interval.getStart(), TIMEZONE_OFFSET),
+						toLocalDate(start),
+						toLocalTime(start),
 						readAttrs).one();
 
 		LocalDate startDate = startPage == null ? LocalDate.MIN : startPage.getStartDate();
 		LocalTime startTime = startPage == null ? LocalTime.MIN : startPage.getStartTime();
-		LocalDate endDate = LocalDate.ofInstant(interval.getEnd(), TIMEZONE_OFFSET);
-		LocalTime endTime = LocalTime.ofInstant(interval.getEnd(), TIMEZONE_OFFSET);
+		LocalDate endDate = toLocalDate(end);
+		LocalTime endTime = toLocalTime(end);
 
 		return operators.getPageOperator().getPagesForInterval(
 				bookId.getName(),
@@ -1408,16 +1393,15 @@ protected CompletableFuture<Iterator<PageInfo>> doGetPagesAsync (BookId bookId,
 
 	@Override
 	protected Iterator<PageInfo> doGetPages(BookId bookId, Interval interval) throws CradleStorageException {
-		String queryInfo = String.format(
-				"Getting pages for book %s between  %s - %s ",
-				bookId.getName(),
-				interval.getStart(),
-				interval.getEnd());
-
 		try {
 			return doGetPagesAsync(bookId, interval).get();
 		} catch (InterruptedException | ExecutionException e) {
-			throw new CradleStorageException("Error while " + queryInfo, e);
+			String error = String.format(
+					"Error while Getting pages for book %s between  %s - %s ",
+					bookId.getName(),
+					interval.getStart(),
+					interval.getEnd());
+			throw new CradleStorageException(error, e);
 		}
 	}
 }