Perf test

README.md

@@ -1,4 +1,4 @@
-# Overview (5.4.0)
+# Overview (5.5.0)
 
 Event store (estore) is an important th2 component responsible for storing events into Cradle. Please refer to [Cradle repository] (https://github.com/th2-net/cradleapi/blob/master/README.md) for more details. This component has a pin for listening events via MQ.
 
@@ -73,6 +73,11 @@ Please see more details about this feature via [link](https://github.com/th2-net
 
 # Changes
 
+## 5.5.0
+
+* Updated bom: `4.6.0`
+* Updated common: `5.9.1-dev`
+
 ## 5.4.0
 
 * Updated cradle api: `5.2.0-dev`

build.gradle

@@ -9,9 +9,10 @@ plugins {
     id 'java-library'
     id 'application'
     id 'com.palantir.docker' version '0.25.0'
-    id "org.owasp.dependencycheck" version "8.4.0"
+    id "org.owasp.dependencycheck" version "9.0.9"
     id 'com.github.jk1.dependency-license-report' version '2.5'
     id "de.undercouch.download" version "5.4.0"
+    id "me.champeau.jmh" version "0.7.2"
 }
 
 ext {
@@ -66,6 +67,14 @@ jar {
     }
 }
 
+jmh {
+    jmhTimeout = "1m"
+    iterations = 3
+    fork = 2
+    warmupIterations = 3
+    warmupForks = 2
+}
+
 configurations.configureEach {
     resolutionStrategy {
         force "com.exactpro.th2:cradle-core:$cradleVersion"
@@ -161,8 +170,8 @@ signing {
 }
 
 dependencies {
-    api platform('com.exactpro.th2:bom:4.5.0')
-    implementation 'com.exactpro.th2:common:5.8.0-dev'
+    api platform('com.exactpro.th2:bom:4.6.0')
+    implementation 'com.exactpro.th2:common:5.9.1-dev'
     implementation("com.exactpro.th2:common-utils:2.2.2-dev") {
         because("executor service utils is used")
     }
@@ -182,13 +191,18 @@ dependencies {
         because("Error collector serialise Instant values")
     }
 
+    jmh 'org.openjdk.jmh:jmh-core:1.37'
+    jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.37'
+
     testImplementation 'org.apache.commons:commons-lang3'
     testImplementation 'org.junit.jupiter:junit-jupiter:5.10.2'
     testImplementation 'org.mockito:mockito-junit-jupiter:5.10.0'
+    testImplementation 'com.exactpro.th2:junit-jupiter-integration:0.0.1-master-6956603819-5241ee5-SNAPSHOT'
 }
 
 test {
     useJUnitPlatform()
+    maxHeapSize = "3G"
 }
 
 application {

gradle.properties

@@ -1,3 +1,3 @@
-release_version=5.4.0
+release_version=5.5.0
 description='th2 estore component'
 vcs_url=https://github.com/th2-net/th2-estore

src/jmh/java/com/exactpro/th2/estore/EventWrapperBenchmark.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright 2024 Exactpro (Exactpro Systems Limited)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.exactpro.th2.estore;
+
+import com.exactpro.cradle.CradleEntitiesFactory;
+import com.exactpro.cradle.utils.CradleStorageException;
+import com.exactpro.th2.common.grpc.Direction;
+import com.exactpro.th2.common.grpc.Event;
+import com.exactpro.th2.common.grpc.EventBatch;
+import com.exactpro.th2.common.grpc.EventID;
+import com.exactpro.th2.common.grpc.MessageID;
+import com.google.protobuf.UnsafeByteOperations;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+
+import java.time.Instant;
+import java.util.Set;
+import java.util.UUID;
+
+import static com.exactpro.cradle.CoreStorageSettings.DEFAULT_BOOK_REFRESH_INTERVAL_MILLIS;
+import static com.exactpro.cradle.CradleStorage.DEFAULT_MAX_MESSAGE_BATCH_SIZE;
+import static com.exactpro.cradle.CradleStorage.DEFAULT_MAX_TEST_EVENT_BATCH_SIZE;
+import static com.exactpro.th2.common.utils.message.MessageUtilsKt.toTimestamp;
+import static org.openjdk.jmh.annotations.Mode.Throughput;
+
+@State(Scope.Benchmark)
+public class EventWrapperBenchmark {
+    private static final CradleEntitiesFactory FACTORY = new CradleEntitiesFactory(DEFAULT_MAX_MESSAGE_BATCH_SIZE, DEFAULT_MAX_TEST_EVENT_BATCH_SIZE, DEFAULT_BOOK_REFRESH_INTERVAL_MILLIS);
+    public static final String BOOK = "benchmark-book";
+    private static final String SCOPE = "benchmark-scope";
+    private static final String SESSION_ALIAS_PREFIX = "benchmark-alias-";
+    private static final String EVENT_NAME_PREFIX = "benchmark-event-";
+    private static final int CONTENT_SIZE = 500;
+    private static final int EVENT_NUMBER = 100;
+    private static final int SESSION_ALIAS_NUMBER = 5;
+    private static final int MESSAGES_PER_DIRECTION = 2;
+    @State(Scope.Thread)
+    public static class EventBatchState {
+        private EventBatch batch;
+        @Setup
+        public void init() {
+            EventID parentId = EventID.newBuilder()
+                    .setBookName(BOOK)
+                    .setScope(SCOPE)
+                    .setStartTimestamp(toTimestamp(Instant.now()))
+                    .setId(UUID.randomUUID().toString())
+                    .build();
+            EventBatch.Builder batchBuilder = EventBatch.newBuilder()
+                            .setParentEventId(parentId);
+
+            int seqCounter = 0;
+            for (int eventIndex = 0; eventIndex < EVENT_NUMBER; eventIndex++) {
+                Event.Builder eventBuilder = Event.newBuilder()
+                        .setId(EventID.newBuilder()
+                                .setBookName(BOOK)
+                                .setScope(SCOPE)
+                                .setStartTimestamp(toTimestamp(Instant.now()))
+                                .setId(UUID.randomUUID().toString()))
+                        .setParentId(parentId)
+                        .setName(EVENT_NAME_PREFIX + eventIndex)
+                        .setBody(UnsafeByteOperations.unsafeWrap(RandomStringUtils.random(CONTENT_SIZE, true, true).getBytes()));
+
+                for (int aliasIndex = 0; aliasIndex < SESSION_ALIAS_NUMBER; aliasIndex++) {
+                    for (Direction direction : Set.of(Direction.FIRST, Direction.SECOND)) {
+                        for (int msgIndex = 0; msgIndex < MESSAGES_PER_DIRECTION; msgIndex++) {
+                            MessageID.Builder messageIdBuilder = MessageID.newBuilder()
+                                    .setBookName(BOOK)
+                                    .setDirection(direction)
+                                    .setTimestamp(toTimestamp(Instant.now()))
+                                    .setSequence(++seqCounter);
+                            messageIdBuilder.getConnectionIdBuilder()
+                                            .setSessionAlias(SESSION_ALIAS_PREFIX + aliasIndex);
+                            eventBuilder.addAttachedMessageIds(messageIdBuilder.build());
+                        }
+                    }
+                }
+                batchBuilder.addEvents(eventBuilder.build());
+            }
+            batch = batchBuilder.build();
+        }
+    }
+
+    @Benchmark
+    @BenchmarkMode({Throughput})
+    public void benchmarkToCradleBatch(EventBatchState state) throws CradleStorageException {
+        IEventWrapper.ProtoEventWrapper.toCradleBatch(FACTORY, state.batch);
+    }
+}

src/main/java/com/exactpro/th2/estore/ErrorCollector.java

@@ -46,19 +46,19 @@
 @SuppressWarnings("unused")
 public class ErrorCollector implements AutoCloseable {
     private static final Logger LOGGER = LoggerFactory.getLogger(ErrorCollector.class);
-    private static final Callback<TestEventToStore> PERSIST_CALL_BACK = new LogCallBack(LOGGER, Level.TRACE);
+    private static final Callback<IEventWrapper> PERSIST_CALL_BACK = new LogCallBack(LOGGER, Level.TRACE);
     private static final ThreadLocal<ObjectMapper> OBJECT_MAPPER = ThreadLocal.withInitial(() ->
             new ObjectMapper()
                     .registerModule(new JavaTimeModule())
                     // otherwise, type supported by JavaTimeModule will be serialized as array of date component
                     .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
                     .setSerializationInclusion(NON_NULL));
-    private static final Persistor<TestEventToStore> DYMMY_PERSISTOR = new DymmyPersistor();
+    private static final Persistor<IEventWrapper> DYMMY_PERSISTOR = new DymmyPersistor();
     private final ScheduledFuture<?> drainFuture;
     private final CradleEntitiesFactory entitiesFactory;
     private final Lock lock = new ReentrantLock();
     private volatile StoredTestEventId rootEvent;
-    private volatile Persistor<TestEventToStore> persistor = DYMMY_PERSISTOR;
+    private volatile Persistor<IEventWrapper> persistor = DYMMY_PERSISTOR;
     private Map<String, ErrorMetadata> errors = new HashMap<>();
 
     public ErrorCollector(@NotNull ScheduledExecutorService executor,
@@ -76,7 +76,7 @@ public ErrorCollector(@NotNull ScheduledExecutorService executor,
         this(executor, entitiesFactory, 1, TimeUnit.MINUTES);
     }
 
-    public void init(@NotNull Persistor<TestEventToStore> persistor, StoredTestEventId rootEvent) {
+    public void init(@NotNull Persistor<IEventWrapper> persistor, StoredTestEventId rootEvent) {
         this.persistor = requireNonNull(persistor, "Persistor factory can't be null");
         this.rootEvent = requireNonNull(rootEvent, "Root event id can't be null");
     }
@@ -124,7 +124,7 @@ private void drain() {
             if (map.isEmpty()) { return; }
 
             Instant now = Instant.now();
-            TestEventSingleToStore eventToStore = entitiesFactory.testEventBuilder()
+            IEventWrapper eventWrapper = IEventWrapper.wrap(entitiesFactory.testEventBuilder()
                     .id(new StoredTestEventId(rootEvent.getBookId(), rootEvent.getScope(), now, Util.generateId()))
                     .name("estore internal problem(s): " + calculateTotalQty(map.values()))
                     .type("InternalError")
@@ -133,9 +133,9 @@ private void drain() {
                     .endTimestamp(now)
                     // Content wrapped to list to use the same format as mstore
                     .content(OBJECT_MAPPER.get().writeValueAsBytes(List.of(new BodyData(map))))
-                    .build();
+                    .build());
 
-            persistor.persist(eventToStore, PERSIST_CALL_BACK);
+            persistor.persist(eventWrapper, PERSIST_CALL_BACK);
         } catch (Exception e) {
             LOGGER.error("Drain events task failure", e);
         }
@@ -200,10 +200,10 @@ public void setQuantity(int quantity) {
         }
     }
 
-    private static class DymmyPersistor implements Persistor<TestEventToStore> {
+    private static class DymmyPersistor implements Persistor<IEventWrapper> {
 
         @Override
-        public void persist(TestEventToStore data, Callback<TestEventToStore> callback) {
+        public void persist(IEventWrapper data, Callback<IEventWrapper> callback) {
             LOGGER.warn( "{} isn't initialised", ErrorCollector.class.getSimpleName());
         }
     }