diff --git a/README.md b/README.md index a12e7c23..856cda33 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Cradle API (3.1.3) +# Cradle API (3.1.4) ## Overview @@ -37,7 +37,7 @@ repositories { To use Cradle API, add the following dependency to your project: ``` dependencies { - implementation 'com.exactpro.th2:cradle-cassandra:3.1.3' + implementation 'com.exactpro.th2:cradle-cassandra:3.1.4' ... } ``` @@ -142,6 +142,10 @@ Test events have mandatory parameters that are verified when storing an event. T ## Release notes +### 3.1.4 + ++ Fixed poor performance bug while maintaining event batch durations + ### 3.1.3 (This version requires migration from previous versions) + Fixed Unicode string serialization bugs diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraCradleStorage.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraCradleStorage.java index 7c0ad420..bf7ae6ba 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraCradleStorage.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/CassandraCradleStorage.java @@ -35,7 +35,10 @@ import com.exactpro.cradle.cassandra.dao.intervals.IntervalSupplies; import com.exactpro.cradle.cassandra.dao.messages.*; import com.exactpro.cradle.cassandra.dao.messages.converters.TimeMessageConverter; -import com.exactpro.cradle.cassandra.dao.testevents.*; +import com.exactpro.cradle.cassandra.dao.testevents.ChildrenDatesEventEntity; +import com.exactpro.cradle.cassandra.dao.testevents.DateEventEntity; +import com.exactpro.cradle.cassandra.dao.testevents.DateTimeEventEntity; +import com.exactpro.cradle.cassandra.dao.testevents.DetailedTestEventEntity; import com.exactpro.cradle.cassandra.dao.testevents.converters.DateEventEntityConverter; import com.exactpro.cradle.cassandra.iterators.*; import com.exactpro.cradle.cassandra.retries.*; @@ -46,13 +49,16 @@ import com.exactpro.cradle.messages.StoredMessageBatch; import com.exactpro.cradle.messages.StoredMessageFilter; import com.exactpro.cradle.messages.StoredMessageId; -import com.exactpro.cradle.testevents.*; +import com.exactpro.cradle.testevents.StoredTestEvent; +import com.exactpro.cradle.testevents.StoredTestEventId; +import com.exactpro.cradle.testevents.StoredTestEventMetadata; +import com.exactpro.cradle.testevents.StoredTestEventWrapper; import com.exactpro.cradle.utils.CradleStorageException; import com.exactpro.cradle.utils.MessageUtils; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; + +import java.io.IOException; import java.time.*; import java.util.*; import java.util.concurrent.CompletableFuture; @@ -61,7 +67,7 @@ import java.util.function.Function; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.*; -import static com.exactpro.cradle.cassandra.CassandraStorageSettings.*; +import static com.exactpro.cradle.cassandra.CassandraStorageSettings.INSTANCES_TABLE_DEFAULT_NAME; import static com.exactpro.cradle.cassandra.StorageConstants.*; import static java.lang.String.format; @@ -285,22 +291,24 @@ protected CompletableFuture doStoreTestEventAsync(StoredTestEvent event) List> futures = new ArrayList<>(); try { - DetailedTestEventEntity detailedEntity = new DetailedTestEventEntity(event, instanceUuid); + DetailedTestEventEntity entity = new DetailedTestEventEntity(event, instanceUuid); + var wrapper = entity.toStoredTestEventWrapper(); + CompletableFuture updateMaxDuration = CompletableFuture.completedFuture(null); try { // if possible extract and store duration for this batch - if (detailedEntity.getStartTime() != null && detailedEntity.getEndTime() != null) { + if (wrapper.getStartTimestamp() != null && wrapper.getMaxStartTimestamp() != null) { updateMaxDuration = eventBatchDurationWorker.updateMaxDuration( - new EventBatchDurationCache.CacheKey(instanceUuid, detailedEntity.getStartDate()), - Duration.between(detailedEntity.getStartTime(), detailedEntity.getEndTime()).toMillis()); + instanceUuid, + entity.getStartDate(), + Duration.between(wrapper.getStartTimestamp(), wrapper.getMaxStartTimestamp()).toMillis()); } - } catch (CradleStorageException e) { - logger.error("Could not update max length for event batch with date {}", detailedEntity.getStartDate()); + logger.error("Could not update max length for event batch with date {}", entity.getStartDate()); throw new CradleStorageException("Could not update max length for event batch", e); } - futures.add(storeTimeEvent(detailedEntity)); + futures.add(storeTimeEvent(entity)); futures.add(updateMaxDuration); futures.add(storeDateTime(new DateTimeEventEntity(event, instanceUuid))); futures.add(storeChildrenDates(new ChildrenDatesEventEntity(event, instanceUuid))); @@ -1416,10 +1424,7 @@ private interface Callback { adjusts query params accordingly */ private TestEventsQueryParams getAdjustedQueryParams (StoredTestEventId parentId, Instant from, Instant to, Order order) throws CradleStorageException { - long maxBatchDurationMillis = eventBatchDurationWorker.getMaxDuration( - new EventBatchDurationCache.CacheKey(instanceUuid, LocalDateTime.ofInstant(from, TIMEZONE_OFFSET).toLocalDate())); - - + long maxBatchDurationMillis = eventBatchDurationWorker.getMaxDuration(instanceUuid, LocalDateTime.ofInstant(from, TIMEZONE_OFFSET).toLocalDate()); return new TestEventsQueryParams(parentId, null, null, from, to, order, maxBatchDurationMillis); } diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/EventBatchDurationCache.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/EventBatchDurationCache.java index e5b25a48..163d7724 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/EventBatchDurationCache.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/EventBatchDurationCache.java @@ -15,65 +15,60 @@ */ package com.exactpro.cradle.cassandra; -import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.shaded.guava.common.cache.Cache; import com.datastax.oss.driver.shaded.guava.common.cache.CacheBuilder; -import com.exactpro.cradle.cassandra.dao.testevents.EventBatchMaxDurationOperator; -import com.exactpro.cradle.cassandra.dao.testevents.EventBatchMaxDurationEntity; -import com.exactpro.cradle.utils.CradleStorageException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.time.LocalDate; +import java.util.Objects; import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.function.Function; public class EventBatchDurationCache { - - private final Logger logger = LoggerFactory.getLogger(EventBatchDurationCache.class); - - public static class CacheKey { - private final UUID uuid; - private final LocalDate date; - - public CacheKey (UUID uuid, LocalDate date) { - this.uuid = uuid; - this.date = date; - } - - public UUID getUuid() { - return uuid; - } - - public LocalDate getDate() { - return date; - } - } - - private final Cache durationsCache; + private final Cache cache; public EventBatchDurationCache(int limit) { - this.durationsCache = CacheBuilder.newBuilder().maximumSize(limit).build(); + this.cache = CacheBuilder.newBuilder().maximumSize(limit).build(); } public Long getMaxDuration (CacheKey cacheKey) { - synchronized (durationsCache) { - return durationsCache.getIfPresent(cacheKey); + synchronized (cache) { + return cache.getIfPresent(cacheKey); } } public void updateCache(CacheKey key, long duration) { - synchronized (durationsCache) { - Long cached = durationsCache.getIfPresent(key); - + synchronized (cache) { + Long cached = cache.getIfPresent(key); if (cached != null) { if (cached > duration) { return; } } + cache.put(key, duration); + } + } + + public static class CacheKey { + public final UUID uuid; + public final LocalDate date; + + public CacheKey (UUID uuid, LocalDate date) { + this.uuid = uuid; + this.date = date; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (!(o instanceof CacheKey)) + return false; + CacheKey key = (CacheKey) o; + return uuid.equals(key.uuid) && date.equals(key.date); + } - durationsCache.put(key, duration); + @Override + public int hashCode() { + return Objects.hash(uuid, date); } } } \ No newline at end of file diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/EventBatchDurationWorker.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/EventBatchDurationWorker.java index 7b3fa906..2c1ef93d 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/EventBatchDurationWorker.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/EventBatchDurationWorker.java @@ -22,6 +22,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.LocalDate; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.function.Function; @@ -49,31 +51,43 @@ public EventBatchDurationWorker( this.defaultBatchDurationMillis = defaultBatchDurationMillis; } - public CompletableFuture updateMaxDuration(EventBatchDurationCache.CacheKey key, long duration) throws CradleStorageException { - Long cachedDuration = cache.getMaxDuration(key); - if (cachedDuration != null) { - if (cachedDuration > duration) { - return CompletableFuture.completedFuture(null); - } - } + public CompletableFuture updateMaxDuration(UUID instanceId, LocalDate date, long duration) throws CradleStorageException { + return CompletableFuture.supplyAsync(() -> { - return operator.writeMaxDuration(key.getUuid(), key.getDate(), duration, writeAttrs) - .thenAcceptAsync((res) -> operator.updateMaxDuration(key.getUuid(), key.getDate(), duration, duration, writeAttrs)) - .whenComplete((rtn, e) -> { - if (e != null) { - cache.updateCache(key, duration); - } - }); + EventBatchDurationCache.CacheKey key = new EventBatchDurationCache.CacheKey(instanceId, date); + Long cachedDuration = cache.getMaxDuration(key); + var entity = new EventBatchMaxDurationEntity(key.uuid, key.date, duration); + + if (cachedDuration != null) { + if (cachedDuration < duration) { + // we have already persisted some duration before as we have some value in cache, + // so we can just update the record in the database + operator.updateMaxDuration(entity, duration, writeAttrs); + cache.updateCache(key, duration); + return null; + } + } else { + // we don't have any duration cached, so we don't know if record exists in database + // first try to insert and if no success then try to update + boolean inserted = operator.writeMaxDuration(entity, writeAttrs); + if (!inserted) { + operator.updateMaxDuration(entity, duration, writeAttrs); + } + cache.updateCache(key, duration); + } + + return null; + }); } - public long getMaxDuration(EventBatchDurationCache.CacheKey key) { - EventBatchMaxDurationEntity entity = operator.getMaxDuration(key.getUuid(), key.getDate(), readAttrs); - if (entity == null) { - logger.trace("Could not get max duration for key ({}, {}), returning default value", key.getUuid(), key.getDate()); + public long getMaxDuration(UUID instanceId, LocalDate date) { + EventBatchMaxDurationEntity entity = operator.getMaxDuration(instanceId, date, readAttrs); + if (entity == null) { + logger.trace("Could not get max duration for key ({}, {}), returning default value", instanceId, date); return defaultBatchDurationMillis; } diff --git a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/testevents/EventBatchMaxDurationOperator.java b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/testevents/EventBatchMaxDurationOperator.java index f648ec25..a425162e 100644 --- a/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/testevents/EventBatchMaxDurationOperator.java +++ b/cradle-cassandra/src/main/java/com/exactpro/cradle/cassandra/dao/testevents/EventBatchMaxDurationOperator.java @@ -17,31 +17,28 @@ import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.mapper.annotations.Dao; -import com.datastax.oss.driver.api.mapper.annotations.Query; +import com.datastax.oss.driver.api.mapper.annotations.Insert; import com.datastax.oss.driver.api.mapper.annotations.Select; +import com.datastax.oss.driver.api.mapper.annotations.Update; import java.time.LocalDate; import java.util.UUID; -import java.util.concurrent.CompletableFuture; import java.util.function.Function; -import static com.exactpro.cradle.cassandra.StorageConstants.*; +import static com.exactpro.cradle.cassandra.StorageConstants.MAX_BATCH_DURATION; @Dao public interface EventBatchMaxDurationOperator { - @Query("INSERT into ${qualifiedTableId} (" + INSTANCE_ID + ", " + START_DATE + ", " + MAX_BATCH_DURATION + ") " - + "VALUES (:uuid, :startDate, :duration) " - + "IF NOT EXISTS") - CompletableFuture writeMaxDuration(UUID uuid, LocalDate startDate, long duration, Function attributes); + @Insert(ifNotExists = true) + Boolean writeMaxDuration (EventBatchMaxDurationEntity entity, + Function attributes); - - @Query("UPDATE ${qualifiedTableId} SET " + MAX_BATCH_DURATION + "= :duration " - + "WHERE " + INSTANCE_ID + "=:uuid AND " + START_DATE + "= :startDate " - + "IF " + MAX_BATCH_DURATION + "<:maxDuration") - void updateMaxDuration(UUID uuid, LocalDate startDate, Long duration, Long maxDuration, Function attributes); + @Update(customIfClause = MAX_BATCH_DURATION + "<:maxDuration") + Boolean updateMaxDuration(EventBatchMaxDurationEntity entity, Long maxDuration, + Function attributes); @Select - EventBatchMaxDurationEntity getMaxDuration(UUID uuid, LocalDate localDate, Function attributes); + EventBatchMaxDurationEntity getMaxDuration(UUID uuid, LocalDate localDate, Function attributes); } diff --git a/gradle.properties b/gradle.properties index a7145d63..408c4b9a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -release_version = 3.1.3 +release_version = 3.1.4 description = 'Cradle API'