From 54d94d610a8cc516d34d54dc039bda318e509361 Mon Sep 17 00:00:00 2001 From: Satyam Saxena Date: Mon, 6 May 2024 13:40:04 +0530 Subject: [PATCH 01/12] Add Weighted Id Generator --- .../bundle/id/DistributedIdGenerator.java | 286 ++++++++++++++++++ .../discovery/bundle/id/GenerationResult.java | 9 + .../discovery/bundle/id/IdGenerator.java | 32 +- .../ranger/discovery/bundle/id/IdInfo.java | 14 + .../ranger/discovery/bundle/id/IdPool.java | 17 ++ .../bundle/id/IdValidationState.java | 7 + .../bundle/id/PartitionIdTracker.java | 20 ++ .../discovery/bundle/id/PartitionRange.java | 30 ++ .../discovery/bundle/id/WeightedIdConfig.java | 39 +++ .../bundle/id/WeightedIdGenerator.java | 66 ++++ .../bundle/id/WeightedPartition.java | 24 ++ .../constraints/KeyValidationConstraint.java | 10 + .../id/formatter/DistributedIdFormatter.java | 17 ++ .../bundle/id/formatter/IdFormatters.java | 5 + .../id/DistributedIdGeneratorPerfTest.java | 37 +++ .../bundle/id/DistributedIdGeneratorTest.java | 199 ++++++++++++ .../ranger/discovery/bundle/id/TestUtil.java | 64 ++++ .../id/WeightedIdGeneratorPerfTest.java | 51 ++++ .../bundle/id/WeightedIdGeneratorTest.java | 196 ++++++++++++ 19 files changed, 1096 insertions(+), 27 deletions(-) create mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/DistributedIdGenerator.java create mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/GenerationResult.java create mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdInfo.java create mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdPool.java create mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdValidationState.java create mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java create mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionRange.java create mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdConfig.java create mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java create mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedPartition.java create mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/constraints/KeyValidationConstraint.java create mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/DistributedIdFormatter.java create mode 100644 ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorPerfTest.java create mode 100644 ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorTest.java create mode 100644 ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/TestUtil.java create mode 100644 ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java create mode 100644 ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/DistributedIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/DistributedIdGenerator.java new file mode 100644 index 00000000..bed89b14 --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/DistributedIdGenerator.java @@ -0,0 +1,286 @@ +package io.appform.ranger.discovery.bundle.id; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import dev.failsafe.Failsafe; +import dev.failsafe.FailsafeExecutor; +import dev.failsafe.RetryPolicy; +import io.appform.ranger.discovery.bundle.id.constraints.KeyValidationConstraint; +import io.appform.ranger.discovery.bundle.id.formatter.IdFormatter; +import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.regex.Pattern; + +/** + * Distributed Id Generation + */ +@SuppressWarnings("unused") +@Slf4j +public class DistributedIdGenerator { + + private static final int MINIMUM_ID_LENGTH = 22; + protected static final SecureRandom SECURE_RANDOM = new SecureRandom(Long.toBinaryString(System.currentTimeMillis()).getBytes()); + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormat.forPattern("yyMMddHHmmss"); + private final RetryPolicy RETRY_POLICY = RetryPolicy.builder() + .withMaxAttempts(readRetryCount()) + .handleIf(throwable -> true) + .handleResultIf(Objects::isNull) + .build(); + protected final FailsafeExecutor RETRIER = Failsafe.with(Collections.singletonList(RETRY_POLICY)); + private static final Pattern PATTERN = Pattern.compile("(.*)([0-9]{12})([0-9]{4})([0-9]{6})"); + private static final List GLOBAL_CONSTRAINTS = new ArrayList<>(); + private static final Map> DOMAIN_SPECIFIC_CONSTRAINTS = new HashMap<>(); + protected int nodeId; + @Getter + private final Map> dataStore = new ConcurrentHashMap<>(); + protected final IdFormatter idFormatter; + protected final Function partitionResolver; + protected final int partitionCount; + +// dataStore Structure +// { +// prefix: { +// timestamp: { +// partitions: [ +// { +// ids: [], +// pointer: +// }, +// { +// ids: [], +// pointer: +// } ... +// ], +// counter: +// } +// } +// } + + public DistributedIdGenerator(int node, int partitionSize, final Function partitionResolverSupplier) { + nodeId = node; + partitionCount = partitionSize; + partitionResolver = partitionResolverSupplier; + idFormatter = IdFormatters.distributed(); + val executorService = Executors.newScheduledThreadPool(1); + executorService.scheduleWithFixedDelay(this::deleteExpiredKeys, 60, 60, TimeUnit.SECONDS); + } + + public DistributedIdGenerator(int node, + int partitionSize, + final Function partitionResolverSupplier, + final IdFormatter idFormatterInstance) { + nodeId = node; + partitionCount = partitionSize; + partitionResolver = partitionResolverSupplier; + idFormatter = idFormatterInstance; + } + + public synchronized void registerGlobalConstraints(KeyValidationConstraint... constraints) { + registerGlobalConstraints(ImmutableList.copyOf(constraints)); + } + + public synchronized void registerGlobalConstraints(List constraints) { + Preconditions.checkArgument(null != constraints && !constraints.isEmpty()); + GLOBAL_CONSTRAINTS.addAll(constraints); + } + + public synchronized void registerDomainSpecificConstraints( + String domain, + KeyValidationConstraint... validationConstraints) { + registerDomainSpecificConstraints(domain, ImmutableList.copyOf(validationConstraints)); + } + + public synchronized void registerDomainSpecificConstraints( + String domain, + List validationConstraints) { + Preconditions.checkArgument(null != validationConstraints && !validationConstraints.isEmpty()); + DOMAIN_SPECIFIC_CONSTRAINTS.computeIfAbsent(domain, key -> new ArrayList<>()) + .addAll(validationConstraints); + } + + /** + * Generate id with given prefix + * + * @param prefix String prefix for ID to be generated + * @return Generated Id + */ + public Id generate(String prefix) { + val targetPartitionId = getTargetPartitionId(); + return generateForPartition(prefix, targetPartitionId); + } + + public Id generateForPartition(String prefix, int targetPartitionId) { + val prefixIdMap = dataStore.computeIfAbsent(prefix, k -> new ConcurrentHashMap<>()); + val currentTimestamp = new DateTime(); + val timeKey = currentTimestamp.getMillis() / 1000; + val idCounter = generateForAllPartitions( + prefixIdMap.computeIfAbsent(timeKey, key -> new PartitionIdTracker(partitionCount)), + prefix, + currentTimestamp, + targetPartitionId); + val id = String.format("%s%s", prefix, idFormatter.format(currentTimestamp, nodeId, idCounter)); + return Id.builder() + .id(id) + .exponent(idCounter) + .generatedDate(currentTimestamp.toDate()) + .node(nodeId) + .build(); + } + + private int generateForAllPartitions(final PartitionIdTracker partitionIdTracker, + final String prefix, + final DateTime timestamp, + final int targetPartitionId) { + val idPool = partitionIdTracker.getIdList()[targetPartitionId]; + int idIdx = idPool.getPointer().getAndIncrement(); +// ToDo: Add Retry Limit + while (idPool.getIds().size() <= idIdx) { + val counterValue = partitionIdTracker.getCounter().getAndIncrement(); + val txnId = String.format("%s%s", prefix, idFormatter.format(timestamp, nodeId, counterValue)); + val mappedPartitionId = partitionResolver.apply(txnId); + partitionIdTracker.getIdList()[mappedPartitionId].getIds().add(counterValue); + } + return idPool.getId(idIdx); + } + + /** + * Generate id that matches all passed constraints. + * NOTE: There are performance implications for this. + * The evaluation of constraints will take it's toll on ID generation rates. + * + * @param prefix String prefix + * @param domain Domain for constraint selection + * @return Return generated id or empty if it was impossible to satisfy constraints and generate + */ + public Optional generateWithConstraints(final String prefix, final String domain) { + return generateWithConstraints(prefix, domain, true); + } + + /** + * Generate id that matches all passed constraints. + * NOTE: There are performance implications for this. + * The evaluation of constraints will take it's toll on id generation rates. + * + * @param prefix String prefix + * @param domain Domain for constraint selection + * @param skipGlobal Skip global constrains and use only passed ones + * @return Id if it could be generated + */ + public Optional generateWithConstraints(final String prefix, final String domain, final boolean skipGlobal) { + val targetPartitionId = getTargetPartitionId(DOMAIN_SPECIFIC_CONSTRAINTS.getOrDefault(domain, Collections.emptyList()), skipGlobal); + return targetPartitionId.map(id -> generateForPartition(prefix, id)); + } + + public Optional generateWithConstraints(final String prefix, + final List inConstraints, + final boolean skipGlobal) { + val targetPartitionId = getTargetPartitionId(inConstraints, skipGlobal); + return targetPartitionId.map(id -> generateForPartition(prefix, id)); + } + + /** + * Generate id by parsing given string + * + * @param idString String idString + * @return Id if it could be generated + */ + public Optional parse(final String idString) { + if (idString == null + || idString.length() < MINIMUM_ID_LENGTH) { + return Optional.empty(); + } + try { + val matcher = PATTERN.matcher(idString); + if (matcher.find()) { + return Optional.of(Id.builder() + .id(idString) + .node(Integer.parseInt(matcher.group(3))) + .exponent(Integer.parseInt(matcher.group(4))) + .generatedDate(DATE_TIME_FORMATTER.parseDateTime(matcher.group(2)).toDate()) + .build()); + } + return Optional.empty(); + } + catch (Exception e) { + log.warn("Could not parse idString {}", e.getMessage()); + return Optional.empty(); + } + } + + private int getTargetPartitionId() { + return SECURE_RANDOM.nextInt(partitionCount); + } + + private Optional getTargetPartitionId(final List inConstraints, boolean skipGlobal) { +// ToDo: Check if we need Collision Checker here + return Optional.ofNullable( + RETRIER.get(() -> SECURE_RANDOM.nextInt(partitionCount))) + .filter(key -> validateId(inConstraints, key, skipGlobal)); + } + + protected boolean validateId(final List inConstraints, + final int partitionId, + final boolean skipGlobal) { + //First evaluate global constraints + val failedGlobalConstraint + = skipGlobal + ? null + : GLOBAL_CONSTRAINTS.stream() + .filter(constraint -> !constraint.isValid(partitionId)) + .findFirst() + .orElse(null); + if (null != failedGlobalConstraint) { + return false; + } + //Evaluate local + domain constraints + val failedLocalConstraint + = null == inConstraints + ? null + : inConstraints.stream() + .filter(constraint -> !constraint.isValid(partitionId)) + .findFirst() + .orElse(null); + return null == failedLocalConstraint; + } + + protected int readRetryCount() { + try { + val count = Integer.parseInt(System.getenv().getOrDefault("NUM_ID_GENERATION_RETRIES", "512")); + if (count <= 0) { + throw new IllegalArgumentException( + "Negative number of retries does not make sense. Please set a proper value for " + + "NUM_ID_GENERATION_RETRIES"); + } + return count; + } + catch (NumberFormatException e) { + throw new IllegalArgumentException("Please provide a valid positive integer for NUM_ID_GENERATION_RETRIES"); + } + } + + private void deleteExpiredKeys() { + val timeThreshold = DateTime.now().getMillis() / 1000; + for (val entry : dataStore.entrySet()) { + entry.getValue().entrySet().removeIf(partitionIdTrackerEntry -> partitionIdTrackerEntry.getKey() < timeThreshold); + } + } + +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/GenerationResult.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/GenerationResult.java new file mode 100644 index 00000000..b4086901 --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/GenerationResult.java @@ -0,0 +1,9 @@ +package io.appform.ranger.discovery.bundle.id; + +import lombok.Value; + +@Value +class GenerationResult { + Id id; + IdValidationState state; +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGenerator.java index 3a0f9dbd..c8c80b7d 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGenerator.java @@ -26,7 +26,7 @@ import io.appform.ranger.discovery.bundle.id.formatter.IdFormatter; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; import io.appform.ranger.discovery.bundle.id.request.IdGenerationRequest; -import lombok.Value; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.joda.time.DateTime; @@ -67,6 +67,7 @@ public class IdGenerator { private static final List GLOBAL_CONSTRAINTS = new ArrayList<>(); private static final Map> DOMAIN_SPECIFIC_CONSTRAINTS = new HashMap<>(); + @Getter private static int nodeId; public static void initialize(int node) { @@ -126,11 +127,11 @@ public static Id generate(String prefix) { public static Id generate(final String prefix, final IdFormatter idFormatter) { val idInfo = random(); - val dateTime = new DateTime(idInfo.time); - val id = String.format("%s%s", prefix, idFormatter.format(dateTime, nodeId, idInfo.exponent)); + val dateTime = new DateTime(idInfo.getTime()); + val id = String.format("%s%s", prefix, idFormatter.format(dateTime, nodeId, idInfo.getExponent())); return Id.builder() .id(id) - .exponent(idInfo.exponent) + .exponent(idInfo.getExponent()) .generatedDate(dateTime.toDate()) .node(nodeId) .build(); @@ -288,27 +289,4 @@ private static int readRetryCount() { throw new IllegalArgumentException("Please provide a valid positive integer for NUM_ID_GENERATION_RETRIES"); } } - - private enum IdValidationState { - VALID, - INVALID_RETRYABLE, - INVALID_NON_RETRYABLE - } - - @Value - private static class IdInfo { - int exponent; - long time; - - public IdInfo(int exponent, long time) { - this.exponent = exponent; - this.time = time; - } - } - - @Value - private static class GenerationResult { - Id id; - IdValidationState state; - } } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdInfo.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdInfo.java new file mode 100644 index 00000000..742e665d --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdInfo.java @@ -0,0 +1,14 @@ +package io.appform.ranger.discovery.bundle.id; + +import lombok.Value; + +@Value +class IdInfo { + int exponent; + long time; + + public IdInfo(int exponent, long time) { + this.exponent = exponent; + this.time = time; + } +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdPool.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdPool.java new file mode 100644 index 00000000..155d362f --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdPool.java @@ -0,0 +1,17 @@ +package io.appform.ranger.discovery.bundle.id; + +import lombok.Getter; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; + +@Getter +public class IdPool { + private final List ids = new CopyOnWriteArrayList<>(); + private final AtomicInteger pointer = new AtomicInteger(); + + public int getId(int index) { + return ids.get(index); + } +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdValidationState.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdValidationState.java new file mode 100644 index 00000000..b0eb95fa --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdValidationState.java @@ -0,0 +1,7 @@ +package io.appform.ranger.discovery.bundle.id; + +enum IdValidationState { + VALID, + INVALID_RETRYABLE, + INVALID_NON_RETRYABLE +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java new file mode 100644 index 00000000..38fd4fc0 --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java @@ -0,0 +1,20 @@ +package io.appform.ranger.discovery.bundle.id; + +import lombok.Getter; + +import java.util.concurrent.atomic.AtomicInteger; + +@Getter +public class PartitionIdTracker { + private final IdPool[] idList; + private final int partitionSize; + private final AtomicInteger counter = new AtomicInteger(); + + protected PartitionIdTracker(int partitionSize) { + this.partitionSize = partitionSize; + idList = new IdPool[partitionSize]; + for (int i=0; i partitions; + + @ValidationMethod(message = "Invalid Partition Range") + @JsonIgnore + public boolean isPartitionWeightsValid() { + List sortedPartitions = new ArrayList<>(partitions); + sortedPartitions.sort(Comparator.comparingInt(k -> k.getPartitionRange().getStart())); + for (int i = 0; i < sortedPartitions.size() - 1; i++) { + WeightedPartition currentPartition = sortedPartitions.get(i); + WeightedPartition nextPartition = sortedPartitions.get(i + 1); + if (currentPartition.getPartitionRange().getEnd() >= nextPartition.getPartitionRange().getStart()) { + return false; + } + } + return true; + } +} \ No newline at end of file diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java new file mode 100644 index 00000000..bd5f795e --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java @@ -0,0 +1,66 @@ +package io.appform.ranger.discovery.bundle.id; + +import com.google.common.collect.Range; +import com.google.common.collect.RangeMap; +import com.google.common.collect.TreeRangeMap; +import io.appform.ranger.discovery.bundle.id.constraints.KeyValidationConstraint; +import io.appform.ranger.discovery.bundle.id.formatter.IdFormatter; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; + +/** + * Weighted Id Generation + */ +@Slf4j +public class WeightedIdGenerator extends DistributedIdGenerator { + private int maxShardWeight; + private final RangeMap partitionRangeMap; + + public WeightedIdGenerator(int node, + int partitionSize, + final Function partitionResolverSupplier, + final WeightedIdConfig weightedIdConfig) { + super(node, partitionSize, partitionResolverSupplier); + partitionRangeMap = createWeightRangeMap(weightedIdConfig); + } + + public WeightedIdGenerator(int node, + int partitionSize, + final Function partitionResolverSupplier, + final IdFormatter idFormatterInstance, + final WeightedIdConfig weightedIdConfig) { + super(node, partitionSize, partitionResolverSupplier, idFormatterInstance); + partitionRangeMap = createWeightRangeMap(weightedIdConfig); + } + + private RangeMap createWeightRangeMap(WeightedIdConfig weightedIdConfig) { + RangeMap shardGroups = TreeRangeMap.create(); + int start = 0; + int end = -1; + for (val shard : weightedIdConfig.getPartitions()) { + end += shard.getWeight(); + shardGroups.put(Range.closed(start, end), shard.getPartitionRange()); + start = end+1; + } + maxShardWeight = end; + return shardGroups; + } + + private int getTargetPartitionId() { + val randomNum = SECURE_RANDOM.nextInt(maxShardWeight); + val partitionRange = Objects.requireNonNull(partitionRangeMap.getEntry(randomNum)).getValue(); + return SECURE_RANDOM.nextInt(partitionRange.getEnd() - partitionRange.getStart() + 1) + partitionRange.getStart(); + } + + private Optional getTargetPartitionId(final List inConstraints, boolean skipGlobal) { + return Optional.ofNullable( + RETRIER.get(this::getTargetPartitionId)) + .filter(key -> validateId(inConstraints, key, skipGlobal)); + } +} + diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedPartition.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedPartition.java new file mode 100644 index 00000000..ca9969e1 --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedPartition.java @@ -0,0 +1,24 @@ +package io.appform.ranger.discovery.bundle.id; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import javax.validation.Valid; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +@Getter +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class WeightedPartition { + @NotNull + @Valid + private PartitionRange partitionRange; + + @NotNull + @Min(0) + private int weight; +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/constraints/KeyValidationConstraint.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/constraints/KeyValidationConstraint.java new file mode 100644 index 00000000..0d61c7f2 --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/constraints/KeyValidationConstraint.java @@ -0,0 +1,10 @@ +package io.appform.ranger.discovery.bundle.id.constraints; + + +public interface KeyValidationConstraint { + boolean isValid(int id); + + default boolean failFast() { + return false; + } +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/DistributedIdFormatter.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/DistributedIdFormatter.java new file mode 100644 index 00000000..f242fb77 --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/DistributedIdFormatter.java @@ -0,0 +1,17 @@ +package io.appform.ranger.discovery.bundle.id.formatter; + +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +public class DistributedIdFormatter implements IdFormatter { + + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormat.forPattern("yyMMddHHmmss"); + + @Override + public String format(final DateTime dateTime, + final int nodeId, + final int randomNonce) { + return String.format("%s%04d%06d", DATE_TIME_FORMATTER.print(dateTime), nodeId, randomNonce); + } +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatters.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatters.java index 30bca972..f6a4e43c 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatters.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatters.java @@ -23,6 +23,7 @@ public class IdFormatters { private static final IdFormatter originalIdFormatter = new DefaultIdFormatter(); private static final IdFormatter base36IdFormatter = new Base36IdFormatter(originalIdFormatter); + private static final IdFormatter distributedIdFormatter = new DistributedIdFormatter(); public static IdFormatter original() { return originalIdFormatter; @@ -32,4 +33,8 @@ public static IdFormatter base36() { return base36IdFormatter; } + public static IdFormatter distributed() { + return distributedIdFormatter; + } + } diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorPerfTest.java new file mode 100644 index 00000000..a27a5511 --- /dev/null +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorPerfTest.java @@ -0,0 +1,37 @@ +package io.appform.ranger.discovery.bundle.id; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.infra.Blackhole; + +import java.io.IOException; +import java.util.function.Function; + +/** + * Test performance between different constructs + */ +@Slf4j +public class DistributedIdGeneratorPerfTest extends BenchmarkTest { + + @State(Scope.Benchmark) + public static class BenchmarkState { + private DistributedIdGenerator distributedIdGenerator; + final Function partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length()-6)) % 1024; + + @Setup(Level.Trial) + public void setUp() throws IOException { + distributedIdGenerator = new DistributedIdGenerator(23, 1024, partitionResolverSupplier); + } + } + + @SneakyThrows + @Benchmark + public void testGenerate(Blackhole blackhole, BenchmarkState state) { + state.distributedIdGenerator.generate("P"); + } +} \ No newline at end of file diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorTest.java new file mode 100644 index 00000000..a0f744d0 --- /dev/null +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorTest.java @@ -0,0 +1,199 @@ +package io.appform.ranger.discovery.bundle.id; + +import io.appform.ranger.discovery.bundle.id.constraints.KeyValidationConstraint; +import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.math.BigInteger; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Collections; +import java.util.Date; +import java.util.HashSet; +import java.util.Map; +import java.util.function.Function; + +/** + * Test for {@link DistributedIdGenerator} + */ +@Slf4j +@SuppressWarnings({"unused", "FieldMayBeFinal"}) +class DistributedIdGeneratorTest { + final int partitionCount = 1024; + final Function partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length()-6)) % partitionCount; + private DistributedIdGenerator distributedIdGenerator; + + @BeforeEach + void setup() { + distributedIdGenerator = new DistributedIdGenerator(23, partitionCount, partitionResolverSupplier); + } + + @Test + void testGenerateWithBenchmark() throws IOException { + val totalTime = TestUtil.runMTTest(5, 100000, (k) -> distributedIdGenerator.generate("P"), this.getClass().getName() + ".testGenerateWithBenchmark"); + testUniqueIdsInDataStore(distributedIdGenerator.getDataStore()); + } + + @Test + void testGenerateWithConstraints() throws IOException { + KeyValidationConstraint partitionConstraint = (k) -> k % 10 == 0; + distributedIdGenerator.registerGlobalConstraints(partitionConstraint); + val totalTime = TestUtil.runMTTest(5, 100000, (k) -> distributedIdGenerator.generateWithConstraints("P", (String) null, false), this.getClass().getName() + ".testGenerateWithConstraints"); + testUniqueIdsInDataStore(distributedIdGenerator.getDataStore()); + + for (Map.Entry> entry : distributedIdGenerator.getDataStore().entrySet()) { + val prefix = entry.getKey(); + val prefixIds = entry.getValue(); + HashSet uniqueIds = new HashSet<>(); + for (Map.Entry prefixEntry : prefixIds.entrySet()) { + val key = prefixEntry.getKey(); + val partitionIdTracker = prefixEntry.getValue(); + for (int idx=0; idx < partitionIdTracker.getPartitionSize(); idx += 1) { + if (!partitionConstraint.isValid(idx)) { + Assertions.assertEquals(0, partitionIdTracker.getIdList()[idx].getPointer().get()); + } + } + } + } + } + + void testUniqueIdsInDataStore(Map> dataStore) { + boolean allIdsUnique = true; + for (Map.Entry> entry : dataStore.entrySet()) { + val prefix = entry.getKey(); + val prefixIds = entry.getValue(); + for (Map.Entry prefixEntry : prefixIds.entrySet()) { + val key = prefixEntry.getKey(); + val partitionIdTracker = prefixEntry.getValue(); + HashSet uniqueIds = new HashSet<>(); + for (val idPool: partitionIdTracker.getIdList()) { + boolean allIdsUniqueInList = true; + HashSet uniqueIdsInList = new HashSet<>(); + for (val id: idPool.getIds()) { + if (uniqueIdsInList.contains(id)) { + allIdsUniqueInList = false; + allIdsUnique = false; + } else { + uniqueIdsInList.add(id); + } + + if (uniqueIds.contains(id)) { + allIdsUnique = false; + } else { + uniqueIds.add(id); + } + } + Assertions.assertTrue(allIdsUniqueInList); + } + } + } + Assertions.assertTrue(allIdsUnique); + } + + @Test + void testUniqueIds() { + HashSet allIDs = new HashSet<>(); + boolean allIdsUnique = true; + for (int i=0; i < 10000; i+=1) { + val txnId = distributedIdGenerator.generate("P").getId(); + if (allIDs.contains(txnId)) { + log.warn(txnId); + log.warn(String.valueOf(allIDs)); + allIdsUnique = false; + } else { + allIDs.add(txnId); + } + } + Assertions.assertTrue(allIdsUnique); + } + + @Test + void testGenerateOriginal() { + distributedIdGenerator = new DistributedIdGenerator(23, partitionCount, partitionResolverSupplier, IdFormatters.original()); + String id = distributedIdGenerator.generate("TEST").getId(); + Assertions.assertEquals(26, id.length()); + } + + @Test + void testGenerateBase36() { + distributedIdGenerator = new DistributedIdGenerator(23, partitionCount, (txnId) -> new BigInteger(txnId.substring(txnId.length()-6), 36).abs().intValue() % partitionCount, IdFormatters.base36()); + String id = distributedIdGenerator.generate("TEST").getId(); + Assertions.assertEquals(18, id.length()); + } + + @Test + void testConstraintFailure() { + Assertions.assertFalse(distributedIdGenerator.generateWithConstraints( + "TST", + Collections.singletonList((id -> false)), + true).isPresent()); + } + + @Test + void testParseFailure() { + //Null or Empty String + Assertions.assertFalse(distributedIdGenerator.parse(null).isPresent()); + Assertions.assertFalse(distributedIdGenerator.parse("").isPresent()); + + //Invalid length + Assertions.assertFalse(distributedIdGenerator.parse("TEST").isPresent()); + + //Invalid chars + Assertions.assertFalse(distributedIdGenerator.parse("XCL983dfb1ee0a847cd9e7321fcabc2f223").isPresent()); + Assertions.assertFalse(distributedIdGenerator.parse("XCL98-3df-b1e:e0a847cd9e7321fcabc2f223").isPresent()); + + //Invalid month + Assertions.assertFalse(distributedIdGenerator.parse("ABC2032250959030643972247").isPresent()); + //Invalid date + Assertions.assertFalse(distributedIdGenerator.parse("ABC2011450959030643972247").isPresent()); + //Invalid hour + Assertions.assertFalse(distributedIdGenerator.parse("ABC2011259659030643972247").isPresent()); + //Invalid minute + Assertions.assertFalse(distributedIdGenerator.parse("ABC2011250972030643972247").isPresent()); + //Invalid sec + Assertions.assertFalse(distributedIdGenerator.parse("ABC2011250959720643972247").isPresent()); + } + + @Test + void testParseSuccess() { + val idString = "ABC2011250959030643972247"; + val id = distributedIdGenerator.parse(idString).orElse(null); + Assertions.assertNotNull(id); + Assertions.assertEquals(idString, id.getId()); + Assertions.assertEquals(972247, id.getExponent()); + Assertions.assertEquals(643, id.getNode()); + Assertions.assertEquals(generateDate(2020, 11, 25, 9, 59, 3, 0, ZoneId.systemDefault()), + id.getGeneratedDate()); + } + + @Test + void testParseSuccessAfterGeneration() { + val generatedId = distributedIdGenerator.generate("TEST123"); + val parsedId = distributedIdGenerator.parse(generatedId.getId()).orElse(null); + Assertions.assertNotNull(parsedId); + Assertions.assertEquals(parsedId.getId(), generatedId.getId()); + Assertions.assertEquals(parsedId.getExponent(), generatedId.getExponent()); + Assertions.assertEquals(parsedId.getNode(), generatedId.getNode()); + } + + @SuppressWarnings("SameParameterValue") + private Date generateDate(int year, int month, int day, int hour, int min, int sec, int ms, ZoneId zoneId) { + return Date.from( + Instant.from( + ZonedDateTime.of( + LocalDateTime.of( + year, month, day, hour, min, sec, Math.multiplyExact(ms, 1000000) + ), + zoneId + ) + ) + ); + } +} \ No newline at end of file diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/TestUtil.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/TestUtil.java new file mode 100644 index 00000000..26929fff --- /dev/null +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/TestUtil.java @@ -0,0 +1,64 @@ +package io.appform.ranger.discovery.bundle.id; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.dropwizard.logback.shaded.guava.base.Stopwatch; +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +@Slf4j +@UtilityClass +public class TestUtil { + private static final String OUTPUT_PATH = "perf/results/%s.json"; + + public double runMTTest(int numThreads, int iterationCount, final Consumer supplier, String outputFileName) throws IOException { + final ExecutorService executorService = Executors.newFixedThreadPool(numThreads); + final List> futures = IntStream.range(0, numThreads) + .mapToObj(i -> executorService.submit(() -> { + final Stopwatch stopwatch = Stopwatch.createStarted(); + IntStream.range(0, iterationCount).forEach(supplier::accept); + return stopwatch.elapsed(TimeUnit.MILLISECONDS); + })) + .collect(Collectors.toList()); + final long total = futures.stream() + .mapToLong(f -> { + try { + return f.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return 0; + } catch (ExecutionException e) { + throw new IllegalStateException(e); + } + }) + .sum(); + writeToFile(numThreads, iterationCount, total, outputFileName); + log.warn("Finished Execution for {} iterations in avg time: {}", iterationCount, ((double) total) / numThreads); + return total; + } + + public void writeToFile(int numThreads, int iterationCount, long totalMillis, String outputFileName) throws IOException { + val mapper = new ObjectMapper(); + val outputFilePath = Paths.get(String.format(OUTPUT_PATH, outputFileName)); + val outputNode = mapper.createObjectNode(); + outputNode.put("name", outputFileName); + outputNode.put("iterations", iterationCount); + outputNode.put("threads", numThreads); + outputNode.put("totalMillis", totalMillis); + outputNode.put("avgTime", ((double) totalMillis) / numThreads); + Files.write(outputFilePath, mapper.writerWithDefaultPrettyPrinter().writeValueAsBytes(outputNode)); + } +} diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java new file mode 100644 index 00000000..947ad33e --- /dev/null +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java @@ -0,0 +1,51 @@ +package io.appform.ranger.discovery.bundle.id; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.infra.Blackhole; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +/** + * Test performance between different constructs + */ +@Slf4j +public class WeightedIdGeneratorPerfTest extends BenchmarkTest { + + @State(Scope.Benchmark) + public static class BenchmarkState { + private WeightedIdGenerator weightedIdGenerator; + final Function partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length()-6)) % 1024; + + + @Setup(Level.Trial) + public void setUp() throws IOException { + List partitionConfigList = new ArrayList<>(); + partitionConfigList.add(WeightedPartition.builder() + .partitionRange(PartitionRange.builder().start(0).end(511).build()) + .weight(400).build()); + partitionConfigList.add(WeightedPartition.builder() + .partitionRange(PartitionRange.builder().start(512).end(1023).build()) + .weight(600).build()); + val weightedIdConfig = WeightedIdConfig.builder() + .partitions(partitionConfigList) + .build(); + weightedIdGenerator = new WeightedIdGenerator(23, 1024, partitionResolverSupplier, weightedIdConfig); + } + } + + @SneakyThrows + @Benchmark + public void testGenerate(Blackhole blackhole, BenchmarkState state) { + state.weightedIdGenerator.generate("P"); + } +} \ No newline at end of file diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java new file mode 100644 index 00000000..0aeadb2f --- /dev/null +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java @@ -0,0 +1,196 @@ +package io.appform.ranger.discovery.bundle.id; + +import io.appform.ranger.discovery.bundle.id.constraints.KeyValidationConstraint; +import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.math.BigInteger; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +/** + * Test for {@link WeightedIdGenerator} + */ +@Slf4j +@SuppressWarnings({"unused", "FieldMayBeFinal"}) +class WeightedIdGeneratorTest { + final int partitionCount = 1024; + final Function partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length()-6)) % partitionCount; + private WeightedIdGenerator weightedIdGenerator; + private WeightedIdConfig weightedIdConfig; + + @BeforeEach + void setup() { + List partitionConfigList = new ArrayList<>(); + partitionConfigList.add(WeightedPartition.builder() + .partitionRange(PartitionRange.builder().start(0).end(511).build()) + .weight(400).build()); + partitionConfigList.add(WeightedPartition.builder() + .partitionRange(PartitionRange.builder().start(512).end(1023).build()) + .weight(600).build()); + weightedIdConfig = WeightedIdConfig.builder() + .partitions(partitionConfigList) + .build(); + weightedIdGenerator = new WeightedIdGenerator(7, partitionCount, partitionResolverSupplier, weightedIdConfig); + } + + @Test + void testGenerateWithBenchmark() throws IOException { + val totalTime = TestUtil.runMTTest(5, 10000000, (k) -> weightedIdGenerator.generate("P"), this.getClass().getName() + ".testGenerateWithBenchmark"); +// testUniqueIds(weightedIdGenerator.getDataStore()); + } + + @Test + void testGenerateWithConstraints() throws IOException { + KeyValidationConstraint partitionConstraint = (k) -> k % 10 == 0; + weightedIdGenerator.registerGlobalConstraints(partitionConstraint); + val totalTime = TestUtil.runMTTest(5, 1000000, (k) -> weightedIdGenerator.generateWithConstraints("P", (String) null, false), this.getClass().getName() + ".testGenerateWithConstraints"); + testUniqueIds(weightedIdGenerator.getDataStore()); + + for (Map.Entry> entry : weightedIdGenerator.getDataStore().entrySet()) { + val prefix = entry.getKey(); + val prefixIds = entry.getValue(); + HashSet uniqueIds = new HashSet<>(); + for (Map.Entry prefixEntry : prefixIds.entrySet()) { + val key = prefixEntry.getKey(); + val partitionIdTracker = prefixEntry.getValue(); + for (int idx=0; idx < partitionIdTracker.getPartitionSize(); idx += 1) { + if (!partitionConstraint.isValid(idx)) { + Assertions.assertEquals(0, partitionIdTracker.getIdList()[idx].getPointer().get()); + } + } + } + } + } + + void testUniqueIds(Map> dataStore) { + boolean allIdsUnique = true; + for (Map.Entry> entry : dataStore.entrySet()) { + val prefix = entry.getKey(); + val prefixIds = entry.getValue(); + for (Map.Entry prefixEntry : prefixIds.entrySet()) { + val key = prefixEntry.getKey(); + val partitionIdTracker = prefixEntry.getValue(); + HashSet uniqueIds = new HashSet<>(); + for (val idPool: partitionIdTracker.getIdList()) { + boolean allIdsUniqueInList = true; + HashSet uniqueIdsInList = new HashSet<>(); + for (val id: idPool.getIds()) { + if (uniqueIdsInList.contains(id)) { + allIdsUniqueInList = false; + allIdsUnique = false; + } else { + uniqueIdsInList.add(id); + } + + if (uniqueIds.contains(id)) { + allIdsUnique = false; + } else { + uniqueIds.add(id); + } + } + Assertions.assertTrue(allIdsUniqueInList); + } + } + } + Assertions.assertTrue(allIdsUnique); + } + + @Test + void testGenerateOriginal() { + weightedIdGenerator = new WeightedIdGenerator(23, partitionCount, partitionResolverSupplier, IdFormatters.original(), weightedIdConfig); + String id = weightedIdGenerator.generate("TEST").getId(); + Assertions.assertEquals(26, id.length()); + } + + @Test + void testGenerateBase36() { + weightedIdGenerator = new WeightedIdGenerator(23, partitionCount, (txnId) -> new BigInteger(txnId.substring(txnId.length()-6), 36).abs().intValue() % partitionCount, IdFormatters.base36(), weightedIdConfig); + String id = weightedIdGenerator.generate("TEST").getId(); + Assertions.assertEquals(18, id.length()); + } + + @Test + void testConstraintFailure() { + Assertions.assertFalse(weightedIdGenerator.generateWithConstraints( + "TST", + Collections.singletonList((id -> false)), + true).isPresent()); + } + + @Test + void testParseFailure() { + //Null or Empty String + Assertions.assertFalse(weightedIdGenerator.parse(null).isPresent()); + Assertions.assertFalse(weightedIdGenerator.parse("").isPresent()); + + //Invalid length + Assertions.assertFalse(weightedIdGenerator.parse("TEST").isPresent()); + + //Invalid chars + Assertions.assertFalse(weightedIdGenerator.parse("XCL983dfb1ee0a847cd9e7321fcabc2f223").isPresent()); + Assertions.assertFalse(weightedIdGenerator.parse("XCL98-3df-b1e:e0a847cd9e7321fcabc2f223").isPresent()); + + //Invalid month + Assertions.assertFalse(weightedIdGenerator.parse("ABC2032250959030643972247").isPresent()); + //Invalid date + Assertions.assertFalse(weightedIdGenerator.parse("ABC2011450959030643972247").isPresent()); + //Invalid hour + Assertions.assertFalse(weightedIdGenerator.parse("ABC2011259659030643972247").isPresent()); + //Invalid minute + Assertions.assertFalse(weightedIdGenerator.parse("ABC2011250972030643972247").isPresent()); + //Invalid sec + Assertions.assertFalse(weightedIdGenerator.parse("ABC2011250959720643972247").isPresent()); + } + + @Test + void testParseSuccess() { + val idString = "ABC2011250959030643972247"; + val id = weightedIdGenerator.parse(idString).orElse(null); + Assertions.assertNotNull(id); + Assertions.assertEquals(idString, id.getId()); + Assertions.assertEquals(972247, id.getExponent()); + Assertions.assertEquals(643, id.getNode()); + Assertions.assertEquals(generateDate(2020, 11, 25, 9, 59, 3, 0, ZoneId.systemDefault()), + id.getGeneratedDate()); + } + + @Test + void testParseSuccessAfterGeneration() { + val generatedId = weightedIdGenerator.generate("TEST123"); + val parsedId = weightedIdGenerator.parse(generatedId.getId()).orElse(null); + Assertions.assertNotNull(parsedId); + Assertions.assertEquals(parsedId.getId(), generatedId.getId()); + Assertions.assertEquals(parsedId.getExponent(), generatedId.getExponent()); + Assertions.assertEquals(parsedId.getNode(), generatedId.getNode()); + } + + + @SuppressWarnings("SameParameterValue") + private Date generateDate(int year, int month, int day, int hour, int min, int sec, int ms, ZoneId zoneId) { + return Date.from( + Instant.from( + ZonedDateTime.of( + LocalDateTime.of( + year, month, day, hour, min, sec, Math.multiplyExact(ms, 1000000) + ), + zoneId + ) + ) + ); + } +} \ No newline at end of file From f2467169fadcf50a193b466aa1eede3f8e2daab3 Mon Sep 17 00:00:00 2001 From: Satyam Saxena Date: Mon, 6 May 2024 17:46:03 +0530 Subject: [PATCH 02/12] Update partition access --- .../ranger/discovery/bundle/id/Constants.java | 1 + .../bundle/id/DistributedIdGenerator.java | 42 +++++++++++-------- .../bundle/id/PartitionIdTracker.java | 7 ++++ 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java index f191b90e..78e11f76 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java @@ -25,4 +25,5 @@ public class Constants { public static final int MAX_ID_PER_MS = 1000; public static final int MAX_NUM_NODES = 10000; + public static final int ID_DELETION_DELAY_IN_SECONDS = 60; } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/DistributedIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/DistributedIdGenerator.java index bed89b14..2c543b99 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/DistributedIdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/DistributedIdGenerator.java @@ -74,17 +74,23 @@ public class DistributedIdGenerator { // } // } - public DistributedIdGenerator(int node, int partitionSize, final Function partitionResolverSupplier) { + public DistributedIdGenerator(final int node, + final int partitionSize, + final Function partitionResolverSupplier) { nodeId = node; partitionCount = partitionSize; partitionResolver = partitionResolverSupplier; idFormatter = IdFormatters.distributed(); val executorService = Executors.newScheduledThreadPool(1); - executorService.scheduleWithFixedDelay(this::deleteExpiredKeys, 60, 60, TimeUnit.SECONDS); + executorService.scheduleWithFixedDelay( + this::deleteExpiredKeys, + Constants.ID_DELETION_DELAY_IN_SECONDS, + Constants.ID_DELETION_DELAY_IN_SECONDS, + TimeUnit.SECONDS); } - public DistributedIdGenerator(int node, - int partitionSize, + public DistributedIdGenerator(final int node, + final int partitionSize, final Function partitionResolverSupplier, final IdFormatter idFormatterInstance) { nodeId = node; @@ -93,24 +99,24 @@ public DistributedIdGenerator(int node, idFormatter = idFormatterInstance; } - public synchronized void registerGlobalConstraints(KeyValidationConstraint... constraints) { + public synchronized void registerGlobalConstraints(final KeyValidationConstraint... constraints) { registerGlobalConstraints(ImmutableList.copyOf(constraints)); } - public synchronized void registerGlobalConstraints(List constraints) { + public synchronized void registerGlobalConstraints(final List constraints) { Preconditions.checkArgument(null != constraints && !constraints.isEmpty()); GLOBAL_CONSTRAINTS.addAll(constraints); } public synchronized void registerDomainSpecificConstraints( - String domain, - KeyValidationConstraint... validationConstraints) { + final String domain, + final KeyValidationConstraint... validationConstraints) { registerDomainSpecificConstraints(domain, ImmutableList.copyOf(validationConstraints)); } public synchronized void registerDomainSpecificConstraints( - String domain, - List validationConstraints) { + final String domain, + final List validationConstraints) { Preconditions.checkArgument(null != validationConstraints && !validationConstraints.isEmpty()); DOMAIN_SPECIFIC_CONSTRAINTS.computeIfAbsent(domain, key -> new ArrayList<>()) .addAll(validationConstraints); @@ -122,12 +128,12 @@ public synchronized void registerDomainSpecificConstraints( * @param prefix String prefix for ID to be generated * @return Generated Id */ - public Id generate(String prefix) { + public Id generate(final String prefix) { val targetPartitionId = getTargetPartitionId(); return generateForPartition(prefix, targetPartitionId); } - public Id generateForPartition(String prefix, int targetPartitionId) { + public Id generateForPartition(final String prefix, final int targetPartitionId) { val prefixIdMap = dataStore.computeIfAbsent(prefix, k -> new ConcurrentHashMap<>()); val currentTimestamp = new DateTime(); val timeKey = currentTimestamp.getMillis() / 1000; @@ -146,17 +152,17 @@ public Id generateForPartition(String prefix, int targetPartitionId) { } private int generateForAllPartitions(final PartitionIdTracker partitionIdTracker, - final String prefix, - final DateTime timestamp, - final int targetPartitionId) { - val idPool = partitionIdTracker.getIdList()[targetPartitionId]; + final String prefix, + final DateTime timestamp, + final int targetPartitionId) { + val idPool = partitionIdTracker.getPartition(targetPartitionId); int idIdx = idPool.getPointer().getAndIncrement(); // ToDo: Add Retry Limit while (idPool.getIds().size() <= idIdx) { val counterValue = partitionIdTracker.getCounter().getAndIncrement(); val txnId = String.format("%s%s", prefix, idFormatter.format(timestamp, nodeId, counterValue)); val mappedPartitionId = partitionResolver.apply(txnId); - partitionIdTracker.getIdList()[mappedPartitionId].getIds().add(counterValue); + partitionIdTracker.getPartition(mappedPartitionId).getIds().add(counterValue); } return idPool.getId(idIdx); } @@ -229,7 +235,7 @@ private int getTargetPartitionId() { return SECURE_RANDOM.nextInt(partitionCount); } - private Optional getTargetPartitionId(final List inConstraints, boolean skipGlobal) { + private Optional getTargetPartitionId(final List inConstraints, final boolean skipGlobal) { // ToDo: Check if we need Collision Checker here return Optional.ofNullable( RETRIER.get(() -> SECURE_RANDOM.nextInt(partitionCount))) diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java index 38fd4fc0..a4bff737 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java @@ -17,4 +17,11 @@ protected PartitionIdTracker(int partitionSize) { idList[i] = new IdPool(); } } + + public IdPool getPartition(int partitionId) { + if (partitionId <= idList.length) { + return idList[partitionId]; + } + throw new IndexOutOfBoundsException("Invalid partitionId " + partitionId + " for IdPool of size" + idList.length); + } } From 815462140faba386b336cc87773291dbc40d369b Mon Sep 17 00:00:00 2001 From: Satyam Saxena Date: Tue, 7 May 2024 14:35:47 +0530 Subject: [PATCH 03/12] Remove nodeId from Constructor --- ...e.id.IdGeneratorPerfTest.testGenerate.json | 2 +- ...dGeneratorPerfTest.testGenerateBase36.json | 2 +- .../bundle/id/DistributedIdGenerator.java | 16 ++++------ .../bundle/id/WeightedIdGenerator.java | 18 +++++------ .../id/DistributedIdGeneratorPerfTest.java | 2 +- .../bundle/id/DistributedIdGeneratorTest.java | 25 +++------------ .../discovery/bundle/id/IdGeneratorTest.java | 22 ++----------- .../ranger/discovery/bundle/id/TestUtil.java | 18 +++++++++++ .../id/WeightedIdGeneratorPerfTest.java | 2 +- .../bundle/id/WeightedIdGeneratorTest.java | 32 ++++--------------- 10 files changed, 50 insertions(+), 89 deletions(-) diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json index f19b937b..e2d661ae 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 644166.1778513143 + "mean_ops" : 800941.387237486 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json index 272db000..d8e026c4 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 502644.4941310657 + "mean_ops" : 643473.2832640476 } \ No newline at end of file diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/DistributedIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/DistributedIdGenerator.java index 2c543b99..54b37371 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/DistributedIdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/DistributedIdGenerator.java @@ -48,7 +48,7 @@ public class DistributedIdGenerator { private static final Pattern PATTERN = Pattern.compile("(.*)([0-9]{12})([0-9]{4})([0-9]{6})"); private static final List GLOBAL_CONSTRAINTS = new ArrayList<>(); private static final Map> DOMAIN_SPECIFIC_CONSTRAINTS = new HashMap<>(); - protected int nodeId; + protected static final int NODE_ID = IdGenerator.getNodeId(); @Getter private final Map> dataStore = new ConcurrentHashMap<>(); protected final IdFormatter idFormatter; @@ -74,10 +74,8 @@ public class DistributedIdGenerator { // } // } - public DistributedIdGenerator(final int node, - final int partitionSize, + public DistributedIdGenerator(final int partitionSize, final Function partitionResolverSupplier) { - nodeId = node; partitionCount = partitionSize; partitionResolver = partitionResolverSupplier; idFormatter = IdFormatters.distributed(); @@ -89,11 +87,9 @@ public DistributedIdGenerator(final int node, TimeUnit.SECONDS); } - public DistributedIdGenerator(final int node, - final int partitionSize, + public DistributedIdGenerator(final int partitionSize, final Function partitionResolverSupplier, final IdFormatter idFormatterInstance) { - nodeId = node; partitionCount = partitionSize; partitionResolver = partitionResolverSupplier; idFormatter = idFormatterInstance; @@ -142,12 +138,12 @@ public Id generateForPartition(final String prefix, final int targetPartitionId) prefix, currentTimestamp, targetPartitionId); - val id = String.format("%s%s", prefix, idFormatter.format(currentTimestamp, nodeId, idCounter)); + val id = String.format("%s%s", prefix, idFormatter.format(currentTimestamp, NODE_ID, idCounter)); return Id.builder() .id(id) .exponent(idCounter) .generatedDate(currentTimestamp.toDate()) - .node(nodeId) + .node(NODE_ID) .build(); } @@ -160,7 +156,7 @@ private int generateForAllPartitions(final PartitionIdTracker partitionIdTracker // ToDo: Add Retry Limit while (idPool.getIds().size() <= idIdx) { val counterValue = partitionIdTracker.getCounter().getAndIncrement(); - val txnId = String.format("%s%s", prefix, idFormatter.format(timestamp, nodeId, counterValue)); + val txnId = String.format("%s%s", prefix, idFormatter.format(timestamp, NODE_ID, counterValue)); val mappedPartitionId = partitionResolver.apply(txnId); partitionIdTracker.getPartition(mappedPartitionId).getIds().add(counterValue); } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java index bd5f795e..351cec61 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java @@ -21,24 +21,22 @@ public class WeightedIdGenerator extends DistributedIdGenerator { private int maxShardWeight; private final RangeMap partitionRangeMap; - public WeightedIdGenerator(int node, - int partitionSize, + public WeightedIdGenerator(final int partitionSize, final Function partitionResolverSupplier, final WeightedIdConfig weightedIdConfig) { - super(node, partitionSize, partitionResolverSupplier); + super(partitionSize, partitionResolverSupplier); partitionRangeMap = createWeightRangeMap(weightedIdConfig); } - public WeightedIdGenerator(int node, - int partitionSize, + public WeightedIdGenerator(final int partitionSize, final Function partitionResolverSupplier, - final IdFormatter idFormatterInstance, - final WeightedIdConfig weightedIdConfig) { - super(node, partitionSize, partitionResolverSupplier, idFormatterInstance); + final WeightedIdConfig weightedIdConfig, + final IdFormatter idFormatterInstance) { + super(partitionSize, partitionResolverSupplier, idFormatterInstance); partitionRangeMap = createWeightRangeMap(weightedIdConfig); } - private RangeMap createWeightRangeMap(WeightedIdConfig weightedIdConfig) { + private RangeMap createWeightRangeMap(final WeightedIdConfig weightedIdConfig) { RangeMap shardGroups = TreeRangeMap.create(); int start = 0; int end = -1; @@ -57,7 +55,7 @@ private int getTargetPartitionId() { return SECURE_RANDOM.nextInt(partitionRange.getEnd() - partitionRange.getStart() + 1) + partitionRange.getStart(); } - private Optional getTargetPartitionId(final List inConstraints, boolean skipGlobal) { + private Optional getTargetPartitionId(final List inConstraints, final boolean skipGlobal) { return Optional.ofNullable( RETRIER.get(this::getTargetPartitionId)) .filter(key -> validateId(inConstraints, key, skipGlobal)); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorPerfTest.java index a27a5511..7317903b 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorPerfTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorPerfTest.java @@ -25,7 +25,7 @@ public static class BenchmarkState { @Setup(Level.Trial) public void setUp() throws IOException { - distributedIdGenerator = new DistributedIdGenerator(23, 1024, partitionResolverSupplier); + distributedIdGenerator = new DistributedIdGenerator(1024, partitionResolverSupplier); } } diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorTest.java index a0f744d0..394a8179 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorTest.java @@ -10,12 +10,8 @@ import java.io.IOException; import java.math.BigInteger; -import java.time.Instant; -import java.time.LocalDateTime; import java.time.ZoneId; -import java.time.ZonedDateTime; import java.util.Collections; -import java.util.Date; import java.util.HashSet; import java.util.Map; import java.util.function.Function; @@ -32,7 +28,7 @@ class DistributedIdGeneratorTest { @BeforeEach void setup() { - distributedIdGenerator = new DistributedIdGenerator(23, partitionCount, partitionResolverSupplier); + distributedIdGenerator = new DistributedIdGenerator(partitionCount, partitionResolverSupplier); } @Test @@ -116,14 +112,14 @@ void testUniqueIds() { @Test void testGenerateOriginal() { - distributedIdGenerator = new DistributedIdGenerator(23, partitionCount, partitionResolverSupplier, IdFormatters.original()); + distributedIdGenerator = new DistributedIdGenerator(partitionCount, partitionResolverSupplier, IdFormatters.original()); String id = distributedIdGenerator.generate("TEST").getId(); Assertions.assertEquals(26, id.length()); } @Test void testGenerateBase36() { - distributedIdGenerator = new DistributedIdGenerator(23, partitionCount, (txnId) -> new BigInteger(txnId.substring(txnId.length()-6), 36).abs().intValue() % partitionCount, IdFormatters.base36()); + distributedIdGenerator = new DistributedIdGenerator(partitionCount, (txnId) -> new BigInteger(txnId.substring(txnId.length()-6), 36).abs().intValue() % partitionCount, IdFormatters.base36()); String id = distributedIdGenerator.generate("TEST").getId(); Assertions.assertEquals(18, id.length()); } @@ -169,7 +165,7 @@ void testParseSuccess() { Assertions.assertEquals(idString, id.getId()); Assertions.assertEquals(972247, id.getExponent()); Assertions.assertEquals(643, id.getNode()); - Assertions.assertEquals(generateDate(2020, 11, 25, 9, 59, 3, 0, ZoneId.systemDefault()), + Assertions.assertEquals(TestUtil.generateDate(2020, 11, 25, 9, 59, 3, 0, ZoneId.systemDefault()), id.getGeneratedDate()); } @@ -183,17 +179,4 @@ void testParseSuccessAfterGeneration() { Assertions.assertEquals(parsedId.getNode(), generatedId.getNode()); } - @SuppressWarnings("SameParameterValue") - private Date generateDate(int year, int month, int day, int hour, int min, int sec, int ms, ZoneId zoneId) { - return Date.from( - Instant.from( - ZonedDateTime.of( - LocalDateTime.of( - year, month, day, hour, min, sec, Math.multiplyExact(ms, 1000000) - ), - zoneId - ) - ) - ); - } } \ No newline at end of file diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdGeneratorTest.java index 0089eaac..b83e4b2a 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdGeneratorTest.java @@ -29,9 +29,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.time.*; +import java.time.Duration; +import java.time.ZoneId; import java.util.Collections; -import java.util.Date; import java.util.Optional; import java.util.concurrent.Callable; import java.util.concurrent.Executors; @@ -176,7 +176,7 @@ void testParseSuccess() { Assertions.assertEquals(idString, id.getId()); Assertions.assertEquals(247, id.getExponent()); Assertions.assertEquals(3972, id.getNode()); - Assertions.assertEquals(generateDate(2020, 11, 25, 9, 59, 3, 64, ZoneId.systemDefault()), + Assertions.assertEquals(TestUtil.generateDate(2020, 11, 25, 9, 59, 3, 64, ZoneId.systemDefault()), id.getGeneratedDate()); } @@ -191,20 +191,4 @@ void testParseSuccessAfterGeneration() { Assertions.assertEquals(parsedId.getGeneratedDate(), generatedId.getGeneratedDate()); } - - @SuppressWarnings("SameParameterValue") - private Date generateDate(int year, int month, int day, int hour, int min, int sec, int ms, ZoneId zoneId) { - return Date.from( - Instant.from( - ZonedDateTime.of( - LocalDateTime.of( - year, month, day, hour, min, sec, Math.multiplyExact(ms, 1000000) - ), - zoneId - ) - ) - ); - } - - } \ No newline at end of file diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/TestUtil.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/TestUtil.java index 26929fff..9de1e39f 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/TestUtil.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/TestUtil.java @@ -9,6 +9,11 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Date; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -61,4 +66,17 @@ public void writeToFile(int numThreads, int iterationCount, long totalMillis, St outputNode.put("avgTime", ((double) totalMillis) / numThreads); Files.write(outputFilePath, mapper.writerWithDefaultPrettyPrinter().writeValueAsBytes(outputNode)); } + + public Date generateDate(int year, int month, int day, int hour, int min, int sec, int ms, ZoneId zoneId) { + return Date.from( + Instant.from( + ZonedDateTime.of( + LocalDateTime.of( + year, month, day, hour, min, sec, Math.multiplyExact(ms, 1000000) + ), + zoneId + ) + ) + ); + } } diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java index 947ad33e..9b8dbd24 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java @@ -39,7 +39,7 @@ public void setUp() throws IOException { val weightedIdConfig = WeightedIdConfig.builder() .partitions(partitionConfigList) .build(); - weightedIdGenerator = new WeightedIdGenerator(23, 1024, partitionResolverSupplier, weightedIdConfig); + weightedIdGenerator = new WeightedIdGenerator(1024, partitionResolverSupplier, weightedIdConfig); } } diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java index 0aeadb2f..9f915bbc 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java @@ -10,13 +10,9 @@ import java.io.IOException; import java.math.BigInteger; -import java.time.Instant; -import java.time.LocalDateTime; import java.time.ZoneId; -import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Collections; -import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -45,20 +41,20 @@ void setup() { weightedIdConfig = WeightedIdConfig.builder() .partitions(partitionConfigList) .build(); - weightedIdGenerator = new WeightedIdGenerator(7, partitionCount, partitionResolverSupplier, weightedIdConfig); + weightedIdGenerator = new WeightedIdGenerator(partitionCount, partitionResolverSupplier, weightedIdConfig); } @Test void testGenerateWithBenchmark() throws IOException { - val totalTime = TestUtil.runMTTest(5, 10000000, (k) -> weightedIdGenerator.generate("P"), this.getClass().getName() + ".testGenerateWithBenchmark"); -// testUniqueIds(weightedIdGenerator.getDataStore()); + val totalTime = TestUtil.runMTTest(5, 100000, (k) -> weightedIdGenerator.generate("P"), this.getClass().getName() + ".testGenerateWithBenchmark"); + testUniqueIds(weightedIdGenerator.getDataStore()); } @Test void testGenerateWithConstraints() throws IOException { KeyValidationConstraint partitionConstraint = (k) -> k % 10 == 0; weightedIdGenerator.registerGlobalConstraints(partitionConstraint); - val totalTime = TestUtil.runMTTest(5, 1000000, (k) -> weightedIdGenerator.generateWithConstraints("P", (String) null, false), this.getClass().getName() + ".testGenerateWithConstraints"); + val totalTime = TestUtil.runMTTest(5, 100000, (k) -> weightedIdGenerator.generateWithConstraints("P", (String) null, false), this.getClass().getName() + ".testGenerateWithConstraints"); testUniqueIds(weightedIdGenerator.getDataStore()); for (Map.Entry> entry : weightedIdGenerator.getDataStore().entrySet()) { @@ -112,14 +108,14 @@ void testUniqueIds(Map> dataStore) { @Test void testGenerateOriginal() { - weightedIdGenerator = new WeightedIdGenerator(23, partitionCount, partitionResolverSupplier, IdFormatters.original(), weightedIdConfig); + weightedIdGenerator = new WeightedIdGenerator(partitionCount, partitionResolverSupplier, weightedIdConfig, IdFormatters.original()); String id = weightedIdGenerator.generate("TEST").getId(); Assertions.assertEquals(26, id.length()); } @Test void testGenerateBase36() { - weightedIdGenerator = new WeightedIdGenerator(23, partitionCount, (txnId) -> new BigInteger(txnId.substring(txnId.length()-6), 36).abs().intValue() % partitionCount, IdFormatters.base36(), weightedIdConfig); + weightedIdGenerator = new WeightedIdGenerator(partitionCount, (txnId) -> new BigInteger(txnId.substring(txnId.length()-6), 36).abs().intValue() % partitionCount, weightedIdConfig, IdFormatters.base36()); String id = weightedIdGenerator.generate("TEST").getId(); Assertions.assertEquals(18, id.length()); } @@ -165,7 +161,7 @@ void testParseSuccess() { Assertions.assertEquals(idString, id.getId()); Assertions.assertEquals(972247, id.getExponent()); Assertions.assertEquals(643, id.getNode()); - Assertions.assertEquals(generateDate(2020, 11, 25, 9, 59, 3, 0, ZoneId.systemDefault()), + Assertions.assertEquals(TestUtil.generateDate(2020, 11, 25, 9, 59, 3, 0, ZoneId.systemDefault()), id.getGeneratedDate()); } @@ -179,18 +175,4 @@ void testParseSuccessAfterGeneration() { Assertions.assertEquals(parsedId.getNode(), generatedId.getNode()); } - - @SuppressWarnings("SameParameterValue") - private Date generateDate(int year, int month, int day, int hour, int min, int sec, int ms, ZoneId zoneId) { - return Date.from( - Instant.from( - ZonedDateTime.of( - LocalDateTime.of( - year, month, day, hour, min, sec, Math.multiplyExact(ms, 1000000) - ), - zoneId - ) - ) - ); - } } \ No newline at end of file From b4222699102bce1de03cf1331f0ff9212e1eb554 Mon Sep 17 00:00:00 2001 From: Satyam Saxena Date: Tue, 7 May 2024 14:35:56 +0530 Subject: [PATCH 04/12] Add Perf tests results --- ...le.id.DistributedIdGeneratorPerfTest.testGenerate.json | 8 ++++++++ ...tributedIdGeneratorTest.testGenerateWithBenchmark.json | 7 +++++++ ...ibutedIdGeneratorTest.testGenerateWithConstraints.json | 7 +++++++ ...undle.id.WeightedIdGeneratorPerfTest.testGenerate.json | 8 ++++++++ ...WeightedIdGeneratorTest.testGenerateWithBenchmark.json | 7 +++++++ ...ightedIdGeneratorTest.testGenerateWithConstraints.json | 7 +++++++ 6 files changed, 44 insertions(+) create mode 100644 ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorPerfTest.testGenerate.json create mode 100644 ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithBenchmark.json create mode 100644 ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithConstraints.json create mode 100644 ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorPerfTest.testGenerate.json create mode 100644 ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json create mode 100644 ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorPerfTest.testGenerate.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorPerfTest.testGenerate.json new file mode 100644 index 00000000..fa3a95de --- /dev/null +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorPerfTest.testGenerate.json @@ -0,0 +1,8 @@ +{ + "name" : "io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorPerfTest.testGenerate", + "mode" : "Throughput", + "iterations" : 4, + "threads" : 1, + "forks" : 3, + "mean_ops" : 406378.3277020471 +} \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithBenchmark.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithBenchmark.json new file mode 100644 index 00000000..8ff858d7 --- /dev/null +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithBenchmark.json @@ -0,0 +1,7 @@ +{ + "name" : "io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithBenchmark", + "iterations" : 100000, + "threads" : 5, + "totalMillis" : 3823, + "avgTime" : 764.6 +} \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithConstraints.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithConstraints.json new file mode 100644 index 00000000..0d0c5e92 --- /dev/null +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithConstraints.json @@ -0,0 +1,7 @@ +{ + "name" : "io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithConstraints", + "iterations" : 100000, + "threads" : 5, + "totalMillis" : 5386, + "avgTime" : 1077.2 +} \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorPerfTest.testGenerate.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorPerfTest.testGenerate.json new file mode 100644 index 00000000..0651b230 --- /dev/null +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorPerfTest.testGenerate.json @@ -0,0 +1,8 @@ +{ + "name" : "io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorPerfTest.testGenerate", + "mode" : "Throughput", + "iterations" : 4, + "threads" : 1, + "forks" : 3, + "mean_ops" : 410789.0590215035 +} \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json new file mode 100644 index 00000000..755919de --- /dev/null +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json @@ -0,0 +1,7 @@ +{ + "name" : "io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark", + "iterations" : 100000, + "threads" : 5, + "totalMillis" : 3929, + "avgTime" : 785.8 +} \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json new file mode 100644 index 00000000..ddb7f17a --- /dev/null +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json @@ -0,0 +1,7 @@ +{ + "name" : "io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints", + "iterations" : 100000, + "threads" : 5, + "totalMillis" : 2867, + "avgTime" : 573.4 +} \ No newline at end of file From 66cf8b145b137b4ba8d4981812294f5b1c4e0a1f Mon Sep 17 00:00:00 2001 From: Satyam Saxena Date: Tue, 14 May 2024 18:35:47 +0530 Subject: [PATCH 05/12] Refactor Classes --- .../ranger/discovery/bundle/id/Constants.java | 1 + .../ranger/discovery/bundle/id/IdPool.java | 7 +- ...or.java => PartitionAwareIdGenerator.java} | 87 ++++++++++--------- .../bundle/id/PartitionIdTracker.java | 20 +++-- .../discovery/bundle/id/PartitionRange.java | 4 +- .../discovery/bundle/id/WeightedIdConfig.java | 2 +- .../bundle/id/WeightedIdGenerator.java | 17 ++-- ...ava => PartitionValidationConstraint.java} | 2 +- ...=> PartitionAwareIdGeneratorPerfTest.java} | 8 +- ...ava => PartitionAwareIdGeneratorTest.java} | 68 +++++++-------- .../bundle/id/WeightedIdGeneratorTest.java | 16 ++-- 11 files changed, 122 insertions(+), 110 deletions(-) rename ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/{DistributedIdGenerator.java => PartitionAwareIdGenerator.java} (80%) rename ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/constraints/{KeyValidationConstraint.java => PartitionValidationConstraint.java} (75%) rename ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/{DistributedIdGeneratorPerfTest.java => PartitionAwareIdGeneratorPerfTest.java} (75%) rename ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/{DistributedIdGeneratorTest.java => PartitionAwareIdGeneratorTest.java} (60%) diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java index 78e11f76..e70cb10a 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java @@ -26,4 +26,5 @@ public class Constants { public static final int MAX_ID_PER_MS = 1000; public static final int MAX_NUM_NODES = 10000; public static final int ID_DELETION_DELAY_IN_SECONDS = 60; + public static final int DELETION_THRESHOLD_IN_SECONDS = 60; } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdPool.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdPool.java index 155d362f..a8618262 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdPool.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdPool.java @@ -8,10 +8,13 @@ @Getter public class IdPool { - private final List ids = new CopyOnWriteArrayList<>(); + // List of IDs for the specific IdPool + private final List idList = new CopyOnWriteArrayList<>(); + + // Pointer to track the index of the next usable ID private final AtomicInteger pointer = new AtomicInteger(); public int getId(int index) { - return ids.get(index); + return idList.get(index); } } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/DistributedIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGenerator.java similarity index 80% rename from ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/DistributedIdGenerator.java rename to ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGenerator.java index 54b37371..d4ac8ddf 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/DistributedIdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGenerator.java @@ -5,7 +5,7 @@ import dev.failsafe.Failsafe; import dev.failsafe.FailsafeExecutor; import dev.failsafe.RetryPolicy; -import io.appform.ranger.discovery.bundle.id.constraints.KeyValidationConstraint; +import io.appform.ranger.discovery.bundle.id.constraints.PartitionValidationConstraint; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatter; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; import lombok.Getter; @@ -34,7 +34,7 @@ */ @SuppressWarnings("unused") @Slf4j -public class DistributedIdGenerator { +public class PartitionAwareIdGenerator { private static final int MINIMUM_ID_LENGTH = 22; protected static final SecureRandom SECURE_RANDOM = new SecureRandom(Long.toBinaryString(System.currentTimeMillis()).getBytes()); @@ -46,36 +46,37 @@ public class DistributedIdGenerator { .build(); protected final FailsafeExecutor RETRIER = Failsafe.with(Collections.singletonList(RETRY_POLICY)); private static final Pattern PATTERN = Pattern.compile("(.*)([0-9]{12})([0-9]{4})([0-9]{6})"); - private static final List GLOBAL_CONSTRAINTS = new ArrayList<>(); - private static final Map> DOMAIN_SPECIFIC_CONSTRAINTS = new HashMap<>(); + private static final List GLOBAL_CONSTRAINTS = new ArrayList<>(); + private static final Map> DOMAIN_SPECIFIC_CONSTRAINTS = new HashMap<>(); protected static final int NODE_ID = IdGenerator.getNodeId(); @Getter - private final Map> dataStore = new ConcurrentHashMap<>(); + private final Map> idStore = new ConcurrentHashMap<>(); protected final IdFormatter idFormatter; protected final Function partitionResolver; protected final int partitionCount; -// dataStore Structure -// { -// prefix: { -// timestamp: { -// partitions: [ -// { -// ids: [], -// pointer: -// }, -// { -// ids: [], -// pointer: -// } ... -// ], -// counter: -// } -// } -// } +/* dataStore Structure + { + prefix: { + timestamp: { + partitions: [ + { + ids: [], + pointer: + }, + { + ids: [], + pointer: + } ... + ], + counter: + } + } + } + */ - public DistributedIdGenerator(final int partitionSize, - final Function partitionResolverSupplier) { + public PartitionAwareIdGenerator(final int partitionSize, + final Function partitionResolverSupplier) { partitionCount = partitionSize; partitionResolver = partitionResolverSupplier; idFormatter = IdFormatters.distributed(); @@ -87,32 +88,32 @@ public DistributedIdGenerator(final int partitionSize, TimeUnit.SECONDS); } - public DistributedIdGenerator(final int partitionSize, - final Function partitionResolverSupplier, - final IdFormatter idFormatterInstance) { + public PartitionAwareIdGenerator(final int partitionSize, + final Function partitionResolverSupplier, + final IdFormatter idFormatterInstance) { partitionCount = partitionSize; partitionResolver = partitionResolverSupplier; idFormatter = idFormatterInstance; } - public synchronized void registerGlobalConstraints(final KeyValidationConstraint... constraints) { + public synchronized void registerGlobalConstraints(final PartitionValidationConstraint... constraints) { registerGlobalConstraints(ImmutableList.copyOf(constraints)); } - public synchronized void registerGlobalConstraints(final List constraints) { + public synchronized void registerGlobalConstraints(final List constraints) { Preconditions.checkArgument(null != constraints && !constraints.isEmpty()); GLOBAL_CONSTRAINTS.addAll(constraints); } public synchronized void registerDomainSpecificConstraints( final String domain, - final KeyValidationConstraint... validationConstraints) { + final PartitionValidationConstraint... validationConstraints) { registerDomainSpecificConstraints(domain, ImmutableList.copyOf(validationConstraints)); } public synchronized void registerDomainSpecificConstraints( final String domain, - final List validationConstraints) { + final List validationConstraints) { Preconditions.checkArgument(null != validationConstraints && !validationConstraints.isEmpty()); DOMAIN_SPECIFIC_CONSTRAINTS.computeIfAbsent(domain, key -> new ArrayList<>()) .addAll(validationConstraints); @@ -130,7 +131,7 @@ public Id generate(final String prefix) { } public Id generateForPartition(final String prefix, final int targetPartitionId) { - val prefixIdMap = dataStore.computeIfAbsent(prefix, k -> new ConcurrentHashMap<>()); + val prefixIdMap = idStore.computeIfAbsent(prefix, k -> new ConcurrentHashMap<>()); val currentTimestamp = new DateTime(); val timeKey = currentTimestamp.getMillis() / 1000; val idCounter = generateForAllPartitions( @@ -154,11 +155,11 @@ private int generateForAllPartitions(final PartitionIdTracker partitionIdTracker val idPool = partitionIdTracker.getPartition(targetPartitionId); int idIdx = idPool.getPointer().getAndIncrement(); // ToDo: Add Retry Limit - while (idPool.getIds().size() <= idIdx) { - val counterValue = partitionIdTracker.getCounter().getAndIncrement(); + while (idPool.getIdList().size() <= idIdx) { + val counterValue = partitionIdTracker.getNextIdCounter().getAndIncrement(); val txnId = String.format("%s%s", prefix, idFormatter.format(timestamp, NODE_ID, counterValue)); val mappedPartitionId = partitionResolver.apply(txnId); - partitionIdTracker.getPartition(mappedPartitionId).getIds().add(counterValue); + partitionIdTracker.getPartition(mappedPartitionId).getIdList().add(counterValue); } return idPool.getId(idIdx); } @@ -192,7 +193,7 @@ public Optional generateWithConstraints(final String prefix, final String do } public Optional generateWithConstraints(final String prefix, - final List inConstraints, + final List inConstraints, final boolean skipGlobal) { val targetPartitionId = getTargetPartitionId(inConstraints, skipGlobal); return targetPartitionId.map(id -> generateForPartition(prefix, id)); @@ -231,14 +232,13 @@ private int getTargetPartitionId() { return SECURE_RANDOM.nextInt(partitionCount); } - private Optional getTargetPartitionId(final List inConstraints, final boolean skipGlobal) { -// ToDo: Check if we need Collision Checker here + private Optional getTargetPartitionId(final List inConstraints, final boolean skipGlobal) { return Optional.ofNullable( RETRIER.get(() -> SECURE_RANDOM.nextInt(partitionCount))) .filter(key -> validateId(inConstraints, key, skipGlobal)); } - protected boolean validateId(final List inConstraints, + protected boolean validateId(final List inConstraints, final int partitionId, final boolean skipGlobal) { //First evaluate global constraints @@ -252,7 +252,7 @@ protected boolean validateId(final List inConstraints, if (null != failedGlobalConstraint) { return false; } - //Evaluate local + domain constraints + //Evaluate param constraints val failedLocalConstraint = null == inConstraints ? null @@ -265,6 +265,7 @@ protected boolean validateId(final List inConstraints, protected int readRetryCount() { try { +// Make it config val count = Integer.parseInt(System.getenv().getOrDefault("NUM_ID_GENERATION_RETRIES", "512")); if (count <= 0) { throw new IllegalArgumentException( @@ -279,8 +280,8 @@ protected int readRetryCount() { } private void deleteExpiredKeys() { - val timeThreshold = DateTime.now().getMillis() / 1000; - for (val entry : dataStore.entrySet()) { + val timeThreshold = DateTime.now().getMillis() / 1000 - Constants.DELETION_THRESHOLD_IN_SECONDS; + for (val entry : idStore.entrySet()) { entry.getValue().entrySet().removeIf(partitionIdTrackerEntry -> partitionIdTrackerEntry.getKey() < timeThreshold); } } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java index a4bff737..f4d69932 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java @@ -6,22 +6,28 @@ @Getter public class PartitionIdTracker { - private final IdPool[] idList; + // Array to store IdPools for each partition + private final IdPool[] idPoolList; + + // Size of each partition private final int partitionSize; - private final AtomicInteger counter = new AtomicInteger(); + + // Counter to keep track of the number of IDs created + private final AtomicInteger nextIdCounter = new AtomicInteger(); protected PartitionIdTracker(int partitionSize) { this.partitionSize = partitionSize; - idList = new IdPool[partitionSize]; + idPoolList = new IdPool[partitionSize]; for (int i=0; i= nextPartition.getPartitionRange().getStart()) { + if (currentPartition.getPartitionRange().getEnd() + 1 != nextPartition.getPartitionRange().getStart()) { return false; } } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java index 351cec61..3cbc779a 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java @@ -3,7 +3,7 @@ import com.google.common.collect.Range; import com.google.common.collect.RangeMap; import com.google.common.collect.TreeRangeMap; -import io.appform.ranger.discovery.bundle.id.constraints.KeyValidationConstraint; +import io.appform.ranger.discovery.bundle.id.constraints.PartitionValidationConstraint; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatter; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -17,7 +17,7 @@ * Weighted Id Generation */ @Slf4j -public class WeightedIdGenerator extends DistributedIdGenerator { +public class WeightedIdGenerator extends PartitionAwareIdGenerator { private int maxShardWeight; private final RangeMap partitionRangeMap; @@ -37,25 +37,26 @@ public WeightedIdGenerator(final int partitionSize, } private RangeMap createWeightRangeMap(final WeightedIdConfig weightedIdConfig) { - RangeMap shardGroups = TreeRangeMap.create(); - int start = 0; + RangeMap partitionGroups = TreeRangeMap.create(); int end = -1; for (val shard : weightedIdConfig.getPartitions()) { + int start = end+1; end += shard.getWeight(); - shardGroups.put(Range.closed(start, end), shard.getPartitionRange()); - start = end+1; + partitionGroups.put(Range.closed(start, end), shard.getPartitionRange()); } maxShardWeight = end; - return shardGroups; + return partitionGroups; } + @Override private int getTargetPartitionId() { val randomNum = SECURE_RANDOM.nextInt(maxShardWeight); val partitionRange = Objects.requireNonNull(partitionRangeMap.getEntry(randomNum)).getValue(); return SECURE_RANDOM.nextInt(partitionRange.getEnd() - partitionRange.getStart() + 1) + partitionRange.getStart(); } - private Optional getTargetPartitionId(final List inConstraints, final boolean skipGlobal) { + @Override + private Optional getTargetPartitionId(final List inConstraints, final boolean skipGlobal) { return Optional.ofNullable( RETRIER.get(this::getTargetPartitionId)) .filter(key -> validateId(inConstraints, key, skipGlobal)); diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/constraints/KeyValidationConstraint.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/constraints/PartitionValidationConstraint.java similarity index 75% rename from ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/constraints/KeyValidationConstraint.java rename to ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/constraints/PartitionValidationConstraint.java index 0d61c7f2..7d5710cb 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/constraints/KeyValidationConstraint.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/constraints/PartitionValidationConstraint.java @@ -1,7 +1,7 @@ package io.appform.ranger.discovery.bundle.id.constraints; -public interface KeyValidationConstraint { +public interface PartitionValidationConstraint { boolean isValid(int id); default boolean failFast() { diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java similarity index 75% rename from ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorPerfTest.java rename to ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java index 7317903b..aafcf691 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorPerfTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java @@ -16,22 +16,22 @@ * Test performance between different constructs */ @Slf4j -public class DistributedIdGeneratorPerfTest extends BenchmarkTest { +public class PartitionAwareIdGeneratorPerfTest extends BenchmarkTest { @State(Scope.Benchmark) public static class BenchmarkState { - private DistributedIdGenerator distributedIdGenerator; + private PartitionAwareIdGenerator partitionAwareIdGenerator; final Function partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length()-6)) % 1024; @Setup(Level.Trial) public void setUp() throws IOException { - distributedIdGenerator = new DistributedIdGenerator(1024, partitionResolverSupplier); + partitionAwareIdGenerator = new PartitionAwareIdGenerator(1024, partitionResolverSupplier); } } @SneakyThrows @Benchmark public void testGenerate(Blackhole blackhole, BenchmarkState state) { - state.distributedIdGenerator.generate("P"); + state.partitionAwareIdGenerator.generate("P"); } } \ No newline at end of file diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java similarity index 60% rename from ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorTest.java rename to ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java index 394a8179..7a39fa2a 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/DistributedIdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java @@ -1,6 +1,6 @@ package io.appform.ranger.discovery.bundle.id; -import io.appform.ranger.discovery.bundle.id.constraints.KeyValidationConstraint; +import io.appform.ranger.discovery.bundle.id.constraints.PartitionValidationConstraint; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -17,34 +17,34 @@ import java.util.function.Function; /** - * Test for {@link DistributedIdGenerator} + * Test for {@link PartitionAwareIdGenerator} */ @Slf4j @SuppressWarnings({"unused", "FieldMayBeFinal"}) -class DistributedIdGeneratorTest { +class PartitionAwareIdGeneratorTest { final int partitionCount = 1024; final Function partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length()-6)) % partitionCount; - private DistributedIdGenerator distributedIdGenerator; + private PartitionAwareIdGenerator partitionAwareIdGenerator; @BeforeEach void setup() { - distributedIdGenerator = new DistributedIdGenerator(partitionCount, partitionResolverSupplier); + partitionAwareIdGenerator = new PartitionAwareIdGenerator(partitionCount, partitionResolverSupplier); } @Test void testGenerateWithBenchmark() throws IOException { - val totalTime = TestUtil.runMTTest(5, 100000, (k) -> distributedIdGenerator.generate("P"), this.getClass().getName() + ".testGenerateWithBenchmark"); - testUniqueIdsInDataStore(distributedIdGenerator.getDataStore()); + val totalTime = TestUtil.runMTTest(5, 100000, (k) -> partitionAwareIdGenerator.generate("P"), this.getClass().getName() + ".testGenerateWithBenchmark"); + testUniqueIdsInDataStore(partitionAwareIdGenerator.getIdStore()); } @Test void testGenerateWithConstraints() throws IOException { - KeyValidationConstraint partitionConstraint = (k) -> k % 10 == 0; - distributedIdGenerator.registerGlobalConstraints(partitionConstraint); - val totalTime = TestUtil.runMTTest(5, 100000, (k) -> distributedIdGenerator.generateWithConstraints("P", (String) null, false), this.getClass().getName() + ".testGenerateWithConstraints"); - testUniqueIdsInDataStore(distributedIdGenerator.getDataStore()); + PartitionValidationConstraint partitionConstraint = (k) -> k % 10 == 0; + partitionAwareIdGenerator.registerGlobalConstraints(partitionConstraint); + val totalTime = TestUtil.runMTTest(5, 100000, (k) -> partitionAwareIdGenerator.generateWithConstraints("P", (String) null, false), this.getClass().getName() + ".testGenerateWithConstraints"); + testUniqueIdsInDataStore(partitionAwareIdGenerator.getIdStore()); - for (Map.Entry> entry : distributedIdGenerator.getDataStore().entrySet()) { + for (Map.Entry> entry : partitionAwareIdGenerator.getIdStore().entrySet()) { val prefix = entry.getKey(); val prefixIds = entry.getValue(); HashSet uniqueIds = new HashSet<>(); @@ -53,7 +53,7 @@ void testGenerateWithConstraints() throws IOException { val partitionIdTracker = prefixEntry.getValue(); for (int idx=0; idx < partitionIdTracker.getPartitionSize(); idx += 1) { if (!partitionConstraint.isValid(idx)) { - Assertions.assertEquals(0, partitionIdTracker.getIdList()[idx].getPointer().get()); + Assertions.assertEquals(0, partitionIdTracker.getIdPoolList()[idx].getPointer().get()); } } } @@ -69,10 +69,10 @@ void testUniqueIdsInDataStore(Map> dataSto val key = prefixEntry.getKey(); val partitionIdTracker = prefixEntry.getValue(); HashSet uniqueIds = new HashSet<>(); - for (val idPool: partitionIdTracker.getIdList()) { + for (val idPool: partitionIdTracker.getIdPoolList()) { boolean allIdsUniqueInList = true; HashSet uniqueIdsInList = new HashSet<>(); - for (val id: idPool.getIds()) { + for (val id: idPool.getIdList()) { if (uniqueIdsInList.contains(id)) { allIdsUniqueInList = false; allIdsUnique = false; @@ -98,7 +98,7 @@ void testUniqueIds() { HashSet allIDs = new HashSet<>(); boolean allIdsUnique = true; for (int i=0; i < 10000; i+=1) { - val txnId = distributedIdGenerator.generate("P").getId(); + val txnId = partitionAwareIdGenerator.generate("P").getId(); if (allIDs.contains(txnId)) { log.warn(txnId); log.warn(String.valueOf(allIDs)); @@ -112,21 +112,21 @@ void testUniqueIds() { @Test void testGenerateOriginal() { - distributedIdGenerator = new DistributedIdGenerator(partitionCount, partitionResolverSupplier, IdFormatters.original()); - String id = distributedIdGenerator.generate("TEST").getId(); + partitionAwareIdGenerator = new PartitionAwareIdGenerator(partitionCount, partitionResolverSupplier, IdFormatters.original()); + String id = partitionAwareIdGenerator.generate("TEST").getId(); Assertions.assertEquals(26, id.length()); } @Test void testGenerateBase36() { - distributedIdGenerator = new DistributedIdGenerator(partitionCount, (txnId) -> new BigInteger(txnId.substring(txnId.length()-6), 36).abs().intValue() % partitionCount, IdFormatters.base36()); - String id = distributedIdGenerator.generate("TEST").getId(); + partitionAwareIdGenerator = new PartitionAwareIdGenerator(partitionCount, (txnId) -> new BigInteger(txnId.substring(txnId.length()-6), 36).abs().intValue() % partitionCount, IdFormatters.base36()); + String id = partitionAwareIdGenerator.generate("TEST").getId(); Assertions.assertEquals(18, id.length()); } @Test void testConstraintFailure() { - Assertions.assertFalse(distributedIdGenerator.generateWithConstraints( + Assertions.assertFalse(partitionAwareIdGenerator.generateWithConstraints( "TST", Collections.singletonList((id -> false)), true).isPresent()); @@ -135,32 +135,32 @@ void testConstraintFailure() { @Test void testParseFailure() { //Null or Empty String - Assertions.assertFalse(distributedIdGenerator.parse(null).isPresent()); - Assertions.assertFalse(distributedIdGenerator.parse("").isPresent()); + Assertions.assertFalse(partitionAwareIdGenerator.parse(null).isPresent()); + Assertions.assertFalse(partitionAwareIdGenerator.parse("").isPresent()); //Invalid length - Assertions.assertFalse(distributedIdGenerator.parse("TEST").isPresent()); + Assertions.assertFalse(partitionAwareIdGenerator.parse("TEST").isPresent()); //Invalid chars - Assertions.assertFalse(distributedIdGenerator.parse("XCL983dfb1ee0a847cd9e7321fcabc2f223").isPresent()); - Assertions.assertFalse(distributedIdGenerator.parse("XCL98-3df-b1e:e0a847cd9e7321fcabc2f223").isPresent()); + Assertions.assertFalse(partitionAwareIdGenerator.parse("XCL983dfb1ee0a847cd9e7321fcabc2f223").isPresent()); + Assertions.assertFalse(partitionAwareIdGenerator.parse("XCL98-3df-b1e:e0a847cd9e7321fcabc2f223").isPresent()); //Invalid month - Assertions.assertFalse(distributedIdGenerator.parse("ABC2032250959030643972247").isPresent()); + Assertions.assertFalse(partitionAwareIdGenerator.parse("ABC2032250959030643972247").isPresent()); //Invalid date - Assertions.assertFalse(distributedIdGenerator.parse("ABC2011450959030643972247").isPresent()); + Assertions.assertFalse(partitionAwareIdGenerator.parse("ABC2011450959030643972247").isPresent()); //Invalid hour - Assertions.assertFalse(distributedIdGenerator.parse("ABC2011259659030643972247").isPresent()); + Assertions.assertFalse(partitionAwareIdGenerator.parse("ABC2011259659030643972247").isPresent()); //Invalid minute - Assertions.assertFalse(distributedIdGenerator.parse("ABC2011250972030643972247").isPresent()); + Assertions.assertFalse(partitionAwareIdGenerator.parse("ABC2011250972030643972247").isPresent()); //Invalid sec - Assertions.assertFalse(distributedIdGenerator.parse("ABC2011250959720643972247").isPresent()); + Assertions.assertFalse(partitionAwareIdGenerator.parse("ABC2011250959720643972247").isPresent()); } @Test void testParseSuccess() { val idString = "ABC2011250959030643972247"; - val id = distributedIdGenerator.parse(idString).orElse(null); + val id = partitionAwareIdGenerator.parse(idString).orElse(null); Assertions.assertNotNull(id); Assertions.assertEquals(idString, id.getId()); Assertions.assertEquals(972247, id.getExponent()); @@ -171,8 +171,8 @@ void testParseSuccess() { @Test void testParseSuccessAfterGeneration() { - val generatedId = distributedIdGenerator.generate("TEST123"); - val parsedId = distributedIdGenerator.parse(generatedId.getId()).orElse(null); + val generatedId = partitionAwareIdGenerator.generate("TEST123"); + val parsedId = partitionAwareIdGenerator.parse(generatedId.getId()).orElse(null); Assertions.assertNotNull(parsedId); Assertions.assertEquals(parsedId.getId(), generatedId.getId()); Assertions.assertEquals(parsedId.getExponent(), generatedId.getExponent()); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java index 9f915bbc..205ea483 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java @@ -1,6 +1,6 @@ package io.appform.ranger.discovery.bundle.id; -import io.appform.ranger.discovery.bundle.id.constraints.KeyValidationConstraint; +import io.appform.ranger.discovery.bundle.id.constraints.PartitionValidationConstraint; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -47,17 +47,17 @@ void setup() { @Test void testGenerateWithBenchmark() throws IOException { val totalTime = TestUtil.runMTTest(5, 100000, (k) -> weightedIdGenerator.generate("P"), this.getClass().getName() + ".testGenerateWithBenchmark"); - testUniqueIds(weightedIdGenerator.getDataStore()); + testUniqueIds(weightedIdGenerator.getIdStore()); } @Test void testGenerateWithConstraints() throws IOException { - KeyValidationConstraint partitionConstraint = (k) -> k % 10 == 0; + PartitionValidationConstraint partitionConstraint = (k) -> k % 10 == 0; weightedIdGenerator.registerGlobalConstraints(partitionConstraint); val totalTime = TestUtil.runMTTest(5, 100000, (k) -> weightedIdGenerator.generateWithConstraints("P", (String) null, false), this.getClass().getName() + ".testGenerateWithConstraints"); - testUniqueIds(weightedIdGenerator.getDataStore()); + testUniqueIds(weightedIdGenerator.getIdStore()); - for (Map.Entry> entry : weightedIdGenerator.getDataStore().entrySet()) { + for (Map.Entry> entry : weightedIdGenerator.getIdStore().entrySet()) { val prefix = entry.getKey(); val prefixIds = entry.getValue(); HashSet uniqueIds = new HashSet<>(); @@ -66,7 +66,7 @@ void testGenerateWithConstraints() throws IOException { val partitionIdTracker = prefixEntry.getValue(); for (int idx=0; idx < partitionIdTracker.getPartitionSize(); idx += 1) { if (!partitionConstraint.isValid(idx)) { - Assertions.assertEquals(0, partitionIdTracker.getIdList()[idx].getPointer().get()); + Assertions.assertEquals(0, partitionIdTracker.getIdPoolList()[idx].getPointer().get()); } } } @@ -82,10 +82,10 @@ void testUniqueIds(Map> dataStore) { val key = prefixEntry.getKey(); val partitionIdTracker = prefixEntry.getValue(); HashSet uniqueIds = new HashSet<>(); - for (val idPool: partitionIdTracker.getIdList()) { + for (val idPool: partitionIdTracker.getIdPoolList()) { boolean allIdsUniqueInList = true; HashSet uniqueIdsInList = new HashSet<>(); - for (val id: idPool.getIds()) { + for (val id: idPool.getIdList()) { if (uniqueIdsInList.contains(id)) { allIdsUniqueInList = false; allIdsUnique = false; From 77a88d8eb348d7e4ae95013c09444da4af28328b Mon Sep 17 00:00:00 2001 From: Satyam Saxena Date: Tue, 14 May 2024 19:07:19 +0530 Subject: [PATCH 06/12] Add IdGeneratorRetryConfig --- .../bundle/id/IdGeneratorRetryConfig.java | 24 ++++++ .../bundle/id/PartitionAwareIdGenerator.java | 86 ++++++++----------- .../bundle/id/WeightedIdGenerator.java | 8 +- .../bundle/id/formatter/IdFormatters.java | 6 +- ...er.java => PartitionAwareIdFormatter.java} | 2 +- .../bundle/id/IdGeneratorPerfTest.java | 4 +- .../id/PartitionAwareIdGeneratorPerfTest.java | 5 +- .../id/PartitionAwareIdGeneratorTest.java | 28 ++++-- .../id/WeightedIdGeneratorPerfTest.java | 6 +- .../bundle/id/WeightedIdGeneratorTest.java | 29 +++++-- 10 files changed, 123 insertions(+), 75 deletions(-) create mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGeneratorRetryConfig.java rename ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/{DistributedIdFormatter.java => PartitionAwareIdFormatter.java} (89%) diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGeneratorRetryConfig.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGeneratorRetryConfig.java new file mode 100644 index 00000000..df82ab68 --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGeneratorRetryConfig.java @@ -0,0 +1,24 @@ +package io.appform.ranger.discovery.bundle.id; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class IdGeneratorRetryConfig { + @NotNull + @Min(1) + private int idGenerationRetryCount; + + @NotNull + @Min(1) + private int partitionRetryCount; + +} \ No newline at end of file diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGenerator.java index d4ac8ddf..8a7a3a90 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGenerator.java @@ -39,12 +39,7 @@ public class PartitionAwareIdGenerator { private static final int MINIMUM_ID_LENGTH = 22; protected static final SecureRandom SECURE_RANDOM = new SecureRandom(Long.toBinaryString(System.currentTimeMillis()).getBytes()); private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormat.forPattern("yyMMddHHmmss"); - private final RetryPolicy RETRY_POLICY = RetryPolicy.builder() - .withMaxAttempts(readRetryCount()) - .handleIf(throwable -> true) - .handleResultIf(Objects::isNull) - .build(); - protected final FailsafeExecutor RETRIER = Failsafe.with(Collections.singletonList(RETRY_POLICY)); + protected final FailsafeExecutor retrier; private static final Pattern PATTERN = Pattern.compile("(.*)([0-9]{12})([0-9]{4})([0-9]{6})"); private static final List GLOBAL_CONSTRAINTS = new ArrayList<>(); private static final Map> DOMAIN_SPECIFIC_CONSTRAINTS = new HashMap<>(); @@ -53,9 +48,10 @@ public class PartitionAwareIdGenerator { private final Map> idStore = new ConcurrentHashMap<>(); protected final IdFormatter idFormatter; protected final Function partitionResolver; + protected final IdGeneratorRetryConfig retryConfig; protected final int partitionCount; -/* dataStore Structure +/* idStore Structure { prefix: { timestamp: { @@ -75,11 +71,21 @@ public class PartitionAwareIdGenerator { } */ - public PartitionAwareIdGenerator(final int partitionSize, - final Function partitionResolverSupplier) { - partitionCount = partitionSize; - partitionResolver = partitionResolverSupplier; - idFormatter = IdFormatters.distributed(); + public PartitionAwareIdGenerator(final int partitionCount, + final Function partitionResolverSupplier, + final IdGeneratorRetryConfig retryConfig, + final IdFormatter idFormatterInstance) { + this.partitionCount = partitionCount; + this.retryConfig = retryConfig; + this.partitionResolver = partitionResolverSupplier; + this.idFormatter = idFormatterInstance; + RetryPolicy retryPolicy = RetryPolicy.builder() + .withMaxAttempts(retryConfig.getPartitionRetryCount()) + .handleIf(throwable -> true) + .handleResultIf(Objects::isNull) + .build(); + retrier = Failsafe.with(Collections.singletonList(retryPolicy)); + val executorService = Executors.newScheduledThreadPool(1); executorService.scheduleWithFixedDelay( this::deleteExpiredKeys, @@ -88,12 +94,10 @@ public PartitionAwareIdGenerator(final int partitionSize, TimeUnit.SECONDS); } - public PartitionAwareIdGenerator(final int partitionSize, + public PartitionAwareIdGenerator(final int partitionCount, final Function partitionResolverSupplier, - final IdFormatter idFormatterInstance) { - partitionCount = partitionSize; - partitionResolver = partitionResolverSupplier; - idFormatter = idFormatterInstance; + final IdGeneratorRetryConfig retryConfig) { + this(partitionCount, partitionResolverSupplier, retryConfig, IdFormatters.partitionAware()); } public synchronized void registerGlobalConstraints(final PartitionValidationConstraint... constraints) { @@ -154,12 +158,13 @@ private int generateForAllPartitions(final PartitionIdTracker partitionIdTracker final int targetPartitionId) { val idPool = partitionIdTracker.getPartition(targetPartitionId); int idIdx = idPool.getPointer().getAndIncrement(); -// ToDo: Add Retry Limit - while (idPool.getIdList().size() <= idIdx) { + int retry = 0; + while (idPool.getIdList().size() <= idIdx && retry < retryConfig.getIdGenerationRetryCount()) { val counterValue = partitionIdTracker.getNextIdCounter().getAndIncrement(); val txnId = String.format("%s%s", prefix, idFormatter.format(timestamp, NODE_ID, counterValue)); val mappedPartitionId = partitionResolver.apply(txnId); partitionIdTracker.getPartition(mappedPartitionId).getIdList().add(counterValue); + retry += 1; } return idPool.getId(idIdx); } @@ -221,8 +226,7 @@ public Optional parse(final String idString) { .build()); } return Optional.empty(); - } - catch (Exception e) { + } catch (Exception e) { log.warn("Could not parse idString {}", e.getMessage()); return Optional.empty(); } @@ -234,7 +238,7 @@ private int getTargetPartitionId() { private Optional getTargetPartitionId(final List inConstraints, final boolean skipGlobal) { return Optional.ofNullable( - RETRIER.get(() -> SECURE_RANDOM.nextInt(partitionCount))) + retrier.get(() -> SECURE_RANDOM.nextInt(partitionCount))) .filter(key -> validateId(inConstraints, key, skipGlobal)); } @@ -244,42 +248,26 @@ protected boolean validateId(final List inConstra //First evaluate global constraints val failedGlobalConstraint = skipGlobal - ? null - : GLOBAL_CONSTRAINTS.stream() - .filter(constraint -> !constraint.isValid(partitionId)) - .findFirst() - .orElse(null); + ? null + : GLOBAL_CONSTRAINTS.stream() + .filter(constraint -> !constraint.isValid(partitionId)) + .findFirst() + .orElse(null); if (null != failedGlobalConstraint) { return false; } //Evaluate param constraints val failedLocalConstraint = null == inConstraints - ? null - : inConstraints.stream() - .filter(constraint -> !constraint.isValid(partitionId)) - .findFirst() - .orElse(null); + ? null + : inConstraints.stream() + .filter(constraint -> !constraint.isValid(partitionId)) + .findFirst() + .orElse(null); return null == failedLocalConstraint; } - protected int readRetryCount() { - try { -// Make it config - val count = Integer.parseInt(System.getenv().getOrDefault("NUM_ID_GENERATION_RETRIES", "512")); - if (count <= 0) { - throw new IllegalArgumentException( - "Negative number of retries does not make sense. Please set a proper value for " + - "NUM_ID_GENERATION_RETRIES"); - } - return count; - } - catch (NumberFormatException e) { - throw new IllegalArgumentException("Please provide a valid positive integer for NUM_ID_GENERATION_RETRIES"); - } - } - - private void deleteExpiredKeys() { + private synchronized void deleteExpiredKeys() { val timeThreshold = DateTime.now().getMillis() / 1000 - Constants.DELETION_THRESHOLD_IN_SECONDS; for (val entry : idStore.entrySet()) { entry.getValue().entrySet().removeIf(partitionIdTrackerEntry -> partitionIdTrackerEntry.getKey() < timeThreshold); diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java index 3cbc779a..dae2ed30 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java @@ -23,16 +23,18 @@ public class WeightedIdGenerator extends PartitionAwareIdGenerator { public WeightedIdGenerator(final int partitionSize, final Function partitionResolverSupplier, + final IdGeneratorRetryConfig retryConfig, final WeightedIdConfig weightedIdConfig) { - super(partitionSize, partitionResolverSupplier); + super(partitionSize, partitionResolverSupplier, retryConfig); partitionRangeMap = createWeightRangeMap(weightedIdConfig); } public WeightedIdGenerator(final int partitionSize, final Function partitionResolverSupplier, + final IdGeneratorRetryConfig retryConfig, final WeightedIdConfig weightedIdConfig, final IdFormatter idFormatterInstance) { - super(partitionSize, partitionResolverSupplier, idFormatterInstance); + super(partitionSize, partitionResolverSupplier, retryConfig, idFormatterInstance); partitionRangeMap = createWeightRangeMap(weightedIdConfig); } @@ -58,7 +60,7 @@ private int getTargetPartitionId() { @Override private Optional getTargetPartitionId(final List inConstraints, final boolean skipGlobal) { return Optional.ofNullable( - RETRIER.get(this::getTargetPartitionId)) + retrier.get(this::getTargetPartitionId)) .filter(key -> validateId(inConstraints, key, skipGlobal)); } } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatters.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatters.java index f6a4e43c..da7bbab8 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatters.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatters.java @@ -23,7 +23,7 @@ public class IdFormatters { private static final IdFormatter originalIdFormatter = new DefaultIdFormatter(); private static final IdFormatter base36IdFormatter = new Base36IdFormatter(originalIdFormatter); - private static final IdFormatter distributedIdFormatter = new DistributedIdFormatter(); + private static final IdFormatter partitionAwareIdFormatter = new PartitionAwareIdFormatter(); public static IdFormatter original() { return originalIdFormatter; @@ -33,8 +33,8 @@ public static IdFormatter base36() { return base36IdFormatter; } - public static IdFormatter distributed() { - return distributedIdFormatter; + public static IdFormatter partitionAware() { + return partitionAwareIdFormatter; } } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/DistributedIdFormatter.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/PartitionAwareIdFormatter.java similarity index 89% rename from ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/DistributedIdFormatter.java rename to ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/PartitionAwareIdFormatter.java index f242fb77..9bdbd6e0 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/DistributedIdFormatter.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/PartitionAwareIdFormatter.java @@ -4,7 +4,7 @@ import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; -public class DistributedIdFormatter implements IdFormatter { +public class PartitionAwareIdFormatter implements IdFormatter { private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormat.forPattern("yyMMddHHmmss"); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdGeneratorPerfTest.java index 55f6cd92..e1b0087d 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdGeneratorPerfTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdGeneratorPerfTest.java @@ -6,8 +6,8 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is - * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * Unless required by applicable law or agreed to in writing, software partitionAware under the License is + * partitionAware on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either * express or implied. See the License for the specific language governing permissions and limitations * under the License. */ diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java index aafcf691..469522d8 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java @@ -25,7 +25,10 @@ public static class BenchmarkState { @Setup(Level.Trial) public void setUp() throws IOException { - partitionAwareIdGenerator = new PartitionAwareIdGenerator(1024, partitionResolverSupplier); + partitionAwareIdGenerator = new PartitionAwareIdGenerator( + 1024, partitionResolverSupplier, + IdGeneratorRetryConfig.builder().idGenerationRetryCount(100).partitionRetryCount(100).build() + ); } } diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java index 7a39fa2a..e2f4a7f5 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java @@ -23,12 +23,15 @@ @SuppressWarnings({"unused", "FieldMayBeFinal"}) class PartitionAwareIdGeneratorTest { final int partitionCount = 1024; - final Function partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length()-6)) % partitionCount; + final Function partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length() - 6)) % partitionCount; private PartitionAwareIdGenerator partitionAwareIdGenerator; @BeforeEach void setup() { - partitionAwareIdGenerator = new PartitionAwareIdGenerator(partitionCount, partitionResolverSupplier); + partitionAwareIdGenerator = new PartitionAwareIdGenerator( + partitionCount, partitionResolverSupplier, + IdGeneratorRetryConfig.builder().idGenerationRetryCount(100).partitionRetryCount(100).build() + ); } @Test @@ -51,7 +54,7 @@ void testGenerateWithConstraints() throws IOException { for (Map.Entry prefixEntry : prefixIds.entrySet()) { val key = prefixEntry.getKey(); val partitionIdTracker = prefixEntry.getValue(); - for (int idx=0; idx < partitionIdTracker.getPartitionSize(); idx += 1) { + for (int idx = 0; idx < partitionIdTracker.getPartitionSize(); idx += 1) { if (!partitionConstraint.isValid(idx)) { Assertions.assertEquals(0, partitionIdTracker.getIdPoolList()[idx].getPointer().get()); } @@ -69,10 +72,10 @@ void testUniqueIdsInDataStore(Map> dataSto val key = prefixEntry.getKey(); val partitionIdTracker = prefixEntry.getValue(); HashSet uniqueIds = new HashSet<>(); - for (val idPool: partitionIdTracker.getIdPoolList()) { + for (val idPool : partitionIdTracker.getIdPoolList()) { boolean allIdsUniqueInList = true; HashSet uniqueIdsInList = new HashSet<>(); - for (val id: idPool.getIdList()) { + for (val id : idPool.getIdList()) { if (uniqueIdsInList.contains(id)) { allIdsUniqueInList = false; allIdsUnique = false; @@ -97,7 +100,7 @@ void testUniqueIdsInDataStore(Map> dataSto void testUniqueIds() { HashSet allIDs = new HashSet<>(); boolean allIdsUnique = true; - for (int i=0; i < 10000; i+=1) { + for (int i = 0; i < 10000; i += 1) { val txnId = partitionAwareIdGenerator.generate("P").getId(); if (allIDs.contains(txnId)) { log.warn(txnId); @@ -112,14 +115,23 @@ void testUniqueIds() { @Test void testGenerateOriginal() { - partitionAwareIdGenerator = new PartitionAwareIdGenerator(partitionCount, partitionResolverSupplier, IdFormatters.original()); + partitionAwareIdGenerator = new PartitionAwareIdGenerator( + partitionCount, partitionResolverSupplier, + IdGeneratorRetryConfig.builder().idGenerationRetryCount(100).partitionRetryCount(100).build(), + IdFormatters.original() + ); String id = partitionAwareIdGenerator.generate("TEST").getId(); Assertions.assertEquals(26, id.length()); } @Test void testGenerateBase36() { - partitionAwareIdGenerator = new PartitionAwareIdGenerator(partitionCount, (txnId) -> new BigInteger(txnId.substring(txnId.length()-6), 36).abs().intValue() % partitionCount, IdFormatters.base36()); + partitionAwareIdGenerator = new PartitionAwareIdGenerator( + partitionCount, + (txnId) -> new BigInteger(txnId.substring(txnId.length() - 6), 36).abs().intValue() % partitionCount, + IdGeneratorRetryConfig.builder().idGenerationRetryCount(100).partitionRetryCount(100).build(), + IdFormatters.base36() + ); String id = partitionAwareIdGenerator.generate("TEST").getId(); Assertions.assertEquals(18, id.length()); } diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java index 9b8dbd24..fae1de1a 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java @@ -39,7 +39,11 @@ public void setUp() throws IOException { val weightedIdConfig = WeightedIdConfig.builder() .partitions(partitionConfigList) .build(); - weightedIdGenerator = new WeightedIdGenerator(1024, partitionResolverSupplier, weightedIdConfig); + weightedIdGenerator = new WeightedIdGenerator( + 1024, partitionResolverSupplier, + IdGeneratorRetryConfig.builder().idGenerationRetryCount(100).partitionRetryCount(100).build(), + weightedIdConfig + ); } } diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java index 205ea483..13243954 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java @@ -25,7 +25,7 @@ @SuppressWarnings({"unused", "FieldMayBeFinal"}) class WeightedIdGeneratorTest { final int partitionCount = 1024; - final Function partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length()-6)) % partitionCount; + final Function partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length() - 6)) % partitionCount; private WeightedIdGenerator weightedIdGenerator; private WeightedIdConfig weightedIdConfig; @@ -41,7 +41,11 @@ void setup() { weightedIdConfig = WeightedIdConfig.builder() .partitions(partitionConfigList) .build(); - weightedIdGenerator = new WeightedIdGenerator(partitionCount, partitionResolverSupplier, weightedIdConfig); + weightedIdGenerator = new WeightedIdGenerator( + partitionCount, partitionResolverSupplier, + IdGeneratorRetryConfig.builder().idGenerationRetryCount(100).partitionRetryCount(100).build(), + weightedIdConfig + ); } @Test @@ -64,7 +68,7 @@ void testGenerateWithConstraints() throws IOException { for (Map.Entry prefixEntry : prefixIds.entrySet()) { val key = prefixEntry.getKey(); val partitionIdTracker = prefixEntry.getValue(); - for (int idx=0; idx < partitionIdTracker.getPartitionSize(); idx += 1) { + for (int idx = 0; idx < partitionIdTracker.getPartitionSize(); idx += 1) { if (!partitionConstraint.isValid(idx)) { Assertions.assertEquals(0, partitionIdTracker.getIdPoolList()[idx].getPointer().get()); } @@ -82,10 +86,10 @@ void testUniqueIds(Map> dataStore) { val key = prefixEntry.getKey(); val partitionIdTracker = prefixEntry.getValue(); HashSet uniqueIds = new HashSet<>(); - for (val idPool: partitionIdTracker.getIdPoolList()) { + for (val idPool : partitionIdTracker.getIdPoolList()) { boolean allIdsUniqueInList = true; HashSet uniqueIdsInList = new HashSet<>(); - for (val id: idPool.getIdList()) { + for (val id : idPool.getIdList()) { if (uniqueIdsInList.contains(id)) { allIdsUniqueInList = false; allIdsUnique = false; @@ -108,14 +112,25 @@ void testUniqueIds(Map> dataStore) { @Test void testGenerateOriginal() { - weightedIdGenerator = new WeightedIdGenerator(partitionCount, partitionResolverSupplier, weightedIdConfig, IdFormatters.original()); + weightedIdGenerator = new WeightedIdGenerator( + partitionCount, partitionResolverSupplier, + IdGeneratorRetryConfig.builder().idGenerationRetryCount(100).partitionRetryCount(100).build(), + weightedIdConfig, + IdFormatters.original() + ); String id = weightedIdGenerator.generate("TEST").getId(); Assertions.assertEquals(26, id.length()); } @Test void testGenerateBase36() { - weightedIdGenerator = new WeightedIdGenerator(partitionCount, (txnId) -> new BigInteger(txnId.substring(txnId.length()-6), 36).abs().intValue() % partitionCount, weightedIdConfig, IdFormatters.base36()); + weightedIdGenerator = new WeightedIdGenerator( + partitionCount, + (txnId) -> new BigInteger(txnId.substring(txnId.length() - 6), 36).abs().intValue() % partitionCount, + IdGeneratorRetryConfig.builder().idGenerationRetryCount(100).partitionRetryCount(100).build(), + weightedIdConfig, + IdFormatters.base36() + ); String id = weightedIdGenerator.generate("TEST").getId(); Assertions.assertEquals(18, id.length()); } From 3d31d9335b7624e08f0511fc8faacf009fb870b4 Mon Sep 17 00:00:00 2001 From: Satyam Saxena Date: Wed, 15 May 2024 14:15:36 +0530 Subject: [PATCH 07/12] Add DistributedIdGenerator base class --- .../{ => config}/IdGeneratorRetryConfig.java | 2 +- .../id/{ => config}/PartitionRange.java | 2 +- .../id/{ => config}/WeightedIdConfig.java | 2 +- .../id/{ => config}/WeightedPartition.java | 4 +- .../DistributedIdGenerator.java} | 27 ++++++------ .../bundle/id/{ => weighted}/IdPool.java | 2 +- .../weighted/PartitionAwareIdGenerator.java | 44 +++++++++++++++++++ .../id/{ => weighted}/PartitionIdTracker.java | 2 +- .../{ => weighted}/WeightedIdGenerator.java | 11 +++-- .../bundle/id/IdGeneratorPerfTest.java | 4 +- .../id/PartitionAwareIdGeneratorPerfTest.java | 2 + .../id/PartitionAwareIdGeneratorTest.java | 3 ++ .../id/WeightedIdGeneratorPerfTest.java | 5 +++ .../bundle/id/WeightedIdGeneratorTest.java | 6 +++ 14 files changed, 89 insertions(+), 27 deletions(-) rename ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/{ => config}/IdGeneratorRetryConfig.java (88%) rename ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/{ => config}/PartitionRange.java (92%) rename ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/{ => config}/WeightedIdConfig.java (95%) rename ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/{ => config}/WeightedPartition.java (86%) rename ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/{PartitionAwareIdGenerator.java => weighted/DistributedIdGenerator.java} (93%) rename ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/{ => weighted}/IdPool.java (89%) create mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionAwareIdGenerator.java rename ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/{ => weighted}/PartitionIdTracker.java (94%) rename ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/{ => weighted}/WeightedIdGenerator.java (83%) diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGeneratorRetryConfig.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/IdGeneratorRetryConfig.java similarity index 88% rename from ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGeneratorRetryConfig.java rename to ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/IdGeneratorRetryConfig.java index df82ab68..8ffc9d9a 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGeneratorRetryConfig.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/IdGeneratorRetryConfig.java @@ -1,4 +1,4 @@ -package io.appform.ranger.discovery.bundle.id; +package io.appform.ranger.discovery.bundle.id.config; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionRange.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/PartitionRange.java similarity index 92% rename from ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionRange.java rename to ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/PartitionRange.java index f4dbf2ff..6fb90c73 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionRange.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/PartitionRange.java @@ -1,4 +1,4 @@ -package io.appform.ranger.discovery.bundle.id; +package io.appform.ranger.discovery.bundle.id.config; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdConfig.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/WeightedIdConfig.java similarity index 95% rename from ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdConfig.java rename to ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/WeightedIdConfig.java index 9fd7a291..8f97a7cc 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdConfig.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/WeightedIdConfig.java @@ -1,4 +1,4 @@ -package io.appform.ranger.discovery.bundle.id; +package io.appform.ranger.discovery.bundle.id.config; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedPartition.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/WeightedPartition.java similarity index 86% rename from ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedPartition.java rename to ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/WeightedPartition.java index ca9969e1..4367871a 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedPartition.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/WeightedPartition.java @@ -1,4 +1,4 @@ -package io.appform.ranger.discovery.bundle.id; +package io.appform.ranger.discovery.bundle.id.config; import lombok.AllArgsConstructor; import lombok.Builder; @@ -19,6 +19,6 @@ public class WeightedPartition { private PartitionRange partitionRange; @NotNull - @Min(0) + @Min(1) private int weight; } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java similarity index 93% rename from ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGenerator.java rename to ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java index 8a7a3a90..0391a79e 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java @@ -1,10 +1,14 @@ -package io.appform.ranger.discovery.bundle.id; +package io.appform.ranger.discovery.bundle.id.weighted; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import dev.failsafe.Failsafe; import dev.failsafe.FailsafeExecutor; import dev.failsafe.RetryPolicy; +import io.appform.ranger.discovery.bundle.id.Constants; +import io.appform.ranger.discovery.bundle.id.Id; +import io.appform.ranger.discovery.bundle.id.IdGenerator; +import io.appform.ranger.discovery.bundle.id.config.IdGeneratorRetryConfig; import io.appform.ranger.discovery.bundle.id.constraints.PartitionValidationConstraint; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatter; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; @@ -34,7 +38,7 @@ */ @SuppressWarnings("unused") @Slf4j -public class PartitionAwareIdGenerator { +abstract class DistributedIdGenerator { private static final int MINIMUM_ID_LENGTH = 22; protected static final SecureRandom SECURE_RANDOM = new SecureRandom(Long.toBinaryString(System.currentTimeMillis()).getBytes()); @@ -51,7 +55,7 @@ public class PartitionAwareIdGenerator { protected final IdGeneratorRetryConfig retryConfig; protected final int partitionCount; -/* idStore Structure + /* idStore Structure { prefix: { timestamp: { @@ -69,9 +73,9 @@ public class PartitionAwareIdGenerator { } } } - */ + */ - public PartitionAwareIdGenerator(final int partitionCount, + protected DistributedIdGenerator(final int partitionCount, final Function partitionResolverSupplier, final IdGeneratorRetryConfig retryConfig, final IdFormatter idFormatterInstance) { @@ -94,7 +98,7 @@ public PartitionAwareIdGenerator(final int partitionCount, TimeUnit.SECONDS); } - public PartitionAwareIdGenerator(final int partitionCount, + protected DistributedIdGenerator(final int partitionCount, final Function partitionResolverSupplier, final IdGeneratorRetryConfig retryConfig) { this(partitionCount, partitionResolverSupplier, retryConfig, IdFormatters.partitionAware()); @@ -232,15 +236,10 @@ public Optional parse(final String idString) { } } - private int getTargetPartitionId() { - return SECURE_RANDOM.nextInt(partitionCount); - } + protected abstract int getTargetPartitionId(); - private Optional getTargetPartitionId(final List inConstraints, final boolean skipGlobal) { - return Optional.ofNullable( - retrier.get(() -> SECURE_RANDOM.nextInt(partitionCount))) - .filter(key -> validateId(inConstraints, key, skipGlobal)); - } + protected abstract Optional getTargetPartitionId(final List inConstraints, + final boolean skipGlobal); protected boolean validateId(final List inConstraints, final int partitionId, diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdPool.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/IdPool.java similarity index 89% rename from ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdPool.java rename to ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/IdPool.java index a8618262..753656d9 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdPool.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/IdPool.java @@ -1,4 +1,4 @@ -package io.appform.ranger.discovery.bundle.id; +package io.appform.ranger.discovery.bundle.id.weighted; import lombok.Getter; diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionAwareIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionAwareIdGenerator.java new file mode 100644 index 00000000..12cb4b29 --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionAwareIdGenerator.java @@ -0,0 +1,44 @@ +package io.appform.ranger.discovery.bundle.id.weighted; + +import io.appform.ranger.discovery.bundle.id.config.IdGeneratorRetryConfig; +import io.appform.ranger.discovery.bundle.id.constraints.PartitionValidationConstraint; +import io.appform.ranger.discovery.bundle.id.formatter.IdFormatter; +import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.Optional; +import java.util.function.Function; + +/** + * Partition Aware Id Generation + */ +@SuppressWarnings("unused") +@Slf4j +public class PartitionAwareIdGenerator extends DistributedIdGenerator { + public PartitionAwareIdGenerator(final int partitionCount, + final Function partitionResolverSupplier, + final IdGeneratorRetryConfig retryConfig, + final IdFormatter idFormatterInstance) { + super(partitionCount, partitionResolverSupplier, retryConfig, idFormatterInstance); + } + + public PartitionAwareIdGenerator(final int partitionCount, + final Function partitionResolverSupplier, + final IdGeneratorRetryConfig retryConfig) { + super(partitionCount, partitionResolverSupplier, retryConfig, IdFormatters.partitionAware()); + } + + @Override + protected int getTargetPartitionId() { + return SECURE_RANDOM.nextInt(partitionCount); + } + + @Override + protected Optional getTargetPartitionId(final List inConstraints, final boolean skipGlobal) { + return Optional.ofNullable( + retrier.get(() -> SECURE_RANDOM.nextInt(partitionCount))) + .filter(key -> validateId(inConstraints, key, skipGlobal)); + } + +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionIdTracker.java similarity index 94% rename from ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java rename to ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionIdTracker.java index f4d69932..125e8c4b 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionIdTracker.java @@ -1,4 +1,4 @@ -package io.appform.ranger.discovery.bundle.id; +package io.appform.ranger.discovery.bundle.id.weighted; import lombok.Getter; diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/WeightedIdGenerator.java similarity index 83% rename from ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java rename to ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/WeightedIdGenerator.java index dae2ed30..f54fbe51 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/WeightedIdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/WeightedIdGenerator.java @@ -1,8 +1,11 @@ -package io.appform.ranger.discovery.bundle.id; +package io.appform.ranger.discovery.bundle.id.weighted; import com.google.common.collect.Range; import com.google.common.collect.RangeMap; import com.google.common.collect.TreeRangeMap; +import io.appform.ranger.discovery.bundle.id.config.IdGeneratorRetryConfig; +import io.appform.ranger.discovery.bundle.id.config.PartitionRange; +import io.appform.ranger.discovery.bundle.id.config.WeightedIdConfig; import io.appform.ranger.discovery.bundle.id.constraints.PartitionValidationConstraint; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatter; import lombok.extern.slf4j.Slf4j; @@ -17,7 +20,7 @@ * Weighted Id Generation */ @Slf4j -public class WeightedIdGenerator extends PartitionAwareIdGenerator { +public class WeightedIdGenerator extends DistributedIdGenerator { private int maxShardWeight; private final RangeMap partitionRangeMap; @@ -51,14 +54,14 @@ private RangeMap createWeightRangeMap(final WeightedIdC } @Override - private int getTargetPartitionId() { + protected int getTargetPartitionId() { val randomNum = SECURE_RANDOM.nextInt(maxShardWeight); val partitionRange = Objects.requireNonNull(partitionRangeMap.getEntry(randomNum)).getValue(); return SECURE_RANDOM.nextInt(partitionRange.getEnd() - partitionRange.getStart() + 1) + partitionRange.getStart(); } @Override - private Optional getTargetPartitionId(final List inConstraints, final boolean skipGlobal) { + protected Optional getTargetPartitionId(final List inConstraints, final boolean skipGlobal) { return Optional.ofNullable( retrier.get(this::getTargetPartitionId)) .filter(key -> validateId(inConstraints, key, skipGlobal)); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdGeneratorPerfTest.java index e1b0087d..55f6cd92 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdGeneratorPerfTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdGeneratorPerfTest.java @@ -6,8 +6,8 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software partitionAware under the License is - * partitionAware on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * Unless required by applicable law or agreed to in writing, software distributed under the License is + * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either * express or implied. See the License for the specific language governing permissions and limitations * under the License. */ diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java index 469522d8..336fdea9 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java @@ -1,5 +1,7 @@ package io.appform.ranger.discovery.bundle.id; +import io.appform.ranger.discovery.bundle.id.config.IdGeneratorRetryConfig; +import io.appform.ranger.discovery.bundle.id.weighted.PartitionAwareIdGenerator; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.openjdk.jmh.annotations.Benchmark; diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java index e2f4a7f5..7cb7d341 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java @@ -1,7 +1,10 @@ package io.appform.ranger.discovery.bundle.id; +import io.appform.ranger.discovery.bundle.id.config.IdGeneratorRetryConfig; import io.appform.ranger.discovery.bundle.id.constraints.PartitionValidationConstraint; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; +import io.appform.ranger.discovery.bundle.id.weighted.PartitionAwareIdGenerator; +import io.appform.ranger.discovery.bundle.id.weighted.PartitionIdTracker; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.junit.jupiter.api.Assertions; diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java index fae1de1a..dde9dbb2 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java @@ -1,5 +1,10 @@ package io.appform.ranger.discovery.bundle.id; +import io.appform.ranger.discovery.bundle.id.config.IdGeneratorRetryConfig; +import io.appform.ranger.discovery.bundle.id.config.WeightedIdConfig; +import io.appform.ranger.discovery.bundle.id.config.PartitionRange; +import io.appform.ranger.discovery.bundle.id.weighted.WeightedIdGenerator; +import io.appform.ranger.discovery.bundle.id.config.WeightedPartition; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import lombok.val; diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java index 13243954..759eeec0 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java @@ -1,7 +1,13 @@ package io.appform.ranger.discovery.bundle.id; +import io.appform.ranger.discovery.bundle.id.config.IdGeneratorRetryConfig; +import io.appform.ranger.discovery.bundle.id.config.WeightedIdConfig; import io.appform.ranger.discovery.bundle.id.constraints.PartitionValidationConstraint; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; +import io.appform.ranger.discovery.bundle.id.weighted.PartitionIdTracker; +import io.appform.ranger.discovery.bundle.id.config.PartitionRange; +import io.appform.ranger.discovery.bundle.id.weighted.WeightedIdGenerator; +import io.appform.ranger.discovery.bundle.id.config.WeightedPartition; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.junit.jupiter.api.Assertions; From 54f4e47f22f197408e7fb7c30dd478f0c60b76cb Mon Sep 17 00:00:00 2001 From: Satyam Saxena Date: Wed, 15 May 2024 16:04:02 +0530 Subject: [PATCH 08/12] Add Retry Limit for Weighted ID generation --- ...butedIdGeneratorPerfTest.testGenerate.json | 8 ---- ...neratorTest.testGenerateWithBenchmark.json | 7 ---- ...ratorTest.testGenerateWithConstraints.json | 7 ---- ...e.id.IdGeneratorPerfTest.testGenerate.json | 2 +- ...dGeneratorPerfTest.testGenerateBase36.json | 2 +- ...AwareIdGeneratorPerfTest.testGenerate.json | 8 ++++ ...neratorTest.testGenerateWithBenchmark.json | 7 ++++ ...ratorTest.testGenerateWithConstraints.json | 7 ++++ ...ghtedIdGeneratorPerfTest.testGenerate.json | 2 +- ...neratorTest.testGenerateWithBenchmark.json | 4 +- ...ratorTest.testGenerateWithConstraints.json | 4 +- .../id/weighted/DistributedIdGenerator.java | 42 ++++++++++++------- .../id/weighted/WeightedIdGenerator.java | 2 +- .../id/PartitionAwareIdGeneratorPerfTest.java | 2 +- .../id/PartitionAwareIdGeneratorTest.java | 21 +++++----- .../id/WeightedIdGeneratorPerfTest.java | 2 +- .../bundle/id/WeightedIdGeneratorTest.java | 24 ++++------- 17 files changed, 76 insertions(+), 75 deletions(-) delete mode 100644 ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorPerfTest.testGenerate.json delete mode 100644 ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithBenchmark.json delete mode 100644 ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithConstraints.json create mode 100644 ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorPerfTest.testGenerate.json create mode 100644 ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark.json create mode 100644 ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints.json diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorPerfTest.testGenerate.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorPerfTest.testGenerate.json deleted file mode 100644 index fa3a95de..00000000 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorPerfTest.testGenerate.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name" : "io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorPerfTest.testGenerate", - "mode" : "Throughput", - "iterations" : 4, - "threads" : 1, - "forks" : 3, - "mean_ops" : 406378.3277020471 -} \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithBenchmark.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithBenchmark.json deleted file mode 100644 index 8ff858d7..00000000 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithBenchmark.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name" : "io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithBenchmark", - "iterations" : 100000, - "threads" : 5, - "totalMillis" : 3823, - "avgTime" : 764.6 -} \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithConstraints.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithConstraints.json deleted file mode 100644 index 0d0c5e92..00000000 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithConstraints.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name" : "io.appform.ranger.discovery.bundle.id.DistributedIdGeneratorTest.testGenerateWithConstraints", - "iterations" : 100000, - "threads" : 5, - "totalMillis" : 5386, - "avgTime" : 1077.2 -} \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json index e2d661ae..47d6122a 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 800941.387237486 + "mean_ops" : 826587.8324153908 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json index d8e026c4..77d69a81 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 643473.2832640476 + "mean_ops" : 654567.0311484573 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorPerfTest.testGenerate.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorPerfTest.testGenerate.json new file mode 100644 index 00000000..9a36d5e1 --- /dev/null +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorPerfTest.testGenerate.json @@ -0,0 +1,8 @@ +{ + "name" : "io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorPerfTest.testGenerate", + "mode" : "Throughput", + "iterations" : 4, + "threads" : 1, + "forks" : 3, + "mean_ops" : 433192.4270233689 +} \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark.json new file mode 100644 index 00000000..a06bfe0f --- /dev/null +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark.json @@ -0,0 +1,7 @@ +{ + "name" : "io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark", + "iterations" : 100000, + "threads" : 5, + "totalMillis" : 6411, + "avgTime" : 1282.2 +} \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints.json new file mode 100644 index 00000000..6f814e59 --- /dev/null +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints.json @@ -0,0 +1,7 @@ +{ + "name" : "io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints", + "iterations" : 100000, + "threads" : 5, + "totalMillis" : 5189, + "avgTime" : 1037.8 +} \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorPerfTest.testGenerate.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorPerfTest.testGenerate.json index 0651b230..bfe6257c 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorPerfTest.testGenerate.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorPerfTest.testGenerate.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 410789.0590215035 + "mean_ops" : 379648.93091326446 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json index 755919de..d5cea3d2 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json @@ -2,6 +2,6 @@ "name" : "io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark", "iterations" : 100000, "threads" : 5, - "totalMillis" : 3929, - "avgTime" : 785.8 + "totalMillis" : 4128, + "avgTime" : 825.6 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json index ddb7f17a..dccc61e1 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json @@ -2,6 +2,6 @@ "name" : "io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints", "iterations" : 100000, "threads" : 5, - "totalMillis" : 2867, - "avgTime" : 573.4 + "totalMillis" : 4820, + "avgTime" : 964.0 } \ No newline at end of file diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java index 0391a79e..5c4a6bd8 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java @@ -133,12 +133,12 @@ public synchronized void registerDomainSpecificConstraints( * @param prefix String prefix for ID to be generated * @return Generated Id */ - public Id generate(final String prefix) { + public Optional generate(final String prefix) { val targetPartitionId = getTargetPartitionId(); return generateForPartition(prefix, targetPartitionId); } - public Id generateForPartition(final String prefix, final int targetPartitionId) { + public Optional generateForPartition(final String prefix, final int targetPartitionId) { val prefixIdMap = idStore.computeIfAbsent(prefix, k -> new ConcurrentHashMap<>()); val currentTimestamp = new DateTime(); val timeKey = currentTimestamp.getMillis() / 1000; @@ -147,19 +147,24 @@ public Id generateForPartition(final String prefix, final int targetPartitionId) prefix, currentTimestamp, targetPartitionId); - val id = String.format("%s%s", prefix, idFormatter.format(currentTimestamp, NODE_ID, idCounter)); - return Id.builder() - .id(id) - .exponent(idCounter) - .generatedDate(currentTimestamp.toDate()) - .node(NODE_ID) - .build(); + if (idCounter.isPresent()) { + val id = String.format("%s%s", prefix, idFormatter.format(currentTimestamp, NODE_ID, idCounter.get())); + return Optional.of( + Id.builder() + .id(id) + .exponent(idCounter.get()) + .generatedDate(currentTimestamp.toDate()) + .node(NODE_ID) + .build()); + } else { + return Optional.empty(); + } } - private int generateForAllPartitions(final PartitionIdTracker partitionIdTracker, - final String prefix, - final DateTime timestamp, - final int targetPartitionId) { + private Optional generateForAllPartitions(final PartitionIdTracker partitionIdTracker, + final String prefix, + final DateTime timestamp, + final int targetPartitionId) { val idPool = partitionIdTracker.getPartition(targetPartitionId); int idIdx = idPool.getPointer().getAndIncrement(); int retry = 0; @@ -170,7 +175,12 @@ private int generateForAllPartitions(final PartitionIdTracker partitionIdTracker partitionIdTracker.getPartition(mappedPartitionId).getIdList().add(counterValue); retry += 1; } - return idPool.getId(idIdx); + if (idIdx < idPool.getIdList().size()) { + return Optional.of(idPool.getId(idIdx)); + } else { + log.warn("Retry Limit reached - {} - {} - {}", retry, idIdx, targetPartitionId); + return Optional.empty(); + } } /** @@ -198,14 +208,14 @@ public Optional generateWithConstraints(final String prefix, final String do */ public Optional generateWithConstraints(final String prefix, final String domain, final boolean skipGlobal) { val targetPartitionId = getTargetPartitionId(DOMAIN_SPECIFIC_CONSTRAINTS.getOrDefault(domain, Collections.emptyList()), skipGlobal); - return targetPartitionId.map(id -> generateForPartition(prefix, id)); + return targetPartitionId.map(id -> generateForPartition(prefix, id).get()); } public Optional generateWithConstraints(final String prefix, final List inConstraints, final boolean skipGlobal) { val targetPartitionId = getTargetPartitionId(inConstraints, skipGlobal); - return targetPartitionId.map(id -> generateForPartition(prefix, id)); + return targetPartitionId.map(id -> generateForPartition(prefix, id).get()); } /** diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/WeightedIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/WeightedIdGenerator.java index f54fbe51..5769e569 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/WeightedIdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/WeightedIdGenerator.java @@ -45,7 +45,7 @@ private RangeMap createWeightRangeMap(final WeightedIdC RangeMap partitionGroups = TreeRangeMap.create(); int end = -1; for (val shard : weightedIdConfig.getPartitions()) { - int start = end+1; + int start = end + 1; end += shard.getWeight(); partitionGroups.put(Range.closed(start, end), shard.getPartitionRange()); } diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java index 336fdea9..11762482 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java @@ -29,7 +29,7 @@ public static class BenchmarkState { public void setUp() throws IOException { partitionAwareIdGenerator = new PartitionAwareIdGenerator( 1024, partitionResolverSupplier, - IdGeneratorRetryConfig.builder().idGenerationRetryCount(100).partitionRetryCount(100).build() + IdGeneratorRetryConfig.builder().idGenerationRetryCount(1024).partitionRetryCount(1024).build() ); } } diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java index 7cb7d341..ddb40b22 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java @@ -27,13 +27,13 @@ class PartitionAwareIdGeneratorTest { final int partitionCount = 1024; final Function partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length() - 6)) % partitionCount; + final IdGeneratorRetryConfig retryConfig = IdGeneratorRetryConfig.builder().idGenerationRetryCount(4096).partitionRetryCount(4096).build(); private PartitionAwareIdGenerator partitionAwareIdGenerator; @BeforeEach void setup() { partitionAwareIdGenerator = new PartitionAwareIdGenerator( - partitionCount, partitionResolverSupplier, - IdGeneratorRetryConfig.builder().idGenerationRetryCount(100).partitionRetryCount(100).build() + partitionCount, partitionResolverSupplier, retryConfig ); } @@ -45,7 +45,7 @@ void testGenerateWithBenchmark() throws IOException { @Test void testGenerateWithConstraints() throws IOException { - PartitionValidationConstraint partitionConstraint = (k) -> k % 10 == 0; + PartitionValidationConstraint partitionConstraint = (k) -> k % 2 == 0; partitionAwareIdGenerator.registerGlobalConstraints(partitionConstraint); val totalTime = TestUtil.runMTTest(5, 100000, (k) -> partitionAwareIdGenerator.generateWithConstraints("P", (String) null, false), this.getClass().getName() + ".testGenerateWithConstraints"); testUniqueIdsInDataStore(partitionAwareIdGenerator.getIdStore()); @@ -104,7 +104,7 @@ void testUniqueIds() { HashSet allIDs = new HashSet<>(); boolean allIdsUnique = true; for (int i = 0; i < 10000; i += 1) { - val txnId = partitionAwareIdGenerator.generate("P").getId(); + val txnId = partitionAwareIdGenerator.generate("P").get().getId(); if (allIDs.contains(txnId)) { log.warn(txnId); log.warn(String.valueOf(allIDs)); @@ -119,11 +119,9 @@ void testUniqueIds() { @Test void testGenerateOriginal() { partitionAwareIdGenerator = new PartitionAwareIdGenerator( - partitionCount, partitionResolverSupplier, - IdGeneratorRetryConfig.builder().idGenerationRetryCount(100).partitionRetryCount(100).build(), - IdFormatters.original() + partitionCount, partitionResolverSupplier, retryConfig, IdFormatters.original() ); - String id = partitionAwareIdGenerator.generate("TEST").getId(); + String id = partitionAwareIdGenerator.generate("TEST").get().getId(); Assertions.assertEquals(26, id.length()); } @@ -132,10 +130,11 @@ void testGenerateBase36() { partitionAwareIdGenerator = new PartitionAwareIdGenerator( partitionCount, (txnId) -> new BigInteger(txnId.substring(txnId.length() - 6), 36).abs().intValue() % partitionCount, - IdGeneratorRetryConfig.builder().idGenerationRetryCount(100).partitionRetryCount(100).build(), + retryConfig, IdFormatters.base36() ); - String id = partitionAwareIdGenerator.generate("TEST").getId(); + val idOptional = partitionAwareIdGenerator.generate("TEST"); + String id = idOptional.isPresent() ? idOptional.get().getId() : ""; Assertions.assertEquals(18, id.length()); } @@ -186,7 +185,7 @@ void testParseSuccess() { @Test void testParseSuccessAfterGeneration() { - val generatedId = partitionAwareIdGenerator.generate("TEST123"); + val generatedId = partitionAwareIdGenerator.generate("TEST123").get(); val parsedId = partitionAwareIdGenerator.parse(generatedId.getId()).orElse(null); Assertions.assertNotNull(parsedId); Assertions.assertEquals(parsedId.getId(), generatedId.getId()); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java index dde9dbb2..fd8ddfe4 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java @@ -46,7 +46,7 @@ public void setUp() throws IOException { .build(); weightedIdGenerator = new WeightedIdGenerator( 1024, partitionResolverSupplier, - IdGeneratorRetryConfig.builder().idGenerationRetryCount(100).partitionRetryCount(100).build(), + IdGeneratorRetryConfig.builder().idGenerationRetryCount(1024).partitionRetryCount(1024).build(), weightedIdConfig ); } diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java index 759eeec0..34b40db6 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java @@ -32,6 +32,7 @@ class WeightedIdGeneratorTest { final int partitionCount = 1024; final Function partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length() - 6)) % partitionCount; + final IdGeneratorRetryConfig retryConfig = IdGeneratorRetryConfig.builder().idGenerationRetryCount(4096).partitionRetryCount(4096).build(); private WeightedIdGenerator weightedIdGenerator; private WeightedIdConfig weightedIdConfig; @@ -47,11 +48,7 @@ void setup() { weightedIdConfig = WeightedIdConfig.builder() .partitions(partitionConfigList) .build(); - weightedIdGenerator = new WeightedIdGenerator( - partitionCount, partitionResolverSupplier, - IdGeneratorRetryConfig.builder().idGenerationRetryCount(100).partitionRetryCount(100).build(), - weightedIdConfig - ); + weightedIdGenerator = new WeightedIdGenerator(partitionCount, partitionResolverSupplier, retryConfig, weightedIdConfig); } @Test @@ -118,13 +115,10 @@ void testUniqueIds(Map> dataStore) { @Test void testGenerateOriginal() { - weightedIdGenerator = new WeightedIdGenerator( - partitionCount, partitionResolverSupplier, - IdGeneratorRetryConfig.builder().idGenerationRetryCount(100).partitionRetryCount(100).build(), - weightedIdConfig, - IdFormatters.original() + weightedIdGenerator = new WeightedIdGenerator(partitionCount, partitionResolverSupplier, retryConfig, + weightedIdConfig, IdFormatters.original() ); - String id = weightedIdGenerator.generate("TEST").getId(); + String id = weightedIdGenerator.generate("TEST").get().getId(); Assertions.assertEquals(26, id.length()); } @@ -133,11 +127,9 @@ void testGenerateBase36() { weightedIdGenerator = new WeightedIdGenerator( partitionCount, (txnId) -> new BigInteger(txnId.substring(txnId.length() - 6), 36).abs().intValue() % partitionCount, - IdGeneratorRetryConfig.builder().idGenerationRetryCount(100).partitionRetryCount(100).build(), - weightedIdConfig, - IdFormatters.base36() + retryConfig, weightedIdConfig, IdFormatters.base36() ); - String id = weightedIdGenerator.generate("TEST").getId(); + String id = weightedIdGenerator.generate("TEST").get().getId(); Assertions.assertEquals(18, id.length()); } @@ -188,7 +180,7 @@ void testParseSuccess() { @Test void testParseSuccessAfterGeneration() { - val generatedId = weightedIdGenerator.generate("TEST123"); + val generatedId = weightedIdGenerator.generate("TEST123").get(); val parsedId = weightedIdGenerator.parse(generatedId.getId()).orElse(null); Assertions.assertNotNull(parsedId); Assertions.assertEquals(parsedId.getId(), generatedId.getId()); From f269db41db46f046c4fcf37bd60c527983b63f90 Mon Sep 17 00:00:00 2001 From: Satyam Saxena Date: Mon, 10 Jun 2024 13:39:29 +0530 Subject: [PATCH 09/12] Add ToDos --- .../discovery/bundle/id/config/WeightedIdConfig.java | 1 + .../bundle/id/weighted/DistributedIdGenerator.java | 6 ++++++ .../ranger/discovery/bundle/id/weighted/IdPool.java | 6 ++++-- .../discovery/bundle/id/weighted/WeightedIdGenerator.java | 8 +++++--- .../bundle/id/PartitionAwareIdGeneratorTest.java | 1 + 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/WeightedIdConfig.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/WeightedIdConfig.java index 8f97a7cc..7736036f 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/WeightedIdConfig.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/WeightedIdConfig.java @@ -35,5 +35,6 @@ public boolean isPartitionWeightsValid() { } } return true; +// ToDo: Add check for inclusion of all partitions } } \ No newline at end of file diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java index 5c4a6bd8..8c977ecd 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java @@ -74,6 +74,7 @@ abstract class DistributedIdGenerator { } } */ +// ToDo: Reuse timestamp keys by using them as index arrays and having an additional pointer per IdList to point to the index for next write. protected DistributedIdGenerator(final int partitionCount, final Function partitionResolverSupplier, @@ -141,6 +142,7 @@ public Optional generate(final String prefix) { public Optional generateForPartition(final String prefix, final int targetPartitionId) { val prefixIdMap = idStore.computeIfAbsent(prefix, k -> new ConcurrentHashMap<>()); val currentTimestamp = new DateTime(); +// ToDo: Use clock instead of datetime val timeKey = currentTimestamp.getMillis() / 1000; val idCounter = generateForAllPartitions( prefixIdMap.computeIfAbsent(timeKey, key -> new PartitionIdTracker(partitionCount)), @@ -152,6 +154,7 @@ public Optional generateForPartition(final String prefix, final int targetPa return Optional.of( Id.builder() .id(id) +// ToDo: Check if exponent is needed and its usage. .exponent(idCounter.get()) .generatedDate(currentTimestamp.toDate()) .node(NODE_ID) @@ -167,17 +170,20 @@ private Optional generateForAllPartitions(final PartitionIdTracker part final int targetPartitionId) { val idPool = partitionIdTracker.getPartition(targetPartitionId); int idIdx = idPool.getPointer().getAndIncrement(); +// ToDo: revisit retry logic int retry = 0; while (idPool.getIdList().size() <= idIdx && retry < retryConfig.getIdGenerationRetryCount()) { val counterValue = partitionIdTracker.getNextIdCounter().getAndIncrement(); val txnId = String.format("%s%s", prefix, idFormatter.format(timestamp, NODE_ID, counterValue)); val mappedPartitionId = partitionResolver.apply(txnId); +// ToDo: Move id addition operation to partitionidtracker partitionIdTracker.getPartition(mappedPartitionId).getIdList().add(counterValue); retry += 1; } if (idIdx < idPool.getIdList().size()) { return Optional.of(idPool.getId(idIdx)); } else { +// ToDo: Add metrics for retry count limit exceeded per prefix. log.warn("Retry Limit reached - {} - {} - {}", retry, idIdx, targetPartitionId); return Optional.empty(); } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/IdPool.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/IdPool.java index 753656d9..b8828887 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/IdPool.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/IdPool.java @@ -8,10 +8,12 @@ @Getter public class IdPool { - // List of IDs for the specific IdPool +// List of IDs for the specific IdPool +// ToDo: Check for a better DS +// ToDo: Check for static sized circular array with additional pointer. private final List idList = new CopyOnWriteArrayList<>(); - // Pointer to track the index of the next usable ID +// Pointer to track the index of the next usable ID private final AtomicInteger pointer = new AtomicInteger(); public int getId(int index) { diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/WeightedIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/WeightedIdGenerator.java index 5769e569..21c27e21 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/WeightedIdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/WeightedIdGenerator.java @@ -44,10 +44,10 @@ public WeightedIdGenerator(final int partitionSize, private RangeMap createWeightRangeMap(final WeightedIdConfig weightedIdConfig) { RangeMap partitionGroups = TreeRangeMap.create(); int end = -1; - for (val shard : weightedIdConfig.getPartitions()) { + for (val partition : weightedIdConfig.getPartitions()) { int start = end + 1; - end += shard.getWeight(); - partitionGroups.put(Range.closed(start, end), shard.getPartitionRange()); + end += partition.getWeight(); + partitionGroups.put(Range.closed(start, end), partition.getPartitionRange()); } maxShardWeight = end; return partitionGroups; @@ -55,9 +55,11 @@ private RangeMap createWeightRangeMap(final WeightedIdC @Override protected int getTargetPartitionId() { +// ToDo: Check for randomness of all partitions being picked equally. Check for test case for same. val randomNum = SECURE_RANDOM.nextInt(maxShardWeight); val partitionRange = Objects.requireNonNull(partitionRangeMap.getEntry(randomNum)).getValue(); return SECURE_RANDOM.nextInt(partitionRange.getEnd() - partitionRange.getStart() + 1) + partitionRange.getStart(); +// ToDo: Check if every DS is threadsafe. } @Override diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java index ddb40b22..c79f9b35 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java @@ -36,6 +36,7 @@ void setup() { partitionCount, partitionResolverSupplier, retryConfig ); } +// ToDo: Add test for partition distribution spread. @Test void testGenerateWithBenchmark() throws IOException { From 9dbb85d173955fb63c6c5926b2b4e758164db4d2 Mon Sep 17 00:00:00 2001 From: Satyam Saxena Date: Thu, 13 Jun 2024 14:49:03 +0530 Subject: [PATCH 10/12] Update ID Generation Classes and Structure --- ...neratorTest.testGenerateWithBenchmark.json | 4 +- ...ratorTest.testGenerateWithConstraints.json | 4 +- ...neratorTest.testGenerateWithBenchmark.json | 4 +- ...ratorTest.testGenerateWithConstraints.json | 4 +- .../ranger/discovery/bundle/id/Constants.java | 2 + .../bundle/id/config/IdGeneratorConfig.java | 56 ++++++ .../bundle/id/config/WeightedIdConfig.java | 1 - .../formatter/PartitionAwareIdFormatter.java | 2 + .../id/weighted/DistributedIdGenerator.java | 88 +++++---- .../discovery/bundle/id/weighted/IdPool.java | 44 ++++- .../weighted/PartitionAwareIdGenerator.java | 28 +-- .../id/weighted/PartitionIdTracker.java | 31 ++-- .../id/weighted/WeightedIdGenerator.java | 34 ++-- .../id/PartitionAwareIdGeneratorPerfTest.java | 12 +- .../id/PartitionAwareIdGeneratorTest.java | 127 ++++++------- .../id/WeightedIdGeneratorPerfTest.java | 14 +- .../bundle/id/WeightedIdGeneratorTest.java | 168 +++++++++++------- 17 files changed, 404 insertions(+), 219 deletions(-) create mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/IdGeneratorConfig.java diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark.json index a06bfe0f..0e37ed32 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark.json @@ -2,6 +2,6 @@ "name" : "io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark", "iterations" : 100000, "threads" : 5, - "totalMillis" : 6411, - "avgTime" : 1282.2 + "totalMillis" : 2313, + "avgTime" : 462.6 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints.json index 6f814e59..b9d45d70 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints.json @@ -2,6 +2,6 @@ "name" : "io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints", "iterations" : 100000, "threads" : 5, - "totalMillis" : 5189, - "avgTime" : 1037.8 + "totalMillis" : 7745, + "avgTime" : 1549.0 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json index d5cea3d2..d296550a 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json @@ -2,6 +2,6 @@ "name" : "io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark", "iterations" : 100000, "threads" : 5, - "totalMillis" : 4128, - "avgTime" : 825.6 + "totalMillis" : 31489, + "avgTime" : 6297.8 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json index dccc61e1..4e8fd6d9 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json @@ -2,6 +2,6 @@ "name" : "io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints", "iterations" : 100000, "threads" : 5, - "totalMillis" : 4820, - "avgTime" : 964.0 + "totalMillis" : 105139, + "avgTime" : 21027.8 } \ No newline at end of file diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java index e70cb10a..da1ce220 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java @@ -25,6 +25,8 @@ public class Constants { public static final int MAX_ID_PER_MS = 1000; public static final int MAX_NUM_NODES = 10000; + public static final int MAX_DATA_STORAGE_TIME_LIMIT_IN_SECONDS = 300; + public static final int MAX_IDS_PER_SECOND = 1_000_000; public static final int ID_DELETION_DELAY_IN_SECONDS = 60; public static final int DELETION_THRESHOLD_IN_SECONDS = 60; } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/IdGeneratorConfig.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/IdGeneratorConfig.java new file mode 100644 index 00000000..9f3ea52b --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/IdGeneratorConfig.java @@ -0,0 +1,56 @@ +package io.appform.ranger.discovery.bundle.id.config; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import io.dropwizard.validation.ValidationMethod; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.validation.Valid; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import static io.appform.ranger.discovery.bundle.id.Constants.MAX_DATA_STORAGE_TIME_LIMIT_IN_SECONDS; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class IdGeneratorConfig { + @NotEmpty + private IdGeneratorRetryConfig retryConfig; + + @Valid + private WeightedIdConfig weightedIdConfig; + + @NotNull + @Min(1) + private int idPoolSize; + + @NotNull + @Min(1) + private int partitionCount; + + @Min(1) + @Builder.Default + private int maxDataBufferTimeInSeconds = MAX_DATA_STORAGE_TIME_LIMIT_IN_SECONDS; + + @ValidationMethod(message = "Invalid Partition Range") + @JsonIgnore + public boolean isPartitionCountValid() { + if (weightedIdConfig != null) { + List sortedPartitions = new ArrayList<>(weightedIdConfig.getPartitions()); + sortedPartitions.sort(Comparator.comparingInt(k -> k.getPartitionRange().getStart())); + if (sortedPartitions.get(sortedPartitions.size()-1).getPartitionRange().getEnd() - sortedPartitions.get(sortedPartitions.size()-1).getPartitionRange().getStart() != partitionCount) { + return false; + } + } + return true; + } + +} \ No newline at end of file diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/WeightedIdConfig.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/WeightedIdConfig.java index 7736036f..8f97a7cc 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/WeightedIdConfig.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/WeightedIdConfig.java @@ -35,6 +35,5 @@ public boolean isPartitionWeightsValid() { } } return true; -// ToDo: Add check for inclusion of all partitions } } \ No newline at end of file diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/PartitionAwareIdFormatter.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/PartitionAwareIdFormatter.java index 9bdbd6e0..6f856e95 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/PartitionAwareIdFormatter.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/PartitionAwareIdFormatter.java @@ -1,9 +1,11 @@ package io.appform.ranger.discovery.bundle.id.formatter; +import lombok.extern.slf4j.Slf4j; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; +@Slf4j public class PartitionAwareIdFormatter implements IdFormatter { private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormat.forPattern("yyMMddHHmmss"); diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java index 8c977ecd..f35bae92 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java @@ -1,5 +1,7 @@ package io.appform.ranger.discovery.bundle.id.weighted; +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import dev.failsafe.Failsafe; @@ -8,7 +10,7 @@ import io.appform.ranger.discovery.bundle.id.Constants; import io.appform.ranger.discovery.bundle.id.Id; import io.appform.ranger.discovery.bundle.id.IdGenerator; -import io.appform.ranger.discovery.bundle.id.config.IdGeneratorRetryConfig; +import io.appform.ranger.discovery.bundle.id.config.IdGeneratorConfig; import io.appform.ranger.discovery.bundle.id.constraints.PartitionValidationConstraint; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatter; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; @@ -20,6 +22,7 @@ import org.joda.time.format.DateTimeFormatter; import java.security.SecureRandom; +import java.time.Clock; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -52,8 +55,9 @@ abstract class DistributedIdGenerator { private final Map> idStore = new ConcurrentHashMap<>(); protected final IdFormatter idFormatter; protected final Function partitionResolver; - protected final IdGeneratorRetryConfig retryConfig; - protected final int partitionCount; + protected final IdGeneratorConfig idGeneratorConfig; + private final Clock clock = Clock.systemDefaultZone(); + private final Meter retryLimitBreachedMeter; /* idStore Structure { @@ -74,18 +78,17 @@ abstract class DistributedIdGenerator { } } */ -// ToDo: Reuse timestamp keys by using them as index arrays and having an additional pointer per IdList to point to the index for next write. - protected DistributedIdGenerator(final int partitionCount, + protected DistributedIdGenerator(final IdGeneratorConfig idGeneratorConfig, final Function partitionResolverSupplier, - final IdGeneratorRetryConfig retryConfig, - final IdFormatter idFormatterInstance) { - this.partitionCount = partitionCount; - this.retryConfig = retryConfig; + final IdFormatter idFormatterInstance, + final MetricRegistry metricRegistry) { + this.idGeneratorConfig = idGeneratorConfig; this.partitionResolver = partitionResolverSupplier; this.idFormatter = idFormatterInstance; + this.retryLimitBreachedMeter = metricRegistry.meter("idGenerator.RetryLimitBreached"); RetryPolicy retryPolicy = RetryPolicy.builder() - .withMaxAttempts(retryConfig.getPartitionRetryCount()) + .withMaxAttempts(idGeneratorConfig.getRetryConfig().getPartitionRetryCount()) .handleIf(throwable -> true) .handleResultIf(Objects::isNull) .build(); @@ -99,10 +102,10 @@ protected DistributedIdGenerator(final int partitionCount, TimeUnit.SECONDS); } - protected DistributedIdGenerator(final int partitionCount, + protected DistributedIdGenerator(final IdGeneratorConfig idGeneratorConfig, final Function partitionResolverSupplier, - final IdGeneratorRetryConfig retryConfig) { - this(partitionCount, partitionResolverSupplier, retryConfig, IdFormatters.partitionAware()); + final MetricRegistry metricRegistry) { + this(idGeneratorConfig, partitionResolverSupplier, IdFormatters.partitionAware(), metricRegistry); } public synchronized void registerGlobalConstraints(final PartitionValidationConstraint... constraints) { @@ -142,10 +145,10 @@ public Optional generate(final String prefix) { public Optional generateForPartition(final String prefix, final int targetPartitionId) { val prefixIdMap = idStore.computeIfAbsent(prefix, k -> new ConcurrentHashMap<>()); val currentTimestamp = new DateTime(); -// ToDo: Use clock instead of datetime - val timeKey = currentTimestamp.getMillis() / 1000; + val timeKey = getTimeKey(clock.instant().getEpochSecond()); val idCounter = generateForAllPartitions( - prefixIdMap.computeIfAbsent(timeKey, key -> new PartitionIdTracker(partitionCount)), + prefixIdMap.computeIfAbsent(timeKey, key -> new PartitionIdTracker(idGeneratorConfig.getPartitionCount(), + idGeneratorConfig.getIdPoolSize())), prefix, currentTimestamp, targetPartitionId); @@ -154,7 +157,6 @@ public Optional generateForPartition(final String prefix, final int targetPa return Optional.of( Id.builder() .id(id) -// ToDo: Check if exponent is needed and its usage. .exponent(idCounter.get()) .generatedDate(currentTimestamp.toDate()) .node(NODE_ID) @@ -169,22 +171,25 @@ private Optional generateForAllPartitions(final PartitionIdTracker part final DateTime timestamp, final int targetPartitionId) { val idPool = partitionIdTracker.getPartition(targetPartitionId); - int idIdx = idPool.getPointer().getAndIncrement(); -// ToDo: revisit retry logic - int retry = 0; - while (idPool.getIdList().size() <= idIdx && retry < retryConfig.getIdGenerationRetryCount()) { - val counterValue = partitionIdTracker.getNextIdCounter().getAndIncrement(); - val txnId = String.format("%s%s", prefix, idFormatter.format(timestamp, NODE_ID, counterValue)); - val mappedPartitionId = partitionResolver.apply(txnId); -// ToDo: Move id addition operation to partitionidtracker - partitionIdTracker.getPartition(mappedPartitionId).getIdList().add(counterValue); - retry += 1; - } - if (idIdx < idPool.getIdList().size()) { - return Optional.of(idPool.getId(idIdx)); - } else { -// ToDo: Add metrics for retry count limit exceeded per prefix. - log.warn("Retry Limit reached - {} - {} - {}", retry, idIdx, targetPartitionId); + int idIdx = idPool.getUsableIdIndex(); + int retryCount = 0; + try { + while (!idPool.isIdPresentAtIndex(idIdx) && retryCount < idGeneratorConfig.getRetryConfig().getIdGenerationRetryCount()) { + val counterValue = partitionIdTracker.getIdCounter(); + val txnId = String.format("%s%s", prefix, idFormatter.format(timestamp, NODE_ID, counterValue)); + val mappedPartitionId = partitionResolver.apply(txnId); + partitionIdTracker.addId(mappedPartitionId, counterValue); + retryCount += 1; + } + if (idPool.isIdPresentAtIndex(idIdx)) { + return Optional.of(idPool.getId(idIdx)); + } else { + retryLimitBreachedMeter.mark(); + log.warn("Retry Limit reached - {} - {} - {}", retryCount, idIdx, targetPartitionId); + return Optional.empty(); + } + } catch (Exception e) { + log.error("Error while generating IDs", e); return Optional.empty(); } } @@ -214,14 +219,20 @@ public Optional generateWithConstraints(final String prefix, final String do */ public Optional generateWithConstraints(final String prefix, final String domain, final boolean skipGlobal) { val targetPartitionId = getTargetPartitionId(DOMAIN_SPECIFIC_CONSTRAINTS.getOrDefault(domain, Collections.emptyList()), skipGlobal); - return targetPartitionId.map(id -> generateForPartition(prefix, id).get()); + return targetPartitionId.map(partitionId -> { + val id = generateForPartition(prefix, partitionId); + return id.orElse(null); + }); } public Optional generateWithConstraints(final String prefix, final List inConstraints, final boolean skipGlobal) { val targetPartitionId = getTargetPartitionId(inConstraints, skipGlobal); - return targetPartitionId.map(id -> generateForPartition(prefix, id).get()); + return targetPartitionId.map(partitionId -> { + val id = generateForPartition(prefix, partitionId); + return id.orElse(null); + }); } /** @@ -282,8 +293,13 @@ protected boolean validateId(final List inConstra return null == failedLocalConstraint; } + private long getTimeKey(long timeInSeconds) { +// log.warn("{}:{}", timeInSeconds, timeInSeconds % idGeneratorConfig.getMaxDataBufferTimeInSeconds()); + return timeInSeconds % idGeneratorConfig.getMaxDataBufferTimeInSeconds(); + } + private synchronized void deleteExpiredKeys() { - val timeThreshold = DateTime.now().getMillis() / 1000 - Constants.DELETION_THRESHOLD_IN_SECONDS; + val timeThreshold = getTimeKey(clock.instant().getEpochSecond()) - Constants.DELETION_THRESHOLD_IN_SECONDS; for (val entry : idStore.entrySet()) { entry.getValue().entrySet().removeIf(partitionIdTrackerEntry -> partitionIdTrackerEntry.getKey() < timeThreshold); } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/IdPool.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/IdPool.java index b8828887..89dad9d0 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/IdPool.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/IdPool.java @@ -1,22 +1,48 @@ package io.appform.ranger.discovery.bundle.id.weighted; -import lombok.Getter; +import lombok.val; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; -@Getter public class IdPool { // List of IDs for the specific IdPool -// ToDo: Check for a better DS -// ToDo: Check for static sized circular array with additional pointer. - private final List idList = new CopyOnWriteArrayList<>(); + private final AtomicIntegerArray idList; // Pointer to track the index of the next usable ID - private final AtomicInteger pointer = new AtomicInteger(); + private final AtomicInteger nextUsableIdx = new AtomicInteger(); + +// Pointer to track index of last usable ID. Helps to determine which index the next ID should go to. + private final AtomicInteger lastUsableIdx = new AtomicInteger(); + +// Size of the IdPool + private final int capacity; + + public IdPool(int size) { + this.capacity = size; + idList = new AtomicIntegerArray(capacity); + } + + public boolean isIdPresentAtIndex(int index) { + return lastUsableIdx.get() > index; + } public int getId(int index) { - return idList.get(index); + val arrayIdx = index % capacity; + return idList.get(arrayIdx); + } + + public void setId(int id) { +// Don't set new IDs if the idPool is already full of unused IDs. + if (lastUsableIdx.get() >= nextUsableIdx.get() + capacity) { + return; + } + val arrayIdx = lastUsableIdx.get() % capacity; + idList.set(arrayIdx, id); + lastUsableIdx.incrementAndGet(); + } + + public int getUsableIdIndex() { + return nextUsableIdx.getAndIncrement(); } } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionAwareIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionAwareIdGenerator.java index 12cb4b29..4fa04446 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionAwareIdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionAwareIdGenerator.java @@ -1,10 +1,12 @@ package io.appform.ranger.discovery.bundle.id.weighted; -import io.appform.ranger.discovery.bundle.id.config.IdGeneratorRetryConfig; +import com.codahale.metrics.MetricRegistry; +import io.appform.ranger.discovery.bundle.id.config.IdGeneratorConfig; import io.appform.ranger.discovery.bundle.id.constraints.PartitionValidationConstraint; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatter; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; import lombok.extern.slf4j.Slf4j; +import lombok.val; import java.util.List; import java.util.Optional; @@ -16,29 +18,31 @@ @SuppressWarnings("unused") @Slf4j public class PartitionAwareIdGenerator extends DistributedIdGenerator { - public PartitionAwareIdGenerator(final int partitionCount, + public PartitionAwareIdGenerator(final IdGeneratorConfig idGeneratorConfig, final Function partitionResolverSupplier, - final IdGeneratorRetryConfig retryConfig, - final IdFormatter idFormatterInstance) { - super(partitionCount, partitionResolverSupplier, retryConfig, idFormatterInstance); + final IdFormatter idFormatterInstance, + final MetricRegistry metricRegistry) { + super(idGeneratorConfig, partitionResolverSupplier, idFormatterInstance, metricRegistry); } - public PartitionAwareIdGenerator(final int partitionCount, - final Function partitionResolverSupplier, - final IdGeneratorRetryConfig retryConfig) { - super(partitionCount, partitionResolverSupplier, retryConfig, IdFormatters.partitionAware()); + public PartitionAwareIdGenerator(final IdGeneratorConfig idGeneratorConfig, final Function partitionResolverSupplier, + final MetricRegistry metricRegistry) { + super(idGeneratorConfig, partitionResolverSupplier, IdFormatters.partitionAware(), metricRegistry); } @Override protected int getTargetPartitionId() { - return SECURE_RANDOM.nextInt(partitionCount); + return SECURE_RANDOM.nextInt(idGeneratorConfig.getPartitionCount()); } @Override protected Optional getTargetPartitionId(final List inConstraints, final boolean skipGlobal) { return Optional.ofNullable( - retrier.get(() -> SECURE_RANDOM.nextInt(partitionCount))) - .filter(key -> validateId(inConstraints, key, skipGlobal)); + retrier.get(() -> { + val partitionId = SECURE_RANDOM.nextInt(idGeneratorConfig.getPartitionCount()); + return validateId(inConstraints, partitionId, skipGlobal) ? partitionId : null; + }) + ); } } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionIdTracker.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionIdTracker.java index 125e8c4b..9fced327 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionIdTracker.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionIdTracker.java @@ -1,33 +1,44 @@ package io.appform.ranger.discovery.bundle.id.weighted; -import lombok.Getter; - import java.util.concurrent.atomic.AtomicInteger; -@Getter +import static io.appform.ranger.discovery.bundle.id.Constants.MAX_IDS_PER_SECOND; + public class PartitionIdTracker { // Array to store IdPools for each partition private final IdPool[] idPoolList; - // Size of each partition - private final int partitionSize; - // Counter to keep track of the number of IDs created private final AtomicInteger nextIdCounter = new AtomicInteger(); - protected PartitionIdTracker(int partitionSize) { - this.partitionSize = partitionSize; + protected PartitionIdTracker(int partitionSize, int idPoolSize) { idPoolList = new IdPool[partitionSize]; for (int i=0; i partitionRangeMap; - public WeightedIdGenerator(final int partitionSize, + public WeightedIdGenerator(final IdGeneratorConfig idGeneratorConfig, final Function partitionResolverSupplier, - final IdGeneratorRetryConfig retryConfig, - final WeightedIdConfig weightedIdConfig) { - super(partitionSize, partitionResolverSupplier, retryConfig); - partitionRangeMap = createWeightRangeMap(weightedIdConfig); + final MetricRegistry metricRegistry) { +// Preconditions.checkNotNull() + super(idGeneratorConfig, partitionResolverSupplier, metricRegistry); + partitionRangeMap = createWeightRangeMap(idGeneratorConfig.getWeightedIdConfig()); } - public WeightedIdGenerator(final int partitionSize, + public WeightedIdGenerator(final IdGeneratorConfig idGeneratorConfig, final Function partitionResolverSupplier, - final IdGeneratorRetryConfig retryConfig, - final WeightedIdConfig weightedIdConfig, - final IdFormatter idFormatterInstance) { - super(partitionSize, partitionResolverSupplier, retryConfig, idFormatterInstance); - partitionRangeMap = createWeightRangeMap(weightedIdConfig); + final IdFormatter idFormatterInstance, + final MetricRegistry metricRegistry) { + super(idGeneratorConfig, partitionResolverSupplier, idFormatterInstance, metricRegistry); + partitionRangeMap = createWeightRangeMap(idGeneratorConfig.getWeightedIdConfig()); } private RangeMap createWeightRangeMap(final WeightedIdConfig weightedIdConfig) { @@ -55,18 +57,18 @@ private RangeMap createWeightRangeMap(final WeightedIdC @Override protected int getTargetPartitionId() { -// ToDo: Check for randomness of all partitions being picked equally. Check for test case for same. val randomNum = SECURE_RANDOM.nextInt(maxShardWeight); val partitionRange = Objects.requireNonNull(partitionRangeMap.getEntry(randomNum)).getValue(); return SECURE_RANDOM.nextInt(partitionRange.getEnd() - partitionRange.getStart() + 1) + partitionRange.getStart(); -// ToDo: Check if every DS is threadsafe. } @Override protected Optional getTargetPartitionId(final List inConstraints, final boolean skipGlobal) { return Optional.ofNullable( - retrier.get(this::getTargetPartitionId)) - .filter(key -> validateId(inConstraints, key, skipGlobal)); + retrier.get(() -> { + val partitionId = getTargetPartitionId(); + return validateId(inConstraints, partitionId, skipGlobal) ? partitionId : null; + })); } } diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java index 11762482..cdad5279 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java @@ -1,5 +1,7 @@ package io.appform.ranger.discovery.bundle.id; +import com.codahale.metrics.MetricRegistry; +import io.appform.ranger.discovery.bundle.id.config.IdGeneratorConfig; import io.appform.ranger.discovery.bundle.id.config.IdGeneratorRetryConfig; import io.appform.ranger.discovery.bundle.id.weighted.PartitionAwareIdGenerator; import lombok.SneakyThrows; @@ -14,6 +16,8 @@ import java.io.IOException; import java.util.function.Function; +import static org.mockito.Mockito.mock; + /** * Test performance between different constructs */ @@ -28,8 +32,12 @@ public static class BenchmarkState { @Setup(Level.Trial) public void setUp() throws IOException { partitionAwareIdGenerator = new PartitionAwareIdGenerator( - 1024, partitionResolverSupplier, - IdGeneratorRetryConfig.builder().idGenerationRetryCount(1024).partitionRetryCount(1024).build() + IdGeneratorConfig.builder() + .partitionCount(1024) + .retryConfig(IdGeneratorRetryConfig.builder().idGenerationRetryCount(1024).partitionRetryCount(1024).build()) + .build(), + partitionResolverSupplier, + mock(MetricRegistry.class) ); } } diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java index c79f9b35..108a8c73 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java @@ -1,10 +1,12 @@ package io.appform.ranger.discovery.bundle.id; +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; +import io.appform.ranger.discovery.bundle.id.config.IdGeneratorConfig; import io.appform.ranger.discovery.bundle.id.config.IdGeneratorRetryConfig; import io.appform.ranger.discovery.bundle.id.constraints.PartitionValidationConstraint; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; import io.appform.ranger.discovery.bundle.id.weighted.PartitionAwareIdGenerator; -import io.appform.ranger.discovery.bundle.id.weighted.PartitionIdTracker; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.junit.jupiter.api.Assertions; @@ -14,98 +16,98 @@ import java.io.IOException; import java.math.BigInteger; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; -import java.util.Map; +import java.util.List; import java.util.function.Function; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + /** * Test for {@link PartitionAwareIdGenerator} */ @Slf4j @SuppressWarnings({"unused", "FieldMayBeFinal"}) class PartitionAwareIdGeneratorTest { + final int numThreads = 5; + final int iterationCount = 100000; final int partitionCount = 1024; final Function partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length() - 6)) % partitionCount; - final IdGeneratorRetryConfig retryConfig = IdGeneratorRetryConfig.builder().idGenerationRetryCount(4096).partitionRetryCount(4096).build(); + final IdGeneratorConfig idGeneratorConfig = + IdGeneratorConfig.builder() + .partitionCount(partitionCount) + .idPoolSize(100) + .retryConfig(IdGeneratorRetryConfig.builder().idGenerationRetryCount(4096).partitionRetryCount(4096).build()) + .build(); private PartitionAwareIdGenerator partitionAwareIdGenerator; @BeforeEach void setup() { + val metricRegistry = mock(MetricRegistry.class); + val meter = mock(Meter.class); + doReturn(meter).when(metricRegistry).meter(anyString()); + doNothing().when(meter).mark(); partitionAwareIdGenerator = new PartitionAwareIdGenerator( - partitionCount, partitionResolverSupplier, retryConfig + idGeneratorConfig, partitionResolverSupplier, metricRegistry ); } // ToDo: Add test for partition distribution spread. @Test void testGenerateWithBenchmark() throws IOException { - val totalTime = TestUtil.runMTTest(5, 100000, (k) -> partitionAwareIdGenerator.generate("P"), this.getClass().getName() + ".testGenerateWithBenchmark"); - testUniqueIdsInDataStore(partitionAwareIdGenerator.getIdStore()); + val allIdsList = Collections.synchronizedList(new ArrayList()); + val totalTime = TestUtil.runMTTest( + numThreads, + iterationCount, + (k) -> { + val id = partitionAwareIdGenerator.generate("P"); + id.ifPresent(value -> allIdsList.add(value.getId())); + }, + this.getClass().getName() + ".testGenerateWithBenchmark"); + Assertions.assertEquals(numThreads * iterationCount, allIdsList.size()); + checkUniqueIds(allIdsList); } @Test void testGenerateWithConstraints() throws IOException { + val allIdsList = Collections.synchronizedList(new ArrayList()); PartitionValidationConstraint partitionConstraint = (k) -> k % 2 == 0; partitionAwareIdGenerator.registerGlobalConstraints(partitionConstraint); - val totalTime = TestUtil.runMTTest(5, 100000, (k) -> partitionAwareIdGenerator.generateWithConstraints("P", (String) null, false), this.getClass().getName() + ".testGenerateWithConstraints"); - testUniqueIdsInDataStore(partitionAwareIdGenerator.getIdStore()); - - for (Map.Entry> entry : partitionAwareIdGenerator.getIdStore().entrySet()) { - val prefix = entry.getKey(); - val prefixIds = entry.getValue(); - HashSet uniqueIds = new HashSet<>(); - for (Map.Entry prefixEntry : prefixIds.entrySet()) { - val key = prefixEntry.getKey(); - val partitionIdTracker = prefixEntry.getValue(); - for (int idx = 0; idx < partitionIdTracker.getPartitionSize(); idx += 1) { - if (!partitionConstraint.isValid(idx)) { - Assertions.assertEquals(0, partitionIdTracker.getIdPoolList()[idx].getPointer().get()); - } - } - } + val totalTime = TestUtil.runMTTest( + numThreads, + iterationCount, + (k) -> { + val id = partitionAwareIdGenerator.generateWithConstraints("P", (String) null, false); + id.ifPresent(value -> allIdsList.add(value.getId())); + }, + this.getClass().getName() + ".testGenerateWithConstraints"); + + Assertions.assertEquals(numThreads * iterationCount, allIdsList.size()); + checkUniqueIds(allIdsList); + +// Assert No ID was generated for Invalid partitions + for (val id: allIdsList) { + val partitionId = partitionResolverSupplier.apply(id); + Assertions.assertTrue(partitionConstraint.isValid(partitionId)); } } - void testUniqueIdsInDataStore(Map> dataStore) { - boolean allIdsUnique = true; - for (Map.Entry> entry : dataStore.entrySet()) { - val prefix = entry.getKey(); - val prefixIds = entry.getValue(); - for (Map.Entry prefixEntry : prefixIds.entrySet()) { - val key = prefixEntry.getKey(); - val partitionIdTracker = prefixEntry.getValue(); - HashSet uniqueIds = new HashSet<>(); - for (val idPool : partitionIdTracker.getIdPoolList()) { - boolean allIdsUniqueInList = true; - HashSet uniqueIdsInList = new HashSet<>(); - for (val id : idPool.getIdList()) { - if (uniqueIdsInList.contains(id)) { - allIdsUniqueInList = false; - allIdsUnique = false; - } else { - uniqueIdsInList.add(id); - } - - if (uniqueIds.contains(id)) { - allIdsUnique = false; - } else { - uniqueIds.add(id); - } - } - Assertions.assertTrue(allIdsUniqueInList); - } - } - } - Assertions.assertTrue(allIdsUnique); + void checkUniqueIds(List allIdsList) { + HashSet uniqueIds = new HashSet<>(allIdsList); + Assertions.assertEquals(allIdsList.size(), uniqueIds.size()); } @Test void testUniqueIds() { HashSet allIDs = new HashSet<>(); boolean allIdsUnique = true; - for (int i = 0; i < 10000; i += 1) { - val txnId = partitionAwareIdGenerator.generate("P").get().getId(); + for (int i = 0; i < iterationCount; i += 1) { + val txnIdOptional = partitionAwareIdGenerator.generate("P"); + val txnId = txnIdOptional.map(Id::getId).orElse(null); if (allIDs.contains(txnId)) { log.warn(txnId); log.warn(String.valueOf(allIDs)); @@ -120,19 +122,20 @@ void testUniqueIds() { @Test void testGenerateOriginal() { partitionAwareIdGenerator = new PartitionAwareIdGenerator( - partitionCount, partitionResolverSupplier, retryConfig, IdFormatters.original() + idGeneratorConfig, partitionResolverSupplier, mock(MetricRegistry.class) ); - String id = partitionAwareIdGenerator.generate("TEST").get().getId(); + val idOptional = partitionAwareIdGenerator.generate("TEST"); + String id = idOptional.isPresent() ? idOptional.get().getId() : ""; Assertions.assertEquals(26, id.length()); } @Test void testGenerateBase36() { partitionAwareIdGenerator = new PartitionAwareIdGenerator( - partitionCount, + idGeneratorConfig, (txnId) -> new BigInteger(txnId.substring(txnId.length() - 6), 36).abs().intValue() % partitionCount, - retryConfig, - IdFormatters.base36() + IdFormatters.base36(), + mock(MetricRegistry.class) ); val idOptional = partitionAwareIdGenerator.generate("TEST"); String id = idOptional.isPresent() ? idOptional.get().getId() : ""; @@ -186,7 +189,9 @@ void testParseSuccess() { @Test void testParseSuccessAfterGeneration() { - val generatedId = partitionAwareIdGenerator.generate("TEST123").get(); + val generatedIdOptional = partitionAwareIdGenerator.generate("TEST123"); + val generatedId = generatedIdOptional.orElse(null); + Assertions.assertNotNull(generatedId); val parsedId = partitionAwareIdGenerator.parse(generatedId.getId()).orElse(null); Assertions.assertNotNull(parsedId); Assertions.assertEquals(parsedId.getId(), generatedId.getId()); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java index fd8ddfe4..106534ed 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java @@ -1,5 +1,7 @@ package io.appform.ranger.discovery.bundle.id; +import com.codahale.metrics.MetricRegistry; +import io.appform.ranger.discovery.bundle.id.config.IdGeneratorConfig; import io.appform.ranger.discovery.bundle.id.config.IdGeneratorRetryConfig; import io.appform.ranger.discovery.bundle.id.config.WeightedIdConfig; import io.appform.ranger.discovery.bundle.id.config.PartitionRange; @@ -20,6 +22,8 @@ import java.util.List; import java.util.function.Function; +import static org.mockito.Mockito.mock; + /** * Test performance between different constructs */ @@ -45,9 +49,13 @@ public void setUp() throws IOException { .partitions(partitionConfigList) .build(); weightedIdGenerator = new WeightedIdGenerator( - 1024, partitionResolverSupplier, - IdGeneratorRetryConfig.builder().idGenerationRetryCount(1024).partitionRetryCount(1024).build(), - weightedIdConfig + IdGeneratorConfig.builder() + .partitionCount(1024) + .weightedIdConfig(weightedIdConfig) + .retryConfig(IdGeneratorRetryConfig.builder().idGenerationRetryCount(1024).partitionRetryCount(1024).build()) + .build(), + partitionResolverSupplier, + mock(MetricRegistry.class) ); } } diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java index 34b40db6..bb0ac4e9 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java @@ -1,10 +1,12 @@ package io.appform.ranger.discovery.bundle.id; +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; +import io.appform.ranger.discovery.bundle.id.config.IdGeneratorConfig; import io.appform.ranger.discovery.bundle.id.config.IdGeneratorRetryConfig; import io.appform.ranger.discovery.bundle.id.config.WeightedIdConfig; import io.appform.ranger.discovery.bundle.id.constraints.PartitionValidationConstraint; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; -import io.appform.ranger.discovery.bundle.id.weighted.PartitionIdTracker; import io.appform.ranger.discovery.bundle.id.config.PartitionRange; import io.appform.ranger.discovery.bundle.id.weighted.WeightedIdGenerator; import io.appform.ranger.discovery.bundle.id.config.WeightedPartition; @@ -19,11 +21,16 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.function.Function; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + /** * Test for {@link WeightedIdGenerator} */ @@ -32,104 +39,141 @@ class WeightedIdGeneratorTest { final int partitionCount = 1024; final Function partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length() - 6)) % partitionCount; - final IdGeneratorRetryConfig retryConfig = IdGeneratorRetryConfig.builder().idGenerationRetryCount(4096).partitionRetryCount(4096).build(); private WeightedIdGenerator weightedIdGenerator; - private WeightedIdConfig weightedIdConfig; + private IdGeneratorConfig idGeneratorConfig; @BeforeEach void setup() { + val metricRegistry = mock(MetricRegistry.class); + val meter = mock(Meter.class); + doReturn(meter).when(metricRegistry).meter(anyString()); + doNothing().when(meter).mark(); List partitionConfigList = new ArrayList<>(); partitionConfigList.add(WeightedPartition.builder() - .partitionRange(PartitionRange.builder().start(0).end(511).build()) + .partitionRange(PartitionRange.builder().start(0).end(31).build()) .weight(400).build()); partitionConfigList.add(WeightedPartition.builder() - .partitionRange(PartitionRange.builder().start(512).end(1023).build()) + .partitionRange(PartitionRange.builder().start(32).end(63).build()) .weight(600).build()); - weightedIdConfig = WeightedIdConfig.builder() + val weightedIdConfig = WeightedIdConfig.builder() .partitions(partitionConfigList) .build(); - weightedIdGenerator = new WeightedIdGenerator(partitionCount, partitionResolverSupplier, retryConfig, weightedIdConfig); + idGeneratorConfig = + IdGeneratorConfig.builder() + .partitionCount(partitionCount) + .weightedIdConfig(weightedIdConfig) + .idPoolSize(100) + .retryConfig(IdGeneratorRetryConfig.builder().idGenerationRetryCount(4096).partitionRetryCount(4096).build()) + .build(); + weightedIdGenerator = new WeightedIdGenerator(idGeneratorConfig, partitionResolverSupplier, metricRegistry); } @Test void testGenerateWithBenchmark() throws IOException { - val totalTime = TestUtil.runMTTest(5, 100000, (k) -> weightedIdGenerator.generate("P"), this.getClass().getName() + ".testGenerateWithBenchmark"); - testUniqueIds(weightedIdGenerator.getIdStore()); + val allIdsList = Collections.synchronizedList(new ArrayList()); + val totalTime = TestUtil.runMTTest( + 5, + 100000, + (k) -> { + val id = weightedIdGenerator.generate("P"); + id.ifPresent(value -> allIdsList.add(value.getId())); + }, + this.getClass().getName() + ".testGenerateWithBenchmark"); + checkUniqueIds(allIdsList); + checkDistribution(allIdsList); + } + + @Test + void testGenerateAccuracy() throws IOException { + val allIdsList = Collections.synchronizedList(new ArrayList()); + val numThreads = 1; + val iterationCount = 100000; + val totalIdCount = numThreads * iterationCount; + val totalTime = TestUtil.runMTTest( + numThreads, + iterationCount, + (k) -> { + val id = weightedIdGenerator.generate("P"); + id.ifPresent(value -> allIdsList.add(value.getId())); + }, + this.getClass().getName() + ".testGenerateWithBenchmark"); + checkUniqueIds(allIdsList); + checkDistribution(allIdsList); } @Test void testGenerateWithConstraints() throws IOException { + val allIdsList = Collections.synchronizedList(new ArrayList()); PartitionValidationConstraint partitionConstraint = (k) -> k % 10 == 0; weightedIdGenerator.registerGlobalConstraints(partitionConstraint); - val totalTime = TestUtil.runMTTest(5, 100000, (k) -> weightedIdGenerator.generateWithConstraints("P", (String) null, false), this.getClass().getName() + ".testGenerateWithConstraints"); - testUniqueIds(weightedIdGenerator.getIdStore()); - - for (Map.Entry> entry : weightedIdGenerator.getIdStore().entrySet()) { - val prefix = entry.getKey(); - val prefixIds = entry.getValue(); - HashSet uniqueIds = new HashSet<>(); - for (Map.Entry prefixEntry : prefixIds.entrySet()) { - val key = prefixEntry.getKey(); - val partitionIdTracker = prefixEntry.getValue(); - for (int idx = 0; idx < partitionIdTracker.getPartitionSize(); idx += 1) { - if (!partitionConstraint.isValid(idx)) { - Assertions.assertEquals(0, partitionIdTracker.getIdPoolList()[idx].getPointer().get()); - } + val totalTime = TestUtil.runMTTest( + 5, + 100000, + (k) -> { + val id = weightedIdGenerator.generateWithConstraints("P", (String) null, false); + id.ifPresent(value -> allIdsList.add(value.getId())); + }, + this.getClass().getName() + ".testGenerateWithConstraints"); + checkUniqueIds(allIdsList); + + val idCountMap = new HashMap(); + for (val id: allIdsList) { + val partitionId = partitionResolverSupplier.apply(id); + idCountMap.put(partitionId, idCountMap.getOrDefault(partitionId, 0) + 1); + } + + for (WeightedPartition partition: idGeneratorConfig.getWeightedIdConfig().getPartitions()) { + for (int partitionId = partition.getPartitionRange().getStart(); partitionId <= partition.getPartitionRange().getEnd(); partitionId++) { + if (!partitionConstraint.isValid(partitionId)) { + Assertions.assertEquals(0, idCountMap.getOrDefault(partitionId, 0)); } } } } - void testUniqueIds(Map> dataStore) { - boolean allIdsUnique = true; - for (Map.Entry> entry : dataStore.entrySet()) { - val prefix = entry.getKey(); - val prefixIds = entry.getValue(); - for (Map.Entry prefixEntry : prefixIds.entrySet()) { - val key = prefixEntry.getKey(); - val partitionIdTracker = prefixEntry.getValue(); - HashSet uniqueIds = new HashSet<>(); - for (val idPool : partitionIdTracker.getIdPoolList()) { - boolean allIdsUniqueInList = true; - HashSet uniqueIdsInList = new HashSet<>(); - for (val id : idPool.getIdList()) { - if (uniqueIdsInList.contains(id)) { - allIdsUniqueInList = false; - allIdsUnique = false; - } else { - uniqueIdsInList.add(id); - } - - if (uniqueIds.contains(id)) { - allIdsUnique = false; - } else { - uniqueIds.add(id); - } - } - Assertions.assertTrue(allIdsUniqueInList); - } + void checkUniqueIds(List allIdsList) { + HashSet uniqueIds = new HashSet<>(allIdsList); + Assertions.assertEquals(allIdsList.size(), uniqueIds.size()); + } + + void checkDistribution(List allIdsList) { + val idCountMap = new HashMap(); + for (val id: allIdsList) { + val partitionId = partitionResolverSupplier.apply(id); + idCountMap.put(partitionId, idCountMap.getOrDefault(partitionId, 0) + 1); + } + + for (WeightedPartition partition: idGeneratorConfig.getWeightedIdConfig().getPartitions()) { + val expectedIdCount = ((double) partition.getWeight() / weightedIdGenerator.getMaxShardWeight()) * ((double) allIdsList.size() / (partition.getPartitionRange().getEnd()-partition.getPartitionRange().getStart()+1)); + int c = 0; + for (int partitionId = partition.getPartitionRange().getStart(); partitionId <= partition.getPartitionRange().getEnd(); partitionId++) { + log.debug("For {} -- Expected: {} -- Actual: {} -- Perc: {}", partitionId, expectedIdCount, idCountMap.get(partitionId), (double) idCountMap.get(partitionId) * 100 / allIdsList.size()); + Assertions.assertTrue(expectedIdCount * 0.8 <= idCountMap.get(partitionId)); + Assertions.assertTrue(idCountMap.get(partitionId) <= expectedIdCount * 1.2); + c += idCountMap.get(partitionId); } + log.warn("Partition ID Count: {} - Perc: {}", c, (double) c *100 / allIdsList.size()); } - Assertions.assertTrue(allIdsUnique); } @Test void testGenerateOriginal() { - weightedIdGenerator = new WeightedIdGenerator(partitionCount, partitionResolverSupplier, retryConfig, - weightedIdConfig, IdFormatters.original() - ); - String id = weightedIdGenerator.generate("TEST").get().getId(); + weightedIdGenerator = new WeightedIdGenerator(idGeneratorConfig, partitionResolverSupplier, IdFormatters.original(), mock(MetricRegistry.class)); + val idOptional = weightedIdGenerator.generate("TEST"); + String id = idOptional.isPresent() ? idOptional.get().getId() : ""; Assertions.assertEquals(26, id.length()); } @Test void testGenerateBase36() { weightedIdGenerator = new WeightedIdGenerator( - partitionCount, + idGeneratorConfig, (txnId) -> new BigInteger(txnId.substring(txnId.length() - 6), 36).abs().intValue() % partitionCount, - retryConfig, weightedIdConfig, IdFormatters.base36() + IdFormatters.base36(), + mock(MetricRegistry.class) ); - String id = weightedIdGenerator.generate("TEST").get().getId(); + val idOptional = weightedIdGenerator.generate("TEST"); + String id = idOptional.isPresent() ? idOptional.get().getId() : ""; Assertions.assertEquals(18, id.length()); } @@ -180,7 +224,9 @@ void testParseSuccess() { @Test void testParseSuccessAfterGeneration() { - val generatedId = weightedIdGenerator.generate("TEST123").get(); + val generatedIdOptional = weightedIdGenerator.generate("TEST123"); + val generatedId = generatedIdOptional.orElse(null); + Assertions.assertNotNull(generatedId); val parsedId = weightedIdGenerator.parse(generatedId.getId()).orElse(null); Assertions.assertNotNull(parsedId); Assertions.assertEquals(parsedId.getId(), generatedId.getId()); From 709459422312ffa0d7f5fc7b921247fe4ce67d01 Mon Sep 17 00:00:00 2001 From: Satyam Saxena Date: Fri, 14 Jun 2024 14:51:58 +0530 Subject: [PATCH 11/12] Make idStore Get and Set Synchronized --- ...AwareIdGeneratorPerfTest.testGenerate.json | 2 +- ...neratorTest.testGenerateWithBenchmark.json | 4 +- ...ratorTest.testGenerateWithConstraints.json | 4 +- ...ghtedIdGeneratorPerfTest.testGenerate.json | 2 +- ...neratorTest.testGenerateWithBenchmark.json | 4 +- ...ratorTest.testGenerateWithConstraints.json | 4 +- .../discovery/bundle/id/GenerationResult.java | 9 --- .../discovery/bundle/id/IdGenerator.java | 30 +++++++- .../ranger/discovery/bundle/id/IdInfo.java | 14 ---- .../id/weighted/DistributedIdGenerator.java | 70 +++++++++---------- .../discovery/bundle/id/weighted/IdPool.java | 29 +++++--- .../id/weighted/PartitionIdTracker.java | 27 ++++++- .../id/weighted/WeightedIdGenerator.java | 4 +- .../id/PartitionAwareIdGeneratorPerfTest.java | 3 +- .../id/PartitionAwareIdGeneratorTest.java | 19 +++-- .../id/WeightedIdGeneratorPerfTest.java | 3 +- .../bundle/id/WeightedIdGeneratorTest.java | 34 ++++----- 17 files changed, 147 insertions(+), 115 deletions(-) delete mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/GenerationResult.java delete mode 100644 ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdInfo.java diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorPerfTest.testGenerate.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorPerfTest.testGenerate.json index 9a36d5e1..a1e15a01 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorPerfTest.testGenerate.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorPerfTest.testGenerate.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 433192.4270233689 + "mean_ops" : 439867.61740979465 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark.json index 0e37ed32..0e48deb3 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark.json @@ -2,6 +2,6 @@ "name" : "io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark", "iterations" : 100000, "threads" : 5, - "totalMillis" : 2313, - "avgTime" : 462.6 + "totalMillis" : 2722, + "avgTime" : 544.4 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints.json index b9d45d70..8c9c0271 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints.json @@ -2,6 +2,6 @@ "name" : "io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints", "iterations" : 100000, "threads" : 5, - "totalMillis" : 7745, - "avgTime" : 1549.0 + "totalMillis" : 7014, + "avgTime" : 1402.8 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorPerfTest.testGenerate.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorPerfTest.testGenerate.json index bfe6257c..ff9fb07f 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorPerfTest.testGenerate.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorPerfTest.testGenerate.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 379648.93091326446 + "mean_ops" : 419418.9728131592 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json index d296550a..a6dd03e1 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json @@ -2,6 +2,6 @@ "name" : "io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark", "iterations" : 100000, "threads" : 5, - "totalMillis" : 31489, - "avgTime" : 6297.8 + "totalMillis" : 3668, + "avgTime" : 733.6 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json index 4e8fd6d9..db6fcc43 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json @@ -2,6 +2,6 @@ "name" : "io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints", "iterations" : 100000, "threads" : 5, - "totalMillis" : 105139, - "avgTime" : 21027.8 + "totalMillis" : 28683, + "avgTime" : 5736.6 } \ No newline at end of file diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/GenerationResult.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/GenerationResult.java deleted file mode 100644 index b4086901..00000000 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/GenerationResult.java +++ /dev/null @@ -1,9 +0,0 @@ -package io.appform.ranger.discovery.bundle.id; - -import lombok.Value; - -@Value -class GenerationResult { - Id id; - IdValidationState state; -} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGenerator.java index c8c80b7d..f41d5e7e 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGenerator.java @@ -27,6 +27,7 @@ import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; import io.appform.ranger.discovery.bundle.id.request.IdGenerationRequest; import lombok.Getter; +import lombok.Value; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.joda.time.DateTime; @@ -127,11 +128,11 @@ public static Id generate(String prefix) { public static Id generate(final String prefix, final IdFormatter idFormatter) { val idInfo = random(); - val dateTime = new DateTime(idInfo.getTime()); - val id = String.format("%s%s", prefix, idFormatter.format(dateTime, nodeId, idInfo.getExponent())); + val dateTime = new DateTime(idInfo.time); + val id = String.format("%s%s", prefix, idFormatter.format(dateTime, nodeId, idInfo.exponent)); return Id.builder() .id(id) - .exponent(idInfo.getExponent()) + .exponent(idInfo.exponent) .generatedDate(dateTime.toDate()) .node(nodeId) .build(); @@ -289,4 +290,27 @@ private static int readRetryCount() { throw new IllegalArgumentException("Please provide a valid positive integer for NUM_ID_GENERATION_RETRIES"); } } + + private enum IdValidationState { + VALID, + INVALID_RETRYABLE, + INVALID_NON_RETRYABLE + } + + @Value + private static class IdInfo { + int exponent; + long time; + + public IdInfo(int exponent, long time) { + this.exponent = exponent; + this.time = time; + } + } + + @Value + private static class GenerationResult { + Id id; + IdValidationState state; + } } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdInfo.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdInfo.java deleted file mode 100644 index 742e665d..00000000 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdInfo.java +++ /dev/null @@ -1,14 +0,0 @@ -package io.appform.ranger.discovery.bundle.id; - -import lombok.Value; - -@Value -class IdInfo { - int exponent; - long time; - - public IdInfo(int exponent, long time) { - this.exponent = exponent; - this.time = time; - } -} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java index f35bae92..a8f0a896 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/DistributedIdGenerator.java @@ -7,14 +7,12 @@ import dev.failsafe.Failsafe; import dev.failsafe.FailsafeExecutor; import dev.failsafe.RetryPolicy; -import io.appform.ranger.discovery.bundle.id.Constants; import io.appform.ranger.discovery.bundle.id.Id; import io.appform.ranger.discovery.bundle.id.IdGenerator; import io.appform.ranger.discovery.bundle.id.config.IdGeneratorConfig; import io.appform.ranger.discovery.bundle.id.constraints.PartitionValidationConstraint; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatter; import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.joda.time.DateTime; @@ -22,7 +20,6 @@ import org.joda.time.format.DateTimeFormatter; import java.security.SecureRandom; -import java.time.Clock; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -31,8 +28,6 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.regex.Pattern; @@ -51,18 +46,16 @@ abstract class DistributedIdGenerator { private static final List GLOBAL_CONSTRAINTS = new ArrayList<>(); private static final Map> DOMAIN_SPECIFIC_CONSTRAINTS = new HashMap<>(); protected static final int NODE_ID = IdGenerator.getNodeId(); - @Getter - private final Map> idStore = new ConcurrentHashMap<>(); + private final Map idStore = new ConcurrentHashMap<>(); protected final IdFormatter idFormatter; protected final Function partitionResolver; protected final IdGeneratorConfig idGeneratorConfig; - private final Clock clock = Clock.systemDefaultZone(); private final Meter retryLimitBreachedMeter; /* idStore Structure { - prefix: { - timestamp: { + prefix: [ + : { partitions: [ { ids: [], @@ -75,7 +68,7 @@ abstract class DistributedIdGenerator { ], counter: } - } + ] } */ @@ -93,13 +86,6 @@ protected DistributedIdGenerator(final IdGeneratorConfig idGeneratorConfig, .handleResultIf(Objects::isNull) .build(); retrier = Failsafe.with(Collections.singletonList(retryPolicy)); - - val executorService = Executors.newScheduledThreadPool(1); - executorService.scheduleWithFixedDelay( - this::deleteExpiredKeys, - Constants.ID_DELETION_DELAY_IN_SECONDS, - Constants.ID_DELETION_DELAY_IN_SECONDS, - TimeUnit.SECONDS); } protected DistributedIdGenerator(final IdGeneratorConfig idGeneratorConfig, @@ -143,12 +129,12 @@ public Optional generate(final String prefix) { } public Optional generateForPartition(final String prefix, final int targetPartitionId) { - val prefixIdMap = idStore.computeIfAbsent(prefix, k -> new ConcurrentHashMap<>()); val currentTimestamp = new DateTime(); - val timeKey = getTimeKey(clock.instant().getEpochSecond()); + val prefixIdMap = idStore.computeIfAbsent(prefix, k -> new PartitionIdTracker[idGeneratorConfig.getMaxDataBufferTimeInSeconds()]); + val timeKey = getTimeKey(currentTimestamp.getMillis() / 1000); + val partitionTracker = getPartitionTracker(prefixIdMap, currentTimestamp); val idCounter = generateForAllPartitions( - prefixIdMap.computeIfAbsent(timeKey, key -> new PartitionIdTracker(idGeneratorConfig.getPartitionCount(), - idGeneratorConfig.getIdPoolSize())), + partitionTracker, prefix, currentTimestamp, targetPartitionId); @@ -171,23 +157,24 @@ private Optional generateForAllPartitions(final PartitionIdTracker part final DateTime timestamp, final int targetPartitionId) { val idPool = partitionIdTracker.getPartition(targetPartitionId); - int idIdx = idPool.getUsableIdIndex(); int retryCount = 0; try { - while (!idPool.isIdPresentAtIndex(idIdx) && retryCount < idGeneratorConfig.getRetryConfig().getIdGenerationRetryCount()) { + while (retryCount < idGeneratorConfig.getRetryConfig().getIdGenerationRetryCount()) { val counterValue = partitionIdTracker.getIdCounter(); val txnId = String.format("%s%s", prefix, idFormatter.format(timestamp, NODE_ID, counterValue)); val mappedPartitionId = partitionResolver.apply(txnId); partitionIdTracker.addId(mappedPartitionId, counterValue); retryCount += 1; + val idOptional = idPool.getNextId(); + if (idOptional.isPresent()) { + return idOptional; + } } - if (idPool.isIdPresentAtIndex(idIdx)) { - return Optional.of(idPool.getId(idIdx)); - } else { - retryLimitBreachedMeter.mark(); - log.warn("Retry Limit reached - {} - {} - {}", retryCount, idIdx, targetPartitionId); - return Optional.empty(); - } + +// Retry Limit Breached + retryLimitBreachedMeter.mark(); + log.debug("Retry Limit reached - {} - {}", retryCount, targetPartitionId); + return Optional.empty(); } catch (Exception e) { log.error("Error while generating IDs", e); return Optional.empty(); @@ -293,16 +280,23 @@ protected boolean validateId(final List inConstra return null == failedLocalConstraint; } - private long getTimeKey(long timeInSeconds) { -// log.warn("{}:{}", timeInSeconds, timeInSeconds % idGeneratorConfig.getMaxDataBufferTimeInSeconds()); - return timeInSeconds % idGeneratorConfig.getMaxDataBufferTimeInSeconds(); + private int getTimeKey(long timeInSeconds) { + return (int) timeInSeconds % idGeneratorConfig.getMaxDataBufferTimeInSeconds(); } - private synchronized void deleteExpiredKeys() { - val timeThreshold = getTimeKey(clock.instant().getEpochSecond()) - Constants.DELETION_THRESHOLD_IN_SECONDS; - for (val entry : idStore.entrySet()) { - entry.getValue().entrySet().removeIf(partitionIdTrackerEntry -> partitionIdTrackerEntry.getKey() < timeThreshold); + private synchronized PartitionIdTracker getPartitionTracker(PartitionIdTracker[] partitionTrackerList, final DateTime timestamp) { + val timeKey = getTimeKey(timestamp.getMillis() / 1000); + if (timeKey >= partitionTrackerList.length) { + throw new IndexOutOfBoundsException("Key should be less than " + partitionTrackerList.length); + } + if (partitionTrackerList[timeKey] == null) { + partitionTrackerList[timeKey] = new PartitionIdTracker(idGeneratorConfig.getPartitionCount(), idGeneratorConfig.getIdPoolSize(), timestamp); + } + val partitionTracker = partitionTrackerList[timeKey]; + if (partitionTracker.getTimestamp().getMillis() / 1000 != timestamp.getMillis() / 1000) { + partitionTracker.reset(timestamp); } + return partitionTracker; } } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/IdPool.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/IdPool.java index 89dad9d0..ed5a47e8 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/IdPool.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/IdPool.java @@ -2,6 +2,7 @@ import lombok.val; +import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicIntegerArray; @@ -23,26 +24,32 @@ public IdPool(int size) { idList = new AtomicIntegerArray(capacity); } - public boolean isIdPresentAtIndex(int index) { - return lastUsableIdx.get() > index; - } - - public int getId(int index) { + private int getId(int index) { val arrayIdx = index % capacity; return idList.get(arrayIdx); } - public void setId(int id) { + public synchronized void setId(int id) { // Don't set new IDs if the idPool is already full of unused IDs. - if (lastUsableIdx.get() >= nextUsableIdx.get() + capacity) { + if (lastUsableIdx.get() >= nextUsableIdx.get() + capacity - 1) { return; } - val arrayIdx = lastUsableIdx.get() % capacity; + val arrayIdx = lastUsableIdx.getAndIncrement() % capacity; idList.set(arrayIdx, id); - lastUsableIdx.incrementAndGet(); } - public int getUsableIdIndex() { - return nextUsableIdx.getAndIncrement(); + public synchronized Optional getNextId() { + if (nextUsableIdx.get() < lastUsableIdx.get()) { + val id = getId(nextUsableIdx.get()); + nextUsableIdx.getAndIncrement(); + return Optional.of(id); + } else { + return Optional.empty(); + } + } + + public void reset() { + lastUsableIdx.set(0); + nextUsableIdx.set(0); } } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionIdTracker.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionIdTracker.java index 9fced327..da7aea48 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionIdTracker.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/weighted/PartitionIdTracker.java @@ -1,17 +1,27 @@ package io.appform.ranger.discovery.bundle.id.weighted; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.joda.time.DateTime; + import java.util.concurrent.atomic.AtomicInteger; import static io.appform.ranger.discovery.bundle.id.Constants.MAX_IDS_PER_SECOND; +@Slf4j public class PartitionIdTracker { - // Array to store IdPools for each partition +// Array to store IdPools for each partition private final IdPool[] idPoolList; - // Counter to keep track of the number of IDs created +// Counter to keep track of the number of IDs created private final AtomicInteger nextIdCounter = new AtomicInteger(); - protected PartitionIdTracker(int partitionSize, int idPoolSize) { + @Getter + private DateTime timestamp; + + protected PartitionIdTracker(int partitionSize, int idPoolSize, DateTime timestamp) { + this.timestamp = timestamp; idPoolList = new IdPool[partitionSize]; for (int i=0; i partitionResolverSupplier, final MetricRegistry metricRegistry) { -// Preconditions.checkNotNull() super(idGeneratorConfig, partitionResolverSupplier, metricRegistry); + Preconditions.checkNotNull(idGeneratorConfig.getWeightedIdConfig()); partitionRangeMap = createWeightRangeMap(idGeneratorConfig.getWeightedIdConfig()); } @@ -40,6 +41,7 @@ public WeightedIdGenerator(final IdGeneratorConfig idGeneratorConfig, final IdFormatter idFormatterInstance, final MetricRegistry metricRegistry) { super(idGeneratorConfig, partitionResolverSupplier, idFormatterInstance, metricRegistry); + Preconditions.checkNotNull(idGeneratorConfig.getWeightedIdConfig()); partitionRangeMap = createWeightRangeMap(idGeneratorConfig.getWeightedIdConfig()); } diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java index cdad5279..e8a1be59 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorPerfTest.java @@ -34,7 +34,8 @@ public void setUp() throws IOException { partitionAwareIdGenerator = new PartitionAwareIdGenerator( IdGeneratorConfig.builder() .partitionCount(1024) - .retryConfig(IdGeneratorRetryConfig.builder().idGenerationRetryCount(1024).partitionRetryCount(1024).build()) + .idPoolSize(100) + .retryConfig(IdGeneratorRetryConfig.builder().idGenerationRetryCount(4096).partitionRetryCount(4096).build()) .build(), partitionResolverSupplier, mock(MetricRegistry.class) diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java index 108a8c73..1d39b048 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java @@ -18,6 +18,7 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.function.Function; @@ -30,7 +31,6 @@ /** * Test for {@link PartitionAwareIdGenerator} */ -@Slf4j @SuppressWarnings({"unused", "FieldMayBeFinal"}) class PartitionAwareIdGeneratorTest { final int numThreads = 5; @@ -55,7 +55,6 @@ void setup() { idGeneratorConfig, partitionResolverSupplier, metricRegistry ); } -// ToDo: Add test for partition distribution spread. @Test void testGenerateWithBenchmark() throws IOException { @@ -70,6 +69,7 @@ void testGenerateWithBenchmark() throws IOException { this.getClass().getName() + ".testGenerateWithBenchmark"); Assertions.assertEquals(numThreads * iterationCount, allIdsList.size()); checkUniqueIds(allIdsList); + checkDistribution(allIdsList); } @Test @@ -101,6 +101,19 @@ void checkUniqueIds(List allIdsList) { Assertions.assertEquals(allIdsList.size(), uniqueIds.size()); } + void checkDistribution(List allIdsList) { + val idCountMap = new HashMap(); + for (val id: allIdsList) { + val partitionId = partitionResolverSupplier.apply(id); + idCountMap.put(partitionId, idCountMap.getOrDefault(partitionId, 0) + 1); + } + val expectedIdCount = (double) allIdsList.size() / idGeneratorConfig.getPartitionCount(); + for (int partitionId=0; partitionId < idGeneratorConfig.getPartitionCount(); partitionId++) { + Assertions.assertTrue(expectedIdCount * 0.8 <= idCountMap.get(partitionId)); + Assertions.assertTrue(idCountMap.get(partitionId) <= expectedIdCount * 1.2); + } + } + @Test void testUniqueIds() { HashSet allIDs = new HashSet<>(); @@ -109,8 +122,6 @@ void testUniqueIds() { val txnIdOptional = partitionAwareIdGenerator.generate("P"); val txnId = txnIdOptional.map(Id::getId).orElse(null); if (allIDs.contains(txnId)) { - log.warn(txnId); - log.warn(String.valueOf(allIDs)); allIdsUnique = false; } else { allIDs.add(txnId); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java index 106534ed..f2a57bbc 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorPerfTest.java @@ -52,7 +52,8 @@ public void setUp() throws IOException { IdGeneratorConfig.builder() .partitionCount(1024) .weightedIdConfig(weightedIdConfig) - .retryConfig(IdGeneratorRetryConfig.builder().idGenerationRetryCount(1024).partitionRetryCount(1024).build()) + .idPoolSize(100) + .retryConfig(IdGeneratorRetryConfig.builder().idGenerationRetryCount(4096).partitionRetryCount(4096).build()) .build(), partitionResolverSupplier, mock(MetricRegistry.class) diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java index bb0ac4e9..f313b271 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/WeightedIdGeneratorTest.java @@ -37,7 +37,9 @@ @Slf4j @SuppressWarnings({"unused", "FieldMayBeFinal"}) class WeightedIdGeneratorTest { - final int partitionCount = 1024; + final int numThreads = 5; + final int iterationCount = 100000; + final int partitionCount = 64; final Function partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length() - 6)) % partitionCount; private WeightedIdGenerator weightedIdGenerator; private IdGeneratorConfig idGeneratorConfig; @@ -72,13 +74,14 @@ void setup() { void testGenerateWithBenchmark() throws IOException { val allIdsList = Collections.synchronizedList(new ArrayList()); val totalTime = TestUtil.runMTTest( - 5, - 100000, + numThreads, + iterationCount, (k) -> { val id = weightedIdGenerator.generate("P"); id.ifPresent(value -> allIdsList.add(value.getId())); }, this.getClass().getName() + ".testGenerateWithBenchmark"); + Assertions.assertEquals(numThreads * iterationCount, allIdsList.size()); checkUniqueIds(allIdsList); checkDistribution(allIdsList); } @@ -87,7 +90,6 @@ void testGenerateWithBenchmark() throws IOException { void testGenerateAccuracy() throws IOException { val allIdsList = Collections.synchronizedList(new ArrayList()); val numThreads = 1; - val iterationCount = 100000; val totalIdCount = numThreads * iterationCount; val totalTime = TestUtil.runMTTest( numThreads, @@ -97,6 +99,7 @@ void testGenerateAccuracy() throws IOException { id.ifPresent(value -> allIdsList.add(value.getId())); }, this.getClass().getName() + ".testGenerateWithBenchmark"); + Assertions.assertEquals(numThreads * iterationCount, allIdsList.size()); checkUniqueIds(allIdsList); checkDistribution(allIdsList); } @@ -107,8 +110,8 @@ void testGenerateWithConstraints() throws IOException { PartitionValidationConstraint partitionConstraint = (k) -> k % 10 == 0; weightedIdGenerator.registerGlobalConstraints(partitionConstraint); val totalTime = TestUtil.runMTTest( - 5, - 100000, + numThreads, + iterationCount, (k) -> { val id = weightedIdGenerator.generateWithConstraints("P", (String) null, false); id.ifPresent(value -> allIdsList.add(value.getId())); @@ -116,18 +119,10 @@ void testGenerateWithConstraints() throws IOException { this.getClass().getName() + ".testGenerateWithConstraints"); checkUniqueIds(allIdsList); - val idCountMap = new HashMap(); +// Assert No ID was generated for Invalid partitions for (val id: allIdsList) { val partitionId = partitionResolverSupplier.apply(id); - idCountMap.put(partitionId, idCountMap.getOrDefault(partitionId, 0) + 1); - } - - for (WeightedPartition partition: idGeneratorConfig.getWeightedIdConfig().getPartitions()) { - for (int partitionId = partition.getPartitionRange().getStart(); partitionId <= partition.getPartitionRange().getEnd(); partitionId++) { - if (!partitionConstraint.isValid(partitionId)) { - Assertions.assertEquals(0, idCountMap.getOrDefault(partitionId, 0)); - } - } + Assertions.assertTrue(partitionConstraint.isValid(partitionId)); } } @@ -145,14 +140,13 @@ void checkDistribution(List allIdsList) { for (WeightedPartition partition: idGeneratorConfig.getWeightedIdConfig().getPartitions()) { val expectedIdCount = ((double) partition.getWeight() / weightedIdGenerator.getMaxShardWeight()) * ((double) allIdsList.size() / (partition.getPartitionRange().getEnd()-partition.getPartitionRange().getStart()+1)); - int c = 0; + int idCountForPartition = 0; for (int partitionId = partition.getPartitionRange().getStart(); partitionId <= partition.getPartitionRange().getEnd(); partitionId++) { - log.debug("For {} -- Expected: {} -- Actual: {} -- Perc: {}", partitionId, expectedIdCount, idCountMap.get(partitionId), (double) idCountMap.get(partitionId) * 100 / allIdsList.size()); Assertions.assertTrue(expectedIdCount * 0.8 <= idCountMap.get(partitionId)); Assertions.assertTrue(idCountMap.get(partitionId) <= expectedIdCount * 1.2); - c += idCountMap.get(partitionId); + idCountForPartition += idCountMap.get(partitionId); } - log.warn("Partition ID Count: {} - Perc: {}", c, (double) c *100 / allIdsList.size()); + log.debug("Partition ID Count: {} - Percentage: {}", idCountForPartition, (double) idCountForPartition * 100 / allIdsList.size()); } } From 5f632ecfe09c893be8b1d73bf1f7f2b7bfd72383 Mon Sep 17 00:00:00 2001 From: Satyam Saxena Date: Wed, 10 Jul 2024 14:58:05 +0530 Subject: [PATCH 12/12] Sanity Changes --- .../bundle/id/config/IdGeneratorConfig.java | 10 ++++----- .../id/PartitionAwareIdGeneratorTest.java | 2 ++ .../ranger/discovery/bundle/id/TestUtil.java | 6 +++-- .../bundle/id/WeightedIdGeneratorTest.java | 22 ++++++++++--------- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/IdGeneratorConfig.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/IdGeneratorConfig.java index 9f3ea52b..cb266571 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/IdGeneratorConfig.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/IdGeneratorConfig.java @@ -9,7 +9,6 @@ import javax.validation.Valid; import javax.validation.constraints.Min; -import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import java.util.ArrayList; import java.util.Comparator; @@ -22,14 +21,15 @@ @AllArgsConstructor @NoArgsConstructor public class IdGeneratorConfig { - @NotEmpty + @NotNull + @Valid private IdGeneratorRetryConfig retryConfig; @Valid private WeightedIdConfig weightedIdConfig; @NotNull - @Min(1) + @Min(2) private int idPoolSize; @NotNull @@ -46,9 +46,7 @@ public boolean isPartitionCountValid() { if (weightedIdConfig != null) { List sortedPartitions = new ArrayList<>(weightedIdConfig.getPartitions()); sortedPartitions.sort(Comparator.comparingInt(k -> k.getPartitionRange().getStart())); - if (sortedPartitions.get(sortedPartitions.size()-1).getPartitionRange().getEnd() - sortedPartitions.get(sortedPartitions.size()-1).getPartitionRange().getStart() != partitionCount) { - return false; - } + return sortedPartitions.get(sortedPartitions.size() - 1).getPartitionRange().getEnd() - sortedPartitions.get(0).getPartitionRange().getStart() + 1 == partitionCount; } return true; } diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java index 1d39b048..9478642f 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareIdGeneratorTest.java @@ -66,6 +66,7 @@ void testGenerateWithBenchmark() throws IOException { val id = partitionAwareIdGenerator.generate("P"); id.ifPresent(value -> allIdsList.add(value.getId())); }, + true, this.getClass().getName() + ".testGenerateWithBenchmark"); Assertions.assertEquals(numThreads * iterationCount, allIdsList.size()); checkUniqueIds(allIdsList); @@ -84,6 +85,7 @@ void testGenerateWithConstraints() throws IOException { val id = partitionAwareIdGenerator.generateWithConstraints("P", (String) null, false); id.ifPresent(value -> allIdsList.add(value.getId())); }, + false, this.getClass().getName() + ".testGenerateWithConstraints"); Assertions.assertEquals(numThreads * iterationCount, allIdsList.size()); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/TestUtil.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/TestUtil.java index 9de1e39f..83653d27 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/TestUtil.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/TestUtil.java @@ -29,7 +29,7 @@ public class TestUtil { private static final String OUTPUT_PATH = "perf/results/%s.json"; - public double runMTTest(int numThreads, int iterationCount, final Consumer supplier, String outputFileName) throws IOException { + public double runMTTest(int numThreads, int iterationCount, final Consumer supplier, final boolean save_output, final String outputFileName) throws IOException { final ExecutorService executorService = Executors.newFixedThreadPool(numThreads); final List> futures = IntStream.range(0, numThreads) .mapToObj(i -> executorService.submit(() -> { @@ -50,7 +50,9 @@ public double runMTTest(int numThreads, int iterationCount, final Consumer partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length() - 6)) % partitionCount; private WeightedIdGenerator weightedIdGenerator; private IdGeneratorConfig idGeneratorConfig; @@ -52,10 +52,10 @@ void setup() { doNothing().when(meter).mark(); List partitionConfigList = new ArrayList<>(); partitionConfigList.add(WeightedPartition.builder() - .partitionRange(PartitionRange.builder().start(0).end(31).build()) + .partitionRange(PartitionRange.builder().start(0).end(511).build()) .weight(400).build()); partitionConfigList.add(WeightedPartition.builder() - .partitionRange(PartitionRange.builder().start(32).end(63).build()) + .partitionRange(PartitionRange.builder().start(512).end(1023).build()) .weight(600).build()); val weightedIdConfig = WeightedIdConfig.builder() .partitions(partitionConfigList) @@ -80,16 +80,16 @@ void testGenerateWithBenchmark() throws IOException { val id = weightedIdGenerator.generate("P"); id.ifPresent(value -> allIdsList.add(value.getId())); }, + true, this.getClass().getName() + ".testGenerateWithBenchmark"); Assertions.assertEquals(numThreads * iterationCount, allIdsList.size()); checkUniqueIds(allIdsList); - checkDistribution(allIdsList); } @Test void testGenerateAccuracy() throws IOException { val allIdsList = Collections.synchronizedList(new ArrayList()); - val numThreads = 1; + val iterationCount = 400000; val totalIdCount = numThreads * iterationCount; val totalTime = TestUtil.runMTTest( numThreads, @@ -98,8 +98,8 @@ void testGenerateAccuracy() throws IOException { val id = weightedIdGenerator.generate("P"); id.ifPresent(value -> allIdsList.add(value.getId())); }, - this.getClass().getName() + ".testGenerateWithBenchmark"); - Assertions.assertEquals(numThreads * iterationCount, allIdsList.size()); + false, + this.getClass().getName() + ".testGenerateAccuracy"); checkUniqueIds(allIdsList); checkDistribution(allIdsList); } @@ -116,10 +116,11 @@ void testGenerateWithConstraints() throws IOException { val id = weightedIdGenerator.generateWithConstraints("P", (String) null, false); id.ifPresent(value -> allIdsList.add(value.getId())); }, + false, this.getClass().getName() + ".testGenerateWithConstraints"); checkUniqueIds(allIdsList); -// Assert No ID was generated for Invalid partitions +// Assert no ID was generated for Invalid partitions for (val id: allIdsList) { val partitionId = partitionResolverSupplier.apply(id); Assertions.assertTrue(partitionConstraint.isValid(partitionId)); @@ -142,8 +143,9 @@ void checkDistribution(List allIdsList) { val expectedIdCount = ((double) partition.getWeight() / weightedIdGenerator.getMaxShardWeight()) * ((double) allIdsList.size() / (partition.getPartitionRange().getEnd()-partition.getPartitionRange().getStart()+1)); int idCountForPartition = 0; for (int partitionId = partition.getPartitionRange().getStart(); partitionId <= partition.getPartitionRange().getEnd(); partitionId++) { - Assertions.assertTrue(expectedIdCount * 0.8 <= idCountMap.get(partitionId)); - Assertions.assertTrue(idCountMap.get(partitionId) <= expectedIdCount * 1.2); + log.warn("{} - {} - {}", expectedIdCount * 0.9, idCountMap.get(partitionId), expectedIdCount * 1.1); + Assertions.assertTrue(expectedIdCount * 0.9 <= idCountMap.get(partitionId)); + Assertions.assertTrue(idCountMap.get(partitionId) <= expectedIdCount * 1.1); idCountForPartition += idCountMap.get(partitionId); } log.debug("Partition ID Count: {} - Percentage: {}", idCountForPartition, (double) idCountForPartition * 100 / allIdsList.size());