Add Weighted Id Generator

ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json

@@ -4,5 +4,5 @@
   "iterations" : 4,
   "threads" : 1,
   "forks" : 3,
-  "mean_ops" : 644166.1778513143
+  "mean_ops" : 826587.8324153908
 }

ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json

@@ -4,5 +4,5 @@
   "iterations" : 4,
   "threads" : 1,
   "forks" : 3,
-  "mean_ops" : 502644.4941310657
+  "mean_ops" : 654567.0311484573
 }

ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorPerfTest.testGenerate.json

@@ -0,0 +1,8 @@
+{
+  "name" : "io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorPerfTest.testGenerate",
+  "mode" : "Throughput",
+  "iterations" : 4,
+  "threads" : 1,
+  "forks" : 3,
+  "mean_ops" : 439867.61740979465
+}

ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark.json

@@ -0,0 +1,7 @@
+{
+  "name" : "io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithBenchmark",
+  "iterations" : 100000,
+  "threads" : 5,
+  "totalMillis" : 2722,
+  "avgTime" : 544.4
+}

ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints.json

@@ -0,0 +1,7 @@
+{
+  "name" : "io.appform.ranger.discovery.bundle.id.PartitionAwareIdGeneratorTest.testGenerateWithConstraints",
+  "iterations" : 100000,
+  "threads" : 5,
+  "totalMillis" : 7014,
+  "avgTime" : 1402.8
+}

ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorPerfTest.testGenerate.json

@@ -0,0 +1,8 @@
+{
+  "name" : "io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorPerfTest.testGenerate",
+  "mode" : "Throughput",
+  "iterations" : 4,
+  "threads" : 1,
+  "forks" : 3,
+  "mean_ops" : 419418.9728131592
+}

ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark.json

@@ -0,0 +1,7 @@
+{
+  "name" : "io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithBenchmark",
+  "iterations" : 100000,
+  "threads" : 5,
+  "totalMillis" : 3668,
+  "avgTime" : 733.6
+}

ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints.json

@@ -0,0 +1,7 @@
+{
+  "name" : "io.appform.ranger.discovery.bundle.id.WeightedIdGeneratorTest.testGenerateWithConstraints",
+  "iterations" : 100000,
+  "threads" : 5,
+  "totalMillis" : 28683,
+  "avgTime" : 5736.6
+}

ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java

@@ -25,4 +25,8 @@
 public class Constants {
     public static final int MAX_ID_PER_MS = 1000;
     public static final int MAX_NUM_NODES = 10000;
+    public static final int MAX_DATA_STORAGE_TIME_LIMIT_IN_SECONDS = 300;
+    public static final int MAX_IDS_PER_SECOND = 1_000_000;
+    public static final int ID_DELETION_DELAY_IN_SECONDS = 60;
+    public static final int DELETION_THRESHOLD_IN_SECONDS = 60;
 }

ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGenerator.java

@@ -26,6 +26,7 @@
 import io.appform.ranger.discovery.bundle.id.formatter.IdFormatter;
 import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters;
 import io.appform.ranger.discovery.bundle.id.request.IdGenerationRequest;
+import lombok.Getter;
 import lombok.Value;
 import lombok.extern.slf4j.Slf4j;
 import lombok.val;
@@ -67,6 +68,7 @@ public class IdGenerator {
 
     private static final List<IdValidationConstraint> GLOBAL_CONSTRAINTS = new ArrayList<>();
     private static final Map<String, List<IdValidationConstraint>> DOMAIN_SPECIFIC_CONSTRAINTS = new HashMap<>();
+    @Getter
     private static int nodeId;
 
     public static void initialize(int node) {

ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdValidationState.java

@@ -0,0 +1,7 @@
+package io.appform.ranger.discovery.bundle.id;
+
+enum IdValidationState {
+    VALID,
+    INVALID_RETRYABLE,
+    INVALID_NON_RETRYABLE
+}

ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/IdGeneratorConfig.java

@@ -0,0 +1,54 @@
+package io.appform.ranger.discovery.bundle.id.config;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import io.dropwizard.validation.ValidationMethod;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.validation.Valid;
+import javax.validation.constraints.Min;
+import javax.validation.constraints.NotNull;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+import static io.appform.ranger.discovery.bundle.id.Constants.MAX_DATA_STORAGE_TIME_LIMIT_IN_SECONDS;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class IdGeneratorConfig {
+    @NotNull
+    @Valid
+    private IdGeneratorRetryConfig retryConfig;
+
+    @Valid
+    private WeightedIdConfig weightedIdConfig;
+
+    @NotNull
+    @Min(2)
+    private int idPoolSize;
+
+    @NotNull
+    @Min(1)
+    private int partitionCount;
+
+    @Min(1)
+    @Builder.Default
+    private int maxDataBufferTimeInSeconds = MAX_DATA_STORAGE_TIME_LIMIT_IN_SECONDS;
+
+    @ValidationMethod(message = "Invalid Partition Range")
+    @JsonIgnore
+    public boolean isPartitionCountValid() {
+        if (weightedIdConfig != null) {
+            List<WeightedPartition> sortedPartitions = new ArrayList<>(weightedIdConfig.getPartitions());
+            sortedPartitions.sort(Comparator.comparingInt(k -> k.getPartitionRange().getStart()));
+            return sortedPartitions.get(sortedPartitions.size() - 1).getPartitionRange().getEnd() - sortedPartitions.get(0).getPartitionRange().getStart() + 1 == partitionCount;
+        }
+        return true;
+    }
+
+}

ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/IdGeneratorRetryConfig.java

@@ -0,0 +1,24 @@
+package io.appform.ranger.discovery.bundle.id.config;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.validation.constraints.Min;
+import javax.validation.constraints.NotNull;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class IdGeneratorRetryConfig {
+    @NotNull
+    @Min(1)
+    private int idGenerationRetryCount;
+
+    @NotNull
+    @Min(1)
+    private int partitionRetryCount;
+
+}

ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/PartitionRange.java

@@ -0,0 +1,30 @@
+package io.appform.ranger.discovery.bundle.id.config;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import io.dropwizard.validation.ValidationMethod;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.validation.constraints.Min;
+import javax.validation.constraints.NotNull;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class PartitionRange {
+    @NotNull
+    @Min(0)
+    private int start;
+
+    @NotNull
+    private int end;
+
+    @ValidationMethod(message = "Partition Range should be non-decreasing")
+    @JsonIgnore
+    public boolean isRangeValid() {
+        return start <= end;
+    }
+}

ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/WeightedIdConfig.java

@@ -0,0 +1,39 @@
+package io.appform.ranger.discovery.bundle.id.config;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import io.dropwizard.validation.ValidationMethod;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+import javax.validation.Valid;
+import javax.validation.constraints.NotNull;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+@Getter
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class WeightedIdConfig {
+    @NotNull
+    @Valid
+    private List<WeightedPartition> partitions;
+
+    @ValidationMethod(message = "Invalid Partition Range")
+    @JsonIgnore
+    public boolean isPartitionWeightsValid() {
+        List<WeightedPartition> sortedPartitions = new ArrayList<>(partitions);
+        sortedPartitions.sort(Comparator.comparingInt(k -> k.getPartitionRange().getStart()));
+        for (int i = 0; i < sortedPartitions.size() - 1; i++) {
+            WeightedPartition currentPartition = sortedPartitions.get(i);
+            WeightedPartition nextPartition = sortedPartitions.get(i + 1);
+            if (currentPartition.getPartitionRange().getEnd() + 1 != nextPartition.getPartitionRange().getStart()) {
+                return false;
+            }
+        }
+        return true;
+    }
+}

ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/WeightedPartition.java

@@ -0,0 +1,24 @@
+package io.appform.ranger.discovery.bundle.id.config;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+import javax.validation.Valid;
+import javax.validation.constraints.Min;
+import javax.validation.constraints.NotNull;
+
+@Getter
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class WeightedPartition {
+    @NotNull
+    @Valid
+    private PartitionRange partitionRange;
+
+    @NotNull
+    @Min(1)
+    private int weight;
+}

ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/constraints/PartitionValidationConstraint.java

@@ -0,0 +1,10 @@
+package io.appform.ranger.discovery.bundle.id.constraints;
+
+
+public interface PartitionValidationConstraint {
+    boolean isValid(int id);
+
+    default boolean failFast() {
+        return false;
+    }
+}

ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatters.java

@@ -23,6 +23,7 @@ public class IdFormatters {
 
     private static final IdFormatter originalIdFormatter = new DefaultIdFormatter();
     private static final IdFormatter base36IdFormatter = new Base36IdFormatter(originalIdFormatter);
+    private static final IdFormatter partitionAwareIdFormatter = new PartitionAwareIdFormatter();
 
     public static IdFormatter original() {
         return originalIdFormatter;
@@ -32,4 +33,8 @@ public static IdFormatter base36() {
         return base36IdFormatter;
     }
 
+    public static IdFormatter partitionAware() {
+        return partitionAwareIdFormatter;
+    }
+
 }

ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/PartitionAwareIdFormatter.java

@@ -0,0 +1,19 @@
+package io.appform.ranger.discovery.bundle.id.formatter;
+
+import lombok.extern.slf4j.Slf4j;
+import org.joda.time.DateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+@Slf4j
+public class PartitionAwareIdFormatter implements IdFormatter {
+
+    private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormat.forPattern("yyMMddHHmmss");
+
+    @Override
+    public String format(final DateTime dateTime,
+                         final int nodeId,
+                         final int randomNonce) {
+        return String.format("%s%04d%06d", DATE_TIME_FORMATTER.print(dateTime), nodeId, randomNonce);
+    }
+}