nonValidatingSerializer =
- new KafkaJsonSchemaSerializer<>(srClient, nonValidating);
-
- return new JsonValidator() {
- @Override
- public void validate(final String json) {
- throw new UnsupportedOperationException("Not under test");
- }
-
- @Override
- public byte[] serialize(final TestModel model, final boolean validate) {
- return validate
- ? serializer.serialize(TOPIC_NAME, model)
- : nonValidatingSerializer.serialize(TOPIC_NAME, model);
- }
-
- @Override
- public TestModel deserialize(final byte[] data) {
- return deserializer.deserialize(TOPIC_NAME, data);
- }
- };
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- // Final, empty finalize method stops spotbugs CT_CONSTRUCTOR_THROW
- // Can be moved to base type after https://github.com/spotbugs/spotbugs/issues/2665
- @Override
- @SuppressWarnings({"deprecation", "Finalize"})
- protected final void finalize() {}
-}
diff --git a/src/main/java/org/creekservice/kafka/test/perf/implementations/DevHarrelImplementation.java b/src/main/java/org/creekservice/kafka/test/perf/implementations/DevHarrelImplementation.java
index 7901d03..98eb620 100644
--- a/src/main/java/org/creekservice/kafka/test/perf/implementations/DevHarrelImplementation.java
+++ b/src/main/java/org/creekservice/kafka/test/perf/implementations/DevHarrelImplementation.java
@@ -45,7 +45,7 @@ public class DevHarrelImplementation implements Implementation {
private static final MetaData METADATA =
new MetaData(
"json-schema (dev.harrel)",
- "dev.harrel",
+ "DevHarrel",
Language.Java,
Licence.MIT,
Set.of(DRAFT_2020_12, DRAFT_2019_09),
diff --git a/src/main/java/org/creekservice/kafka/test/perf/implementations/EveritImplementation.java b/src/main/java/org/creekservice/kafka/test/perf/implementations/EveritImplementation.java
index 2540275..a74fe44 100644
--- a/src/main/java/org/creekservice/kafka/test/perf/implementations/EveritImplementation.java
+++ b/src/main/java/org/creekservice/kafka/test/perf/implementations/EveritImplementation.java
@@ -25,6 +25,7 @@
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
+import java.awt.Color;
import java.io.ByteArrayInputStream;
import java.util.Collection;
import java.util.Map;
@@ -55,7 +56,8 @@ public class EveritImplementation implements Implementation {
Language.Java,
Licence.Apache_v2_0,
Set.of(DRAFT_04, DRAFT_06, DRAFT_07),
- "https://github.com/everit-org/json-schema");
+ "https://github.com/everit-org/json-schema",
+ new Color(54, 162, 235));
private ObjectMapper mapper = JsonMapper.builder().build();
diff --git a/src/main/java/org/creekservice/kafka/test/perf/implementations/Implementation.java b/src/main/java/org/creekservice/kafka/test/perf/implementations/Implementation.java
index f79f673..9af27a8 100644
--- a/src/main/java/org/creekservice/kafka/test/perf/implementations/Implementation.java
+++ b/src/main/java/org/creekservice/kafka/test/perf/implementations/Implementation.java
@@ -25,6 +25,7 @@
import java.net.URL;
import java.util.Set;
import java.util.TreeSet;
+import java.util.regex.Pattern;
import org.creekservice.kafka.test.perf.model.TestModel;
import org.creekservice.kafka.test.perf.testsuite.AdditionalSchemas;
import org.creekservice.kafka.test.perf.testsuite.SchemaSpec;
@@ -85,6 +86,9 @@ public String toString() {
}
class MetaData {
+
+ public static final Pattern SHORT_NAME_PATTERN = Pattern.compile("[A-Za-z0-9]+");
+
private final String longName;
private final String shortName;
private final Language language;
@@ -97,7 +101,8 @@ class MetaData {
* Construct metadata about a specific validator implementation.
*
* @param longName a more expressive name.
- * @param shortName the short name, as used in reports.
+ * @param shortName the short name, as used in reports. Can only contain alphanumeric
+ * characters.
* @param language the programming language the validator library is written in.
* @param licence the licence the validator library is released under.
* @param supported the set of supported JSON schema draft specifications.
@@ -133,21 +138,11 @@ public MetaData(
if (shortName.isBlank()) {
throw new IllegalArgumentException("Short name blank");
}
- }
- /**
- * Temp constructor to avoid issues for anyone currently adding new implementation.
- *
- * Will be removed soon.
- */
- public MetaData(
- final String longName,
- final String shortName,
- final Language language,
- final Licence licence,
- final Set supported,
- final String url) {
- this(longName, shortName, language, licence, supported, url, new Color(235, 54, 172));
+ if (!SHORT_NAME_PATTERN.matcher(shortName).matches()) {
+ throw new IllegalArgumentException(
+ "Short name not match required pattern: " + SHORT_NAME_PATTERN.pattern());
+ }
}
@JsonProperty("longName")
diff --git a/src/main/java/org/creekservice/kafka/test/perf/implementations/Implementations.java b/src/main/java/org/creekservice/kafka/test/perf/implementations/Implementations.java
index 4c842ab..b607866 100644
--- a/src/main/java/org/creekservice/kafka/test/perf/implementations/Implementations.java
+++ b/src/main/java/org/creekservice/kafka/test/perf/implementations/Implementations.java
@@ -27,6 +27,7 @@ public final class Implementations {
private static final List IMPLS =
List.of(
+ new JacksonImplementation(),
new EveritImplementation(),
new JustifyImplementation(),
new MedeiaImplementation(),
diff --git a/src/main/java/org/creekservice/kafka/test/perf/implementations/JacksonImplementation.java b/src/main/java/org/creekservice/kafka/test/perf/implementations/JacksonImplementation.java
index 0d8eeef..e741450 100644
--- a/src/main/java/org/creekservice/kafka/test/perf/implementations/JacksonImplementation.java
+++ b/src/main/java/org/creekservice/kafka/test/perf/implementations/JacksonImplementation.java
@@ -37,7 +37,7 @@ public class JacksonImplementation implements Implementation {
Licence.Apache_v2_0,
Set.of(SchemaSpec.DRAFT_07),
"https://github.com/FasterXML/jackson-core",
- new Color(255, 255, 255));
+ new Color(20, 84, 166));
private ObjectMapper mapper = JsonMapper.builder().build();
diff --git a/src/main/java/org/creekservice/kafka/test/perf/implementations/SkemaImplementation.java b/src/main/java/org/creekservice/kafka/test/perf/implementations/SkemaImplementation.java
index 8fb5cd1..5a7df18 100644
--- a/src/main/java/org/creekservice/kafka/test/perf/implementations/SkemaImplementation.java
+++ b/src/main/java/org/creekservice/kafka/test/perf/implementations/SkemaImplementation.java
@@ -48,7 +48,7 @@ public class SkemaImplementation implements Implementation {
private static final MetaData METADATA =
new MetaData(
"erosb/json-sKema",
- "sKema",
+ "Skema",
Language.Kotlin,
Licence.MIT,
Set.of(DRAFT_2020_12),
diff --git a/src/main/java/org/creekservice/kafka/test/perf/implementations/VertxImplementation.java b/src/main/java/org/creekservice/kafka/test/perf/implementations/VertxImplementation.java
index 3fc996e..63c3dff 100644
--- a/src/main/java/org/creekservice/kafka/test/perf/implementations/VertxImplementation.java
+++ b/src/main/java/org/creekservice/kafka/test/perf/implementations/VertxImplementation.java
@@ -48,7 +48,7 @@ public class VertxImplementation implements Implementation {
private static final MetaData METADATA =
new MetaData(
"Vert.x Json Schema",
- "Vert.x",
+ "Vertx",
Language.Java,
Licence.Apache_v2_0,
SUPPORTED.keySet(),
diff --git a/src/main/java/org/creekservice/kafka/test/perf/performance/JsonSerdeBenchmark.java b/src/main/java/org/creekservice/kafka/test/perf/performance/JsonSerdeBenchmark.java
index 4d18d10..d37d265 100644
--- a/src/main/java/org/creekservice/kafka/test/perf/performance/JsonSerdeBenchmark.java
+++ b/src/main/java/org/creekservice/kafka/test/perf/performance/JsonSerdeBenchmark.java
@@ -20,7 +20,6 @@
import java.nio.file.Path;
import java.util.Map;
-import org.creekservice.kafka.test.perf.implementations.ConfluentImplementation;
import org.creekservice.kafka.test.perf.implementations.DevHarrelImplementation;
import org.creekservice.kafka.test.perf.implementations.EveritImplementation;
import org.creekservice.kafka.test.perf.implementations.Implementation;
@@ -57,23 +56,8 @@
* use the basic JSON schema features: primitives, enums, arrays, polymorphic types and length
* assertions. This can be extended in the future it needed.
*
- * Most recent results (On 2021 Macbook, M1 Max: 2.06 - 3.22 GHz, in High Power mode, JDK
- * 17.0.6):
- *
- *
- * Benchmark Mode Cnt Score Error Units
- * JsonSerdeBenchmark.measureJacksonIntermediateRoundTrip avgt 20 3.852 ± 0.063 us/op
- * JsonSerdeBenchmark.measureRawJacksonRoundTrip avgt 20 3.890 ± 0.047 us/op
- * JsonSerdeBenchmark.measureConfluentRoundTrip avgt 20 131.029 ± 1.964 us/op
- * JsonSerdeBenchmark.measureEveritRoundTrip avgt 20 116.423 ± 2.763 us/op
- * JsonSerdeBenchmark.measureJustifyRoundTrip avgt 20 75.547 ± 0.819 us/op
- * JsonSerdeBenchmark.measureMedeiaRoundTrip avgt 20 38.443 ± 1.010 us/op
- * JsonSerdeBenchmark.measureNetworkNtRoundTrip avgt 20 898.339 ± 30.028 us/op
- * JsonSerdeBenchmark.measureSchemaFriendRoundTrip avgt 20 127.588 ± 0.897 us/op
- * JsonSerdeBenchmark.measureSkemaRoundTrip avgt 20 111.483 ± 2.036 us/op
- * JsonSerdeBenchmark.measureSnowRoundTrip avgt 20 611.803 ± 6.733 us/op
- * JsonSerdeBenchmark.measureVertxRoundTrip avgt 20 738.511 ± 45.223 us/op
- *
+ * The preferred Schema draft is Draft_7. Draft_2020_12 will be used where implementations do not
+ * support 7.
*/
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(MICROSECONDS)
@@ -81,7 +65,7 @@
@Fork(4) // Note: to debug, set fork to 0.
// @Warmup(iterations = 0, time = 10)
// @Measurement(iterations = 1, time = 10)
-@SuppressWarnings("FieldMayBeFinal") // not final to avoid folding.
+@SuppressWarnings({"FieldMayBeFinal", "MethodName"}) // not final to avoid folding.
public class JsonSerdeBenchmark {
static {
@@ -95,7 +79,7 @@ public JacksonState() {
}
@Benchmark
- public TestModel measureJacksonRoundTrip(final JacksonState impl, final ModelState model) {
+ public TestModel measureDraft_07_Jackson(final JacksonState impl, final ModelState model) {
return impl.roundTrip(model);
}
@@ -106,7 +90,7 @@ public MedeiaState() {
}
@Benchmark
- public TestModel measureMedeiaRoundTrip(final MedeiaState impl, final ModelState model) {
+ public TestModel measureDraft_07_Medeia(final MedeiaState impl, final ModelState model) {
return impl.roundTrip(model);
}
@@ -117,7 +101,7 @@ public EveritState() {
}
@Benchmark
- public TestModel measureEveritRoundTrip(final EveritState impl, final ModelState model) {
+ public TestModel measureDraft_07_Everit(final EveritState impl, final ModelState model) {
return impl.roundTrip(model);
}
@@ -128,18 +112,7 @@ public SkemaState() {
}
@Benchmark
- public TestModel measureSkemaRoundTrip(final SkemaState impl, final ModelState model) {
- return impl.roundTrip(model);
- }
-
- public static class ConfluentState extends ImplementationState {
- public ConfluentState() {
- super(new ConfluentImplementation());
- }
- }
-
- @Benchmark
- public TestModel measureConfluentRoundTrip(final ConfluentState impl, final ModelState model) {
+ public TestModel measureDraft_2020_12_Skema(final SkemaState impl, final ModelState model) {
return impl.roundTrip(model);
}
@@ -150,7 +123,7 @@ public VertxState() {
}
@Benchmark
- public TestModel measureVertxRoundTrip(final VertxState impl, final ModelState model) {
+ public TestModel measureDraft_07_Vertx(final VertxState impl, final ModelState model) {
return impl.roundTrip(model);
}
@@ -161,7 +134,7 @@ public SchemaFriendState() {
}
@Benchmark
- public TestModel measureSchemaFriendRoundTrip(
+ public TestModel measureDraft_07_SchemaFriend(
final SchemaFriendState impl, final ModelState model) {
return impl.roundTrip(model);
}
@@ -173,7 +146,7 @@ public NetworkNtState() {
}
@Benchmark
- public TestModel measureNetworkNtRoundTrip(final NetworkNtState impl, final ModelState model) {
+ public TestModel measureDraft_07_NetworkNt(final NetworkNtState impl, final ModelState model) {
return impl.roundTrip(model);
}
@@ -184,7 +157,7 @@ public SnowState() {
}
@Benchmark
- public TestModel measureSnowRoundTrip(final SnowState impl, final ModelState model) {
+ public TestModel measureDraft_07_Snow(final SnowState impl, final ModelState model) {
return impl.roundTrip(model);
}
@@ -195,7 +168,7 @@ public JustifyState() {
}
@Benchmark
- public TestModel measureJustifyRoundTrip(final JustifyState impl, final ModelState model) {
+ public TestModel measureDraft_07_Justify(final JustifyState impl, final ModelState model) {
return impl.roundTrip(model);
}
@@ -206,7 +179,8 @@ public DevHarrelState() {
}
@Benchmark
- public TestModel measureDevHarrelRoundTrip(final DevHarrelState impl, final ModelState model) {
+ public TestModel measureDraft_2020_12_DevHarrel(
+ final DevHarrelState impl, final ModelState model) {
return impl.roundTrip(model);
}
diff --git a/src/main/java/org/creekservice/kafka/test/perf/performance/JsonValidateBenchmark.java b/src/main/java/org/creekservice/kafka/test/perf/performance/JsonValidateBenchmark.java
index 1eaae8a..8faaf70 100644
--- a/src/main/java/org/creekservice/kafka/test/perf/performance/JsonValidateBenchmark.java
+++ b/src/main/java/org/creekservice/kafka/test/perf/performance/JsonValidateBenchmark.java
@@ -50,42 +50,7 @@
*
The benchmark runs each validator through the standard set of tests
*
- *
The results show the average time it take each impl to run through the test suite, per draft.
- *
- *
Most recent results (On 2021 Macbook, M1 Max: 2.06 - 3.22 GHz, in High Power mode, JDK
- * 17.0.6):
- *
- *
- * Benchmark Mode Cnt Score Error Units
- * JsonValidateBenchmark.measureDraft_2019_09_NetworkNt avgt 20 6.017 ± 0.216 ms/op
- * JsonValidateBenchmark.measureDraft_2019_09_SchemaFriend avgt 20 1.482 ± 0.005 ms/op
- * JsonValidateBenchmark.measureDraft_2019_09_Snow avgt 20 316.178 ± 28.242 ms/op
- * JsonValidateBenchmark.measureDraft_2019_09_Vertx avgt 20 3.818 ± 0.028 ms/op
- * JsonValidateBenchmark.measureDraft_2020_12_NetworkNt avgt 20 7.305 ± 0.073 ms/op
- * JsonValidateBenchmark.measureDraft_2020_12_SchemaFriend avgt 20 1.654 ± 0.005 ms/op
- * JsonValidateBenchmark.measureDraft_2020_12_Skema avgt 20 2.812 ± 0.015 ms/op
- * JsonValidateBenchmark.measureDraft_2020_12_Vertx avgt 20 3.669 ± 0.019 ms/op
- * JsonValidateBenchmark.measureDraft_3_SchemaFriend avgt 20 0.235 ± 0.005 ms/op
- * JsonValidateBenchmark.measureDraft_4_Everit avgt 20 0.328 ± 0.006 ms/op
- * JsonValidateBenchmark.measureDraft_4_Justify avgt 20 0.634 ± 0.009 ms/op
- * JsonValidateBenchmark.measureDraft_4_Medeia avgt 20 0.346 ± 0.006 ms/op
- * JsonValidateBenchmark.measureDraft_4_NetworkNt avgt 20 1.086 ± 0.004 ms/op
- * JsonValidateBenchmark.measureDraft_4_SchemaFriend avgt 20 0.480 ± 0.017 ms/op
- * JsonValidateBenchmark.measureDraft_4_Vertx avgt 20 1.362 ± 0.006 ms/op
- * JsonValidateBenchmark.measureDraft_6_Everit avgt 20 0.400 ± 0.003 ms/op
- * JsonValidateBenchmark.measureDraft_6_Justify avgt 20 0.816 ± 0.008 ms/op
- * JsonValidateBenchmark.measureDraft_6_Medeia avgt 20 0.416 ± 0.007 ms/op
- * JsonValidateBenchmark.measureDraft_6_NetworkNt avgt 20 1.771 ± 0.044 ms/op
- * JsonValidateBenchmark.measureDraft_6_SchemaFriend avgt 20 0.700 ± 0.018 ms/op
- * JsonValidateBenchmark.measureDraft_6_Snow avgt 20 78.241 ± 6.515 ms/op
- * JsonValidateBenchmark.measureDraft_7_Everit avgt 20 0.508 ± 0.005 ms/op
- * JsonValidateBenchmark.measureDraft_7_Justify avgt 20 1.044 ± 0.019 ms/op
- * JsonValidateBenchmark.measureDraft_7_Medeia avgt 20 0.666 ± 0.007 ms/op
- * JsonValidateBenchmark.measureDraft_7_NetworkNt avgt 20 2.573 ± 0.032 ms/op
- * JsonValidateBenchmark.measureDraft_7_SchemaFriend avgt 20 0.918 ± 0.012 ms/op
- * JsonValidateBenchmark.measureDraft_7_Snow avgt 20 76.627 ± 6.336 ms/op
- * JsonValidateBenchmark.measureDraft_7_Vertx avgt 20 2.141 ± 0.072 ms/op
- *
+ * The results show the average time it takes each impl to run through the test suite, per draft.
*/
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(MILLISECONDS)
@@ -114,17 +79,17 @@ public MedeiaValidator() {
}
@Benchmark
- public Result measureDraft_4_Medeia(final MedeiaValidator validator) {
+ public Result measureDraft_04_Medeia(final MedeiaValidator validator) {
return validator.validate(SchemaSpec.DRAFT_04);
}
@Benchmark
- public Result measureDraft_6_Medeia(final MedeiaValidator validator) {
+ public Result measureDraft_06_Medeia(final MedeiaValidator validator) {
return validator.validate(SchemaSpec.DRAFT_06);
}
@Benchmark
- public Result measureDraft_7_Medeia(final MedeiaValidator validator) {
+ public Result measureDraft_07_Medeia(final MedeiaValidator validator) {
return validator.validate(SchemaSpec.DRAFT_07);
}
@@ -136,17 +101,17 @@ public EveritValidator() {
}
@Benchmark
- public Result measureDraft_4_Everit(final EveritValidator validator) {
+ public Result measureDraft_04_Everit(final EveritValidator validator) {
return validator.validate(SchemaSpec.DRAFT_04);
}
@Benchmark
- public Result measureDraft_6_Everit(final EveritValidator validator) {
+ public Result measureDraft_06_Everit(final EveritValidator validator) {
return validator.validate(SchemaSpec.DRAFT_06);
}
@Benchmark
- public Result measureDraft_7_Everit(final EveritValidator validator) {
+ public Result measureDraft_07_Everit(final EveritValidator validator) {
return validator.validate(SchemaSpec.DRAFT_07);
}
@@ -170,12 +135,12 @@ public VertxValidator() {
}
@Benchmark
- public Result measureDraft_4_Vertx(final VertxValidator validator) {
+ public Result measureDraft_04_Vertx(final VertxValidator validator) {
return validator.validate(SchemaSpec.DRAFT_04);
}
@Benchmark
- public Result measureDraft_7_Vertx(final VertxValidator validator) {
+ public Result measureDraft_07_Vertx(final VertxValidator validator) {
return validator.validate(SchemaSpec.DRAFT_07);
}
@@ -197,22 +162,22 @@ public SchemaFriendValidator() {
}
@Benchmark
- public Result measureDraft_3_SchemaFriend(final SchemaFriendValidator validator) {
+ public Result measureDraft_03_SchemaFriend(final SchemaFriendValidator validator) {
return validator.validate(SchemaSpec.DRAFT_03);
}
@Benchmark
- public Result measureDraft_4_SchemaFriend(final SchemaFriendValidator validator) {
+ public Result measureDraft_04_SchemaFriend(final SchemaFriendValidator validator) {
return validator.validate(SchemaSpec.DRAFT_04);
}
@Benchmark
- public Result measureDraft_6_SchemaFriend(final SchemaFriendValidator validator) {
+ public Result measureDraft_06_SchemaFriend(final SchemaFriendValidator validator) {
return validator.validate(SchemaSpec.DRAFT_06);
}
@Benchmark
- public Result measureDraft_7_SchemaFriend(final SchemaFriendValidator validator) {
+ public Result measureDraft_07_SchemaFriend(final SchemaFriendValidator validator) {
return validator.validate(SchemaSpec.DRAFT_07);
}
@@ -234,17 +199,17 @@ public NetworkNtValidator() {
}
@Benchmark
- public Result measureDraft_4_NetworkNt(final NetworkNtValidator validator) {
+ public Result measureDraft_04_NetworkNt(final NetworkNtValidator validator) {
return validator.validate(SchemaSpec.DRAFT_04);
}
@Benchmark
- public Result measureDraft_6_NetworkNt(final NetworkNtValidator validator) {
+ public Result measureDraft_06_NetworkNt(final NetworkNtValidator validator) {
return validator.validate(SchemaSpec.DRAFT_06);
}
@Benchmark
- public Result measureDraft_7_NetworkNt(final NetworkNtValidator validator) {
+ public Result measureDraft_07_NetworkNt(final NetworkNtValidator validator) {
return validator.validate(SchemaSpec.DRAFT_07);
}
@@ -266,12 +231,12 @@ public SnowValidator() {
}
@Benchmark
- public Result measureDraft_6_Snow(final SnowValidator validator) {
+ public Result measureDraft_06_Snow(final SnowValidator validator) {
return validator.validate(SchemaSpec.DRAFT_06);
}
@Benchmark
- public Result measureDraft_7_Snow(final SnowValidator validator) {
+ public Result measureDraft_07_Snow(final SnowValidator validator) {
return validator.validate(SchemaSpec.DRAFT_07);
}
@@ -288,17 +253,17 @@ public JustifyValidator() {
}
@Benchmark
- public Result measureDraft_4_Justify(final JustifyValidator validator) {
+ public Result measureDraft_04_Justify(final JustifyValidator validator) {
return validator.validate(SchemaSpec.DRAFT_04);
}
@Benchmark
- public Result measureDraft_6_Justify(final JustifyValidator validator) {
+ public Result measureDraft_06_Justify(final JustifyValidator validator) {
return validator.validate(SchemaSpec.DRAFT_06);
}
@Benchmark
- public Result measureDraft_7_Justify(final JustifyValidator validator) {
+ public Result measureDraft_07_Justify(final JustifyValidator validator) {
return validator.validate(SchemaSpec.DRAFT_07);
}
diff --git a/src/main/java/org/creekservice/kafka/test/perf/performance/util/JsonToMarkdownConvertor.java b/src/main/java/org/creekservice/kafka/test/perf/performance/util/JsonToMarkdownConvertor.java
new file mode 100644
index 0000000..bf7f981
--- /dev/null
+++ b/src/main/java/org/creekservice/kafka/test/perf/performance/util/JsonToMarkdownConvertor.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2023 Creek Contributors (https://github.com/creek-service)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.creekservice.kafka.test.perf.performance.util;
+
+import static java.lang.System.lineSeparator;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Objects.requireNonNull;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.creekservice.kafka.test.perf.performance.util.model.PerformanceResult;
+
+public final class JsonToMarkdownConvertor {
+
+ private static final List HEADINGS =
+ List.of("Benchmark", "Mode", "Score", "Score Error (99.9%)", "Unit");
+
+ private final PerformanceJsonReader reader;
+
+ public JsonToMarkdownConvertor() {
+ this(new PerformanceJsonReader());
+ }
+
+ JsonToMarkdownConvertor(final PerformanceJsonReader reader) {
+ this.reader = requireNonNull(reader, "reader");
+ }
+
+ /**
+ * Convert a JSON performance result file into markdown.
+ *
+ * @param jsonResult the JSON performance result data to read. Generated by JMH.
+ * @param reportRoot the directory markdown performance results should be written to.
+ */
+ public void convert(final Path jsonResult, final Path reportRoot) {
+ writeMarkdown(convert(jsonResult), reportRoot);
+ }
+
+ Map convert(final Path jsonResult) {
+ final Map> splitResults =
+ splitResults(reader.read(jsonResult));
+ return splitResults.entrySet().stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, e -> convert(e.getValue())));
+ }
+
+ private static String convert(final List results) {
+ final StringBuilder builder = new StringBuilder();
+ builder.append(headings());
+ results.forEach(result -> builder.append(row(result)));
+ return builder.toString();
+ }
+
+ private static Map> splitResults(
+ final PerformanceResult[] results) {
+ return Arrays.stream(results).collect(Collectors.groupingBy(result -> result.testClass()));
+ }
+
+ private static void writeMarkdown(final Map converted, final Path reportRoot) {
+ converted.forEach(
+ (testName, reportContent) ->
+ writeMarkdown(reportContent, reportRoot.resolve(testName + ".md")));
+ }
+
+ private static void writeMarkdown(final String markdown, final Path file) {
+ try {
+ Files.write(file, markdown.getBytes(UTF_8));
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to write to " + file, e);
+ }
+ }
+
+ private static String headings() {
+ final String headings = HEADINGS.stream().collect(Collectors.joining(" | ", "| ", " |"));
+ final String nextLine =
+ HEADINGS.stream()
+ .map(heading -> "-".repeat(heading.length()))
+ .collect(Collectors.joining("-|-", "|-", "-|"));
+ return headings + lineSeparator() + nextLine + lineSeparator();
+ }
+
+ private static String row(final PerformanceResult result) {
+ final List values =
+ List.of(
+ result.testCase(),
+ result.mode(),
+ formatDecimal(result.metric().score()),
+ result.metric()
+ .scoreError()
+ .map(JsonToMarkdownConvertor::formatDecimal)
+ .orElse(""),
+ result.metric().scoreUnit());
+ final String row = values.stream().collect(Collectors.joining(" | ", "| ", " |"));
+ return row + lineSeparator();
+ }
+
+ private static String formatDecimal(final BigDecimal decimal) {
+ final int sigFig = 5;
+ final int exponent = decimal.precision() - decimal.scale();
+ final int scale = Math.max(0, sigFig - exponent);
+ final BigDecimal scaledNumber = decimal.setScale(scale, RoundingMode.HALF_EVEN);
+ return scaledNumber.toPlainString();
+ }
+}
diff --git a/src/main/java/org/creekservice/kafka/test/perf/performance/util/PerformanceDataValidator.java b/src/main/java/org/creekservice/kafka/test/perf/performance/util/PerformanceDataValidator.java
new file mode 100644
index 0000000..0b2fcb0
--- /dev/null
+++ b/src/main/java/org/creekservice/kafka/test/perf/performance/util/PerformanceDataValidator.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2023 Creek Contributors (https://github.com/creek-service)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.creekservice.kafka.test.perf.performance.util;
+
+import static java.util.Objects.requireNonNull;
+
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.creekservice.kafka.test.perf.implementations.Implementation;
+import org.creekservice.kafka.test.perf.implementations.Implementations;
+import org.creekservice.kafka.test.perf.performance.util.model.PerformanceResult;
+import org.creekservice.kafka.test.perf.testsuite.SchemaSpec;
+
+/**
+ * Validate the output results will be compatible and match expected patterns used in
+ * performance.md.
+ */
+public final class PerformanceDataValidator {
+
+ private final PerformanceJsonReader reader;
+ private final List implementations;
+ private final Set specs;
+
+ public PerformanceDataValidator() {
+ this(new PerformanceJsonReader());
+ }
+
+ PerformanceDataValidator(final PerformanceJsonReader reader) {
+ this.reader = requireNonNull(reader, "reader");
+ this.implementations = List.copyOf(Implementations.all());
+ this.specs =
+ Arrays.stream(SchemaSpec.values())
+ .map(SchemaSpec::capitalisedName)
+ .collect(Collectors.toCollection(LinkedHashSet::new));
+ }
+
+ public void validate(final Path jsonResults) {
+ final PerformanceResult[] results = reader.read(jsonResults);
+
+ for (final PerformanceResult result : results) {
+ validate(result);
+ }
+ }
+
+ // Benchmark function name should be in format: measure_
+ // e.g. measureDraft_4_Medeia, or measureDraft_2020_12_Skema
+ private static final Pattern METHOD_PATTERN =
+ Pattern.compile(
+ "measure(?Draft[_0-9]+)_(?"
+ + Implementation.MetaData.SHORT_NAME_PATTERN.pattern()
+ + ")");
+
+ private void validate(final PerformanceResult result) {
+ final Matcher matcher = METHOD_PATTERN.matcher(result.testCase());
+ if (!matcher.matches()) {
+ throw new ValidationException(
+ result,
+ "with a name that does not match the expected pattern."
+ + " Expected pattern: "
+ + METHOD_PATTERN.pattern());
+ }
+
+ validateDraft(matcher.group("draft"), result);
+ validateImplementationName(matcher.group("impl"), result);
+ }
+
+ private void validateDraft(final String draft, final PerformanceResult result) {
+ if (!specs.contains(draft)) {
+ throw new ValidationException(
+ result,
+ "with a name that does not contain a valid schema specification draft."
+ + System.lineSeparator()
+ + "Available versions: "
+ + specs
+ + System.lineSeparator()
+ + "Detected version: "
+ + draft);
+ }
+ }
+
+ private void validateImplementationName(
+ final String implShortName, final PerformanceResult result) {
+ if (implementations.stream()
+ .noneMatch(impl -> impl.metadata().shortName().equals(implShortName))) {
+ throw new ValidationException(
+ result,
+ "with a name that not end with a known implementation's short name."
+ + System.lineSeparator()
+ + "Detected short name: "
+ + implShortName);
+ }
+ }
+
+ private static final class ValidationException extends IllegalArgumentException {
+ ValidationException(final PerformanceResult result, final String msg) {
+ super(
+ "The JSON benchmark results contain a benchmark method "
+ + msg
+ + System.lineSeparator()
+ + "Method name: "
+ + result.testClass()
+ + "."
+ + result.testCase());
+ }
+ }
+}
diff --git a/src/main/java/org/creekservice/kafka/test/perf/performance/util/PerformanceJsonReader.java b/src/main/java/org/creekservice/kafka/test/perf/performance/util/PerformanceJsonReader.java
new file mode 100644
index 0000000..b989295
--- /dev/null
+++ b/src/main/java/org/creekservice/kafka/test/perf/performance/util/PerformanceJsonReader.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2023 Creek Contributors (https://github.com/creek-service)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.creekservice.kafka.test.perf.performance.util;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.json.JsonMapper;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.creekservice.kafka.test.perf.performance.util.model.PerformanceResult;
+
+final class PerformanceJsonReader {
+
+ PerformanceResult[] read(final Path jsonResult) {
+ return parseJson(readJson(jsonResult));
+ }
+
+ private static String readJson(final Path jsonResult) {
+ try {
+ return Files.readString(jsonResult, UTF_8);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to read from " + jsonResult, e);
+ }
+ }
+
+ static PerformanceResult[] parseJson(final String jsonResult) {
+ try {
+ final ObjectMapper mapper =
+ JsonMapper.builder()
+ .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
+ .enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS)
+ .build();
+ return mapper.readValue(jsonResult, PerformanceResult[].class);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to read from " + jsonResult, e);
+ }
+ }
+}
diff --git a/src/main/java/org/creekservice/kafka/test/perf/performance/util/model/Metric.java b/src/main/java/org/creekservice/kafka/test/perf/performance/util/model/Metric.java
new file mode 100644
index 0000000..68bc008
--- /dev/null
+++ b/src/main/java/org/creekservice/kafka/test/perf/performance/util/model/Metric.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2023 Creek Contributors (https://github.com/creek-service)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.creekservice.kafka.test.perf.performance.util.model;
+
+import static java.util.Objects.requireNonNull;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.math.BigDecimal;
+import java.util.Objects;
+import java.util.Optional;
+
+@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
+public final class Metric {
+
+ private final BigDecimal score;
+ private final Optional scoreError;
+ private final String scoreUnit;
+
+ @JsonCreator
+ public Metric(
+ @JsonProperty(value = "score", required = true) final BigDecimal score,
+ @JsonProperty(value = "scoreError", required = true) final Object scoreError,
+ @JsonProperty(value = "scoreUnit", required = true) final String scoreUnit) {
+ this.score = requireNonNull(score, "score");
+ this.scoreError = optionalDecimal(requireNonNull(scoreError, "scoreError"));
+ this.scoreUnit = requireNonNull(scoreUnit, "scoreUnit");
+ }
+
+ public BigDecimal score() {
+ return score;
+ }
+
+ public Optional scoreError() {
+ return scoreError;
+ }
+
+ public String scoreUnit() {
+ return scoreUnit;
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ final Metric metric = (Metric) o;
+ return Objects.equals(score, metric.score)
+ && Objects.equals(scoreError, metric.scoreError)
+ && Objects.equals(scoreUnit, metric.scoreUnit);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(score, scoreError, scoreUnit);
+ }
+
+ private static Optional optionalDecimal(final Object decimal) {
+ return BigDecimal.class.isAssignableFrom(decimal.getClass())
+ ? Optional.of((BigDecimal) decimal)
+ : Optional.empty(); // Handles "NaN" case.
+ }
+}
diff --git a/src/main/java/org/creekservice/kafka/test/perf/performance/util/model/PerformanceResult.java b/src/main/java/org/creekservice/kafka/test/perf/performance/util/model/PerformanceResult.java
new file mode 100644
index 0000000..f07c104
--- /dev/null
+++ b/src/main/java/org/creekservice/kafka/test/perf/performance/util/model/PerformanceResult.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2023 Creek Contributors (https://github.com/creek-service)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.creekservice.kafka.test.perf.performance.util.model;
+
+import static java.util.Objects.requireNonNull;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Objects;
+
+public final class PerformanceResult {
+
+ private static final String PERFORMANCE_PACKAGE =
+ "org.creekservice.kafka.test.perf.performance.";
+
+ private final String testClass;
+ private final String testCase;
+
+ private final String mode;
+ private final Metric metric;
+
+ @JsonCreator
+ public PerformanceResult(
+ @JsonProperty(value = "benchmark", required = true) final String benchmark,
+ @JsonProperty(value = "mode", required = true) final String mode,
+ @JsonProperty(value = "primaryMetric", required = true) final Metric primaryMetric) {
+ this.testClass = extractTestClass(requireNonNull(benchmark, "benchmark"));
+ this.testCase = extractTestCase(benchmark);
+ this.mode = requireNonNull(mode, "mode");
+ this.metric = requireNonNull(primaryMetric, "primaryMetric");
+ }
+
+ public String testClass() {
+ return testClass;
+ }
+
+ public String testCase() {
+ return testCase;
+ }
+
+ public String mode() {
+ return mode;
+ }
+
+ public Metric metric() {
+ return metric;
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ final PerformanceResult that = (PerformanceResult) o;
+ return Objects.equals(testClass, that.testClass)
+ && Objects.equals(testCase, that.testCase)
+ && Objects.equals(mode, that.mode)
+ && Objects.equals(metric, that.metric);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(testClass, testCase, mode, metric);
+ }
+
+ private static String extractTestClass(final String benchmark) {
+ final String test = benchmark.substring(PERFORMANCE_PACKAGE.length());
+ return test.substring(0, test.indexOf("."));
+ }
+
+ private static String extractTestCase(final String benchmark) {
+ return benchmark.substring(benchmark.lastIndexOf(".") + 1);
+ }
+}
diff --git a/src/main/java/org/creekservice/kafka/test/perf/testsuite/SchemaSpec.java b/src/main/java/org/creekservice/kafka/test/perf/testsuite/SchemaSpec.java
index 2324251..4218233 100644
--- a/src/main/java/org/creekservice/kafka/test/perf/testsuite/SchemaSpec.java
+++ b/src/main/java/org/creekservice/kafka/test/perf/testsuite/SchemaSpec.java
@@ -84,6 +84,10 @@ public URI uri() {
return uri;
}
+ public String capitalisedName() {
+ return Character.toUpperCase(name().charAt(0)) + name().substring(1).toLowerCase();
+ }
+
public static Optional fromDir(final String dirName) {
return Arrays.stream(values()).filter(spec -> spec.dirName.equals(dirName)).findAny();
}
diff --git a/src/main/java/org/creekservice/kafka/test/perf/testsuite/output/PerDraftSummary.java b/src/main/java/org/creekservice/kafka/test/perf/testsuite/output/PerDraftSummary.java
index b2ce6aa..3b9ad6e 100644
--- a/src/main/java/org/creekservice/kafka/test/perf/testsuite/output/PerDraftSummary.java
+++ b/src/main/java/org/creekservice/kafka/test/perf/testsuite/output/PerDraftSummary.java
@@ -17,24 +17,19 @@
package org.creekservice.kafka.test.perf.testsuite.output;
import static java.lang.System.lineSeparator;
-import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toMap;
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import java.nio.file.Path;
-import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.function.BinaryOperator;
import java.util.stream.Collectors;
-import java.util.stream.Stream;
import org.creekservice.api.test.util.TestPaths;
import org.creekservice.kafka.test.perf.implementations.Implementation;
import org.creekservice.kafka.test.perf.testsuite.JsonSchemaTestSuite;
import org.creekservice.kafka.test.perf.testsuite.SchemaSpec;
import org.creekservice.kafka.test.perf.util.Table;
-import org.jetbrains.annotations.NotNull;
public final class PerDraftSummary {
@@ -42,83 +37,31 @@ public final class PerDraftSummary {
TestPaths.moduleRoot("json-schema-validation-comparison")
.resolve("build/json-schema-test-suite/tests");
- private final Map results;
+ private final Map results;
public PerDraftSummary(final Map results) {
this.results =
results.entrySet().stream()
- .flatMap(e -> buildResults(e.getKey(), e.getValue()))
.collect(
toMap(
- Map.Entry::getKey,
- e -> e.getValue().build(),
+ e -> e.getKey().metadata().shortName(),
+ e -> new ImplTables(e.getValue()),
throwOnDuplicate(),
TreeMap::new));
}
public String toMarkdown() {
return results.entrySet().stream()
- .map(
- e ->
- "#### "
- + e.getKey()
- + lineSeparator()
- + lineSeparator()
- + e.getValue().toMarkdown())
+ .map(e -> "#### " + e.getKey() + lineSeparator() + e.getValue().toMarkdown())
.collect(Collectors.joining(lineSeparator()));
}
- private Stream> buildResults(
- final Implementation impl, final JsonSchemaTestSuite.Result results) {
- final Map output = new TreeMap<>();
- results.visit(
- (spec, result) -> {
- output.computeIfAbsent(
- new Key(spec, impl.metadata().shortName()), k -> new Builder())
- .add(result, spec);
- });
- return output.entrySet().stream();
- }
-
- private static BinaryOperator throwOnDuplicate() {
+ private static BinaryOperator throwOnDuplicate() {
return (m1, m2) -> {
throw new IllegalStateException("Duplicate!");
};
}
- @SuppressFBWarnings("EQ_COMPARETO_USE_OBJECT_EQUALS")
- private static final class Key implements Comparable {
-
- private static final Comparator COMPARATOR =
- Comparator.comparing(Key::spec).thenComparing(Key::impl);
-
- private final SchemaSpec spec;
- private final String impl;
-
- private Key(final SchemaSpec spec, final String impl) {
- this.spec = requireNonNull(spec, "spec");
- this.impl = requireNonNull(impl, "impl");
- }
-
- SchemaSpec spec() {
- return spec;
- }
-
- String impl() {
- return impl;
- }
-
- @Override
- public int compareTo(@NotNull final Key o) {
- return COMPARATOR.compare(this, o);
- }
-
- @Override
- public String toString() {
- return impl + ": " + spec;
- }
- }
-
private static class Counts {
private int pass;
private int fail;
@@ -156,4 +99,37 @@ public Table build() {
return table;
}
}
+
+ private static class ImplTables {
+
+ private final Map tables;
+
+ ImplTables(final JsonSchemaTestSuite.Result results) {
+ final Map output = new TreeMap<>();
+ results.visit(
+ (spec, result) ->
+ output.computeIfAbsent(spec, k -> new Builder()).add(result, spec));
+
+ this.tables =
+ output.entrySet().stream()
+ .collect(
+ Collectors.toMap(
+ Map.Entry::getKey,
+ e -> e.getValue().build(),
+ throwOnDuplicate(),
+ TreeMap::new));
+ }
+
+ public String toMarkdown() {
+ return tables.entrySet().stream()
+ .map(
+ e ->
+ "##### "
+ + e.getKey().capitalisedName()
+ + lineSeparator()
+ + lineSeparator()
+ + e.getValue().toMarkdown())
+ .collect(Collectors.joining(lineSeparator()));
+ }
+ }
}
diff --git a/src/test/java/org/creekservice/kafka/test/perf/performance/util/JsonToMarkdownConvertorTest.java b/src/test/java/org/creekservice/kafka/test/perf/performance/util/JsonToMarkdownConvertorTest.java
new file mode 100644
index 0000000..4f5be70
--- /dev/null
+++ b/src/test/java/org/creekservice/kafka/test/perf/performance/util/JsonToMarkdownConvertorTest.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2023 Creek Contributors (https://github.com/creek-service)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.creekservice.kafka.test.perf.performance.util;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.when;
+
+import java.math.BigDecimal;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Map;
+import org.creekservice.kafka.test.perf.performance.util.model.Metric;
+import org.creekservice.kafka.test.perf.performance.util.model.PerformanceResult;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+@ExtendWith(MockitoExtension.class)
+class JsonToMarkdownConvertorTest {
+
+ private static final String EXPECTED_HEADINGS =
+ "| Benchmark | Mode | Score | Score Error (99.9%) | Unit |\n"
+ + "|-----------|------|-------|---------------------|------|\n";
+
+ private static final Path SOME_PATH = Paths.get("some/path");
+
+ @Mock private PerformanceJsonReader reader;
+ private JsonToMarkdownConvertor convertor;
+
+ @BeforeEach
+ void setUp() {
+ convertor = new JsonToMarkdownConvertor(reader);
+ }
+
+ @Test
+ void shouldConvertJsonToMarkdown() {
+ // Given:
+ when(reader.read(SOME_PATH))
+ .thenReturn(
+ new PerformanceResult[] {
+ new PerformanceResult(
+ "org.creekservice.kafka.test.perf.performance.JsonValidateBenchmark.measureDraft_4_Medeia",
+ "avgt",
+ new Metric(
+ new BigDecimal("0.34276444437738995"),
+ new BigDecimal("0.0038394222791281593"),
+ "ms/op")),
+ new PerformanceResult(
+ "org.creekservice.kafka.test.perf.performance.JsonValidateBenchmark.measureDraft_7_Medeia",
+ "avgt",
+ new Metric(
+ new BigDecimal("0.893598359837538"),
+ new BigDecimal("0.0035983789573"),
+ "ms/op")),
+ new PerformanceResult(
+ "org.creekservice.kafka.test.perf.performance.JsonSerdeBenchmark.measureEveritRoundTrip",
+ "diff",
+ new Metric(
+ new BigDecimal("2135454.1245"),
+ new BigDecimal("0.003536745566"),
+ "us/op"))
+ });
+
+ // When:
+ final Map results = convertor.convert(SOME_PATH);
+
+ // Then:
+ assertThat(
+ results,
+ is(
+ Map.of(
+ "JsonValidateBenchmark",
+ EXPECTED_HEADINGS
+ + "| measureDraft_4_Medeia | avgt | 0.34276 | 0.0038394 |"
+ + " ms/op |\n"
+ + "| measureDraft_7_Medeia | avgt | 0.89360 | 0.0035984 |"
+ + " ms/op |\n",
+ "JsonSerdeBenchmark",
+ EXPECTED_HEADINGS
+ + "| measureEveritRoundTrip | diff | 2135454 | 0.0035367 |"
+ + " us/op |\n")));
+ }
+
+ @Test
+ void shouldHandleNaN() {
+ // Given:
+ when(reader.read(SOME_PATH))
+ .thenReturn(
+ new PerformanceResult[] {
+ new PerformanceResult(
+ "org.creekservice.kafka.test.perf.performance.JsonValidateBenchmark.measureDraft_7_Medeia",
+ "avgt",
+ new Metric(new BigDecimal("0.893602424"), "NaN", "ms/op"))
+ });
+
+ // When:
+ final Map results = convertor.convert(SOME_PATH);
+
+ // Then:
+ assertThat(
+ results,
+ is(
+ Map.of(
+ "JsonValidateBenchmark",
+ EXPECTED_HEADINGS
+ + "| measureDraft_7_Medeia | avgt | 0.89360 | | ms/op"
+ + " |\n")));
+ }
+}
diff --git a/src/test/java/org/creekservice/kafka/test/perf/performance/util/PerformanceDataValidatorTest.java b/src/test/java/org/creekservice/kafka/test/perf/performance/util/PerformanceDataValidatorTest.java
new file mode 100644
index 0000000..35fa150
--- /dev/null
+++ b/src/test/java/org/creekservice/kafka/test/perf/performance/util/PerformanceDataValidatorTest.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright 2023 Creek Contributors (https://github.com/creek-service)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.creekservice.kafka.test.perf.performance.util;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mock.Strictness.LENIENT;
+import static org.mockito.Mockito.when;
+
+import java.nio.file.Path;
+import org.creekservice.kafka.test.perf.performance.util.model.PerformanceResult;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+@ExtendWith(MockitoExtension.class)
+class PerformanceDataValidatorTest {
+
+ private static final Path SOME_PATH = Path.of("some/path");
+ @Mock private PerformanceJsonReader reader;
+
+ @Mock(strictness = LENIENT)
+ private PerformanceResult result1;
+
+ private PerformanceDataValidator validator;
+
+ @BeforeEach
+ void setUp() {
+ validator = new PerformanceDataValidator(reader);
+
+ when(reader.read(SOME_PATH)).thenReturn(new PerformanceResult[] {result1});
+ when(result1.testClass()).thenReturn("JsonTestBenchmark");
+ }
+
+ @Test
+ void shouldThrowIfBenchmarkMethodDoesNotMatchPattern() {
+ // Given:
+ when(result1.testCase()).thenReturn("invalidPattern");
+
+ // When:
+ final Exception e =
+ assertThrows(IllegalArgumentException.class, () -> validator.validate(SOME_PATH));
+
+ // Then:
+ assertThat(
+ e.getMessage(),
+ is(
+ "The JSON benchmark results contain a benchmark method with a name that"
+ + " does not match the expected pattern. Expected pattern:"
+ + " measure(?Draft[_0-9]+)_(?[A-Za-z0-9]+)\n"
+ + "Method name: JsonTestBenchmark.invalidPattern"));
+ }
+
+ @Test
+ void shouldThrowOnUnknownDraftVersion() {
+ // Given:
+ when(result1.testCase()).thenReturn("measureDraft_11_Snow");
+
+ // When:
+ final Exception e =
+ assertThrows(IllegalArgumentException.class, () -> validator.validate(SOME_PATH));
+
+ // Then:
+ assertThat(
+ e.getMessage(),
+ is(
+ "The JSON benchmark results contain a benchmark method with a name that"
+ + " does not contain a valid schema specification draft.\n"
+ + "Available versions: [Draft_03, Draft_04, Draft_06, Draft_07,"
+ + " Draft_2019_09, Draft_2020_12]\n"
+ + "Detected version: Draft_11\n"
+ + "Method name: JsonTestBenchmark.measureDraft_11_Snow"));
+ }
+
+ @Test
+ void shouldThrowOnUnknownImplementation() {
+ // Given:
+ when(result1.testCase()).thenReturn("measureDraft_07_InvalidImpl");
+
+ // When:
+ final Exception e =
+ assertThrows(IllegalArgumentException.class, () -> validator.validate(SOME_PATH));
+
+ // Then:
+ assertThat(
+ e.getMessage(),
+ is(
+ "The JSON benchmark results contain a benchmark method with a name that not"
+ + " end with a known implementation's short name.\n"
+ + "Detected short name: InvalidImpl\n"
+ + "Method name: JsonTestBenchmark.measureDraft_07_InvalidImpl"));
+ }
+
+ @Test
+ void shouldParseJson() {
+ // Given:
+ when(result1.testCase()).thenReturn("measureDraft_2020_12_Vertx");
+
+ // When:
+ validator.validate(SOME_PATH);
+
+ // Then: did not throw.
+ }
+}
diff --git a/src/test/java/org/creekservice/kafka/test/perf/performance/util/PerformanceJsonReaderTest.java b/src/test/java/org/creekservice/kafka/test/perf/performance/util/PerformanceJsonReaderTest.java
new file mode 100644
index 0000000..5ecbd2d
--- /dev/null
+++ b/src/test/java/org/creekservice/kafka/test/perf/performance/util/PerformanceJsonReaderTest.java
@@ -0,0 +1,179 @@
+/*
+ * Copyright 2023 Creek Contributors (https://github.com/creek-service)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.creekservice.kafka.test.perf.performance.util;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.arrayContaining;
+
+import java.math.BigDecimal;
+import org.creekservice.kafka.test.perf.performance.util.model.Metric;
+import org.creekservice.kafka.test.perf.performance.util.model.PerformanceResult;
+import org.junit.jupiter.api.Test;
+
+class PerformanceJsonReaderTest {
+
+ private static final String JSON_RESULT =
+ "[\n"
+ +
+ // One complete one:
+ " {\n"
+ + " \"jmhVersion\" : \"1.36\",\n"
+ + " \"benchmark\" :"
+ + " \"org.creekservice.kafka.test.perf.performance.JsonValidateBenchmark.measureDraft_4_Medeia\",\n"
+ + " \"mode\" : \"avgt\",\n"
+ + " \"threads\" : 4,\n"
+ + " \"forks\" : 4,\n"
+ + " \"jvm\" :"
+ + " \"/opt/homebrew/Cellar/openjdk@17/17.0.9/libexec/openjdk.jdk/Contents/Home/bin/java\",\n"
+ + " \"jvmArgs\" : [\n"
+ + " \"-Dfile.encoding=UTF-8\",\n"
+ + " \"-Duser.country=GB\",\n"
+ + " \"-Duser.language=en\",\n"
+ + " \"-Duser.variant\"\n"
+ + " ],\n"
+ + " \"jdkVersion\" : \"17.0.9\",\n"
+ + " \"vmName\" : \"OpenJDK 64-Bit Server VM\",\n"
+ + " \"vmVersion\" : \"17.0.9+0\",\n"
+ + " \"warmupIterations\" : 5,\n"
+ + " \"warmupTime\" : \"10 s\",\n"
+ + " \"warmupBatchSize\" : 1,\n"
+ + " \"measurementIterations\" : 5,\n"
+ + " \"measurementTime\" : \"10 s\",\n"
+ + " \"measurementBatchSize\" : 1,\n"
+ + " \"primaryMetric\" : {\n"
+ + " \"score\" : 0.34276444437738995,\n"
+ + " \"scoreError\" : 0.0038394222791281593,\n"
+ + " \"scoreConfidence\" : [\n"
+ + " 0.3389250220982618,\n"
+ + " 0.3466038666565181\n"
+ + " ],\n"
+ + " \"scorePercentiles\" : {\n"
+ + " \"0.0\" : 0.3341750081828707,\n"
+ + " \"50.0\" : 0.3432545406943275,\n"
+ + " \"90.0\" : 0.3482507649509782,\n"
+ + " \"95.0\" : 0.34973757670327116,\n"
+ + " \"99.0\" : 0.34981339737699124,\n"
+ + " \"99.9\" : 0.34981339737699124,\n"
+ + " \"99.99\" : 0.34981339737699124,\n"
+ + " \"99.999\" : 0.34981339737699124,\n"
+ + " \"99.9999\" : 0.34981339737699124,\n"
+ + " \"100.0\" : 0.34981339737699124\n"
+ + " },\n"
+ + " \"scoreUnit\" : \"ms/op\",\n"
+ + " \"rawData\" : [\n"
+ + " [\n"
+ + " 0.3465202925610167,\n"
+ + " 0.34249890961701457,\n"
+ + " 0.34450357678071153,\n"
+ + " 0.34075528555055234,\n"
+ + " 0.3425288988276069\n"
+ + " ],\n"
+ + " [\n"
+ + " 0.34829698390259,\n"
+ + " 0.34396076146988674,\n"
+ + " 0.3433999185464469,\n"
+ + " 0.34549801455831286,\n"
+ + " 0.34390681847990284\n"
+ + " ],\n"
+ + " [\n"
+ + " 0.3430043540612691,\n"
+ + " 0.33511882592058606,\n"
+ + " 0.33611264096826354,\n"
+ + " 0.3341750081828707,\n"
+ + " 0.33599361359531954\n"
+ + " ],\n"
+ + " [\n"
+ + " 0.3478347943864718,\n"
+ + " 0.34221756760803823,\n"
+ + " 0.3460400623117395,\n"
+ + " 0.34981339737699124,\n"
+ + " 0.343109162842208\n"
+ + " ]\n"
+ + " ]\n"
+ + " }\n"
+ + " },\n"
+ // One minimal:
+ + " {\n"
+ + " \"benchmark\" :"
+ + " \"org.creekservice.kafka.test.perf.performance.JsonValidateBenchmark.measureDraft_7_Medeia\",\n"
+ + " \"mode\" : \"diff\",\n"
+ + " \"primaryMetric\" : {\n"
+ + " \"score\" : 0.893598359837538,\n"
+ + " \"scoreError\" : 0.0035983789573,\n"
+ + " \"scoreUnit\" : \"ms/op\"\n"
+ + " },\n"
+ + " \"secondaryMetrics\" : {\n"
+ + " }\n"
+ + " }\n"
+ + "]";
+
+ private static final String JSON_RESULT_WITH_NAN =
+ "[\n"
+ + " {\n"
+ + " \"benchmark\" :"
+ + " \"org.creekservice.kafka.test.perf.performance.JsonValidateBenchmark.measureDraft_7_Medeia\",\n"
+ + " \"mode\" : \"avgt\",\n"
+ + " \"primaryMetric\" : {\n"
+ + " \"score\" : 0.893598359837538,\n"
+ + " \"scoreError\" : \"NaN\",\n"
+ + " \"scoreUnit\" : \"ms/op\"\n"
+ + " },\n"
+ + " \"secondaryMetrics\" : {\n"
+ + " }\n"
+ + " }\n"
+ + "]";
+
+ @Test
+ void shouldParseJson() {
+ // When:
+ final PerformanceResult[] results = PerformanceJsonReader.parseJson(JSON_RESULT);
+
+ // Then:
+ assertThat(
+ results,
+ arrayContaining(
+ new PerformanceResult(
+ "org.creekservice.kafka.test.perf.performance.JsonValidateBenchmark.measureDraft_4_Medeia",
+ "avgt",
+ new Metric(
+ new BigDecimal("0.34276444437738995"),
+ new BigDecimal("0.0038394222791281593"),
+ "ms/op")),
+ new PerformanceResult(
+ "org.creekservice.kafka.test.perf.performance.JsonValidateBenchmark.measureDraft_7_Medeia",
+ "diff",
+ new Metric(
+ new BigDecimal("0.893598359837538"),
+ new BigDecimal("0.0035983789573"),
+ "ms/op"))));
+ }
+
+ @Test
+ void shouldHandleNaN() {
+ // When:
+ final PerformanceResult[] results = PerformanceJsonReader.parseJson(JSON_RESULT_WITH_NAN);
+
+ // Then:
+ assertThat(
+ results,
+ arrayContaining(
+ new PerformanceResult(
+ "org.creekservice.kafka.test.perf.performance.JsonValidateBenchmark.measureDraft_7_Medeia",
+ "avgt",
+ new Metric(new BigDecimal("0.893598359837538"), "NaN", "ms/op"))));
+ }
+}
diff --git a/src/test/java/org/creekservice/kafka/test/perf/util/ImplsJsonFormatterTest.java b/src/test/java/org/creekservice/kafka/test/perf/util/ImplsJsonFormatterTest.java
index e6809e2..4721cbe 100644
--- a/src/test/java/org/creekservice/kafka/test/perf/util/ImplsJsonFormatterTest.java
+++ b/src/test/java/org/creekservice/kafka/test/perf/util/ImplsJsonFormatterTest.java
@@ -37,7 +37,7 @@ class ImplsJsonFormatterTest {
private static final Implementation.MetaData MD_A =
new Implementation.MetaData(
"Implementation A",
- "Impl_A",
+ "ImplA",
Implementation.Language.Java,
Implementation.Licence.Apache_v2_0,
Set.of(SchemaSpec.DRAFT_2019_09, SchemaSpec.DRAFT_04),
@@ -47,7 +47,7 @@ class ImplsJsonFormatterTest {
private static final Implementation.MetaData MD_B =
new Implementation.MetaData(
"Implementation B",
- "Impl_B",
+ "ImplB",
Implementation.Language.Java,
Implementation.Licence.Apache_v2_0,
Set.of(SchemaSpec.DRAFT_07),
@@ -76,14 +76,14 @@ void shouldFormatAsJson() {
json,
is(
"[{\"longName\":\"Implementation A\","
- + "\"shortName\":\"Impl_A\","
+ + "\"shortName\":\"ImplA\","
+ "\"language\":\"Java\","
+ "\"licence\":\"Apache Licence 2.0\","
+ "\"supported\":[\"DRAFT_04\",\"DRAFT_2019_09\"],"
+ "\"url\":\"http://a\","
+ "\"color\":\"rgb(0,0,0)\"},"
+ "{\"longName\":\"Implementation B\","
- + "\"shortName\":\"Impl_B\","
+ + "\"shortName\":\"ImplB\","
+ "\"language\":\"Java\","
+ "\"licence\":\"Apache Licence 2.0\","
+ "\"supported\":[\"DRAFT_07\"],"