From 76e7b0e025969cd41ec5a148c88a5db02f7d6307 Mon Sep 17 00:00:00 2001 From: Big Andy <8012398+big-andy-coates@users.noreply.github.com> Date: Wed, 15 Nov 2023 10:14:36 +0000 Subject: [PATCH 1/3] Include DRAFT 2020-12 to serde benchmark fixes: #59 Add 2020-12 draft benchmarks to the serde benchmark for those implementations that support it. --- docs/_docs/3. performance.md | 15 +-- .../perf/performance/JsonSerdeBenchmark.java | 98 +++++++++++++------ 2 files changed, 76 insertions(+), 37 deletions(-) diff --git a/docs/_docs/3. performance.md b/docs/_docs/3. performance.md index 74daf13..8d5d2c0 100644 --- a/docs/_docs/3. performance.md +++ b/docs/_docs/3. performance.md @@ -21,6 +21,10 @@ The first of these benchmark covers a wide range of JSON schema functionality, w real-world example, using a small common subset of functionality, in the context of using schema validated JSON as a serialization format. Combined, these should give a good comparison of performance. +**Note:** +The benchmarks are run on GitHub's own infrastructure. These may not be dedicated machines, which can influence performance results. +{: .notice--warning} + ### JSON schema test suite benchmark This benchmark measures the average time taken to run through all _positive_ test cases in the standard @@ -92,9 +96,8 @@ for comparison. The serialized form is roughly 1KB of JSON, and the schema is roughly 2KB. -The preferred draft specification for this benchmark is `DRAFT 7`. However, not all implementations support this. -Where an implementation does not support `DRAFT 7`, it is tested with `DRAFT 2020-12`. -[Task 59](https://github.com/creek-service/json-schema-validation-comparison/issues/59) will change this to output results for both. +Rather than test every supported schema version, the benchmark covers `DRAFT 7` and `DRAFT 2020-12`, which covers +all currently implementations, at least once. The schema file for `DRAFT 2020-12` can be found [here][2020-schema], and for `DRAFT 7` [here][7-schema]. @@ -102,8 +105,8 @@ Each of the following graphs compares the average time it took each implementati then validate & deserialize the simple Java object, with the following caveats: **Note:** -As different implementations are tested using different versions of the schema specification, -which may be more or less rich than other versions, comparison across specification versions may be misleading. +Newer schema versions are more feature rich, and this can come at a cost. +Comparison of different implementations across specification versions may be misleading. {: .notice--warning}
@@ -154,7 +157,7 @@ which may be more or less rich than other versions, comparison across specificat plugins: { title: { display: true, - text: draft + ' ' + title + ' (less is better)' + text: draft + ' ' + title + ' (lower is better)' }, legend: { display: false diff --git a/src/main/java/org/creekservice/kafka/test/perf/performance/JsonSerdeBenchmark.java b/src/main/java/org/creekservice/kafka/test/perf/performance/JsonSerdeBenchmark.java index d37d265..b62c215 100644 --- a/src/main/java/org/creekservice/kafka/test/perf/performance/JsonSerdeBenchmark.java +++ b/src/main/java/org/creekservice/kafka/test/perf/performance/JsonSerdeBenchmark.java @@ -56,8 +56,8 @@ * use the basic JSON schema features: primitives, enums, arrays, polymorphic types and length * assertions. This can be extended in the future it needed. * - *

The preferred Schema draft is Draft_7. Draft_2020_12 will be used where implementations do not - * support 7. + *

Benchmark methods should be added for Draft_7 and Draft_2020_12 for each implementation that + * supports them. */ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(MICROSECONDS) @@ -80,7 +80,12 @@ public JacksonState() { @Benchmark public TestModel measureDraft_07_Jackson(final JacksonState impl, final ModelState model) { - return impl.roundTrip(model); + return impl.roundTrip(model, SchemaSpec.DRAFT_07); + } + + @Benchmark + public TestModel measureDraft_2020_12_Jackson(final JacksonState impl, final ModelState model) { + return impl.roundTrip(model, SchemaSpec.DRAFT_2020_12); } public static class MedeiaState extends ImplementationState { @@ -91,7 +96,7 @@ public MedeiaState() { @Benchmark public TestModel measureDraft_07_Medeia(final MedeiaState impl, final ModelState model) { - return impl.roundTrip(model); + return impl.roundTrip(model, SchemaSpec.DRAFT_07); } public static class EveritState extends ImplementationState { @@ -102,7 +107,7 @@ public EveritState() { @Benchmark public TestModel measureDraft_07_Everit(final EveritState impl, final ModelState model) { - return impl.roundTrip(model); + return impl.roundTrip(model, SchemaSpec.DRAFT_07); } public static class SkemaState extends ImplementationState { @@ -113,7 +118,7 @@ public SkemaState() { @Benchmark public TestModel measureDraft_2020_12_Skema(final SkemaState impl, final ModelState model) { - return impl.roundTrip(model); + return impl.roundTrip(model, SchemaSpec.DRAFT_2020_12); } public static class VertxState extends ImplementationState { @@ -124,7 +129,12 @@ public VertxState() { @Benchmark public TestModel measureDraft_07_Vertx(final VertxState impl, final ModelState model) { - return impl.roundTrip(model); + return impl.roundTrip(model, SchemaSpec.DRAFT_07); + } + + @Benchmark + public TestModel measureDraft_2020_12_Vertx(final VertxState impl, final ModelState model) { + return impl.roundTrip(model, SchemaSpec.DRAFT_2020_12); } public static class SchemaFriendState extends ImplementationState { @@ -136,7 +146,13 @@ public SchemaFriendState() { @Benchmark public TestModel measureDraft_07_SchemaFriend( final SchemaFriendState impl, final ModelState model) { - return impl.roundTrip(model); + return impl.roundTrip(model, SchemaSpec.DRAFT_07); + } + + @Benchmark + public TestModel measureDraft_2020_12_SchemaFriend( + final SchemaFriendState impl, final ModelState model) { + return impl.roundTrip(model, SchemaSpec.DRAFT_2020_12); } public static class NetworkNtState extends ImplementationState { @@ -147,7 +163,13 @@ public NetworkNtState() { @Benchmark public TestModel measureDraft_07_NetworkNt(final NetworkNtState impl, final ModelState model) { - return impl.roundTrip(model); + return impl.roundTrip(model, SchemaSpec.DRAFT_07); + } + + @Benchmark + public TestModel measureDraft_2020_12_NetworkNt( + final NetworkNtState impl, final ModelState model) { + return impl.roundTrip(model, SchemaSpec.DRAFT_2020_12); } public static class SnowState extends ImplementationState { @@ -158,7 +180,7 @@ public SnowState() { @Benchmark public TestModel measureDraft_07_Snow(final SnowState impl, final ModelState model) { - return impl.roundTrip(model); + return impl.roundTrip(model, SchemaSpec.DRAFT_07); } public static class JustifyState extends ImplementationState { @@ -169,7 +191,7 @@ public JustifyState() { @Benchmark public TestModel measureDraft_07_Justify(final JustifyState impl, final ModelState model) { - return impl.roundTrip(model); + return impl.roundTrip(model, SchemaSpec.DRAFT_07); } public static class DevHarrelState extends ImplementationState { @@ -181,40 +203,54 @@ public DevHarrelState() { @Benchmark public TestModel measureDraft_2020_12_DevHarrel( final DevHarrelState impl, final ModelState model) { - return impl.roundTrip(model); + return impl.roundTrip(model, SchemaSpec.DRAFT_2020_12); } @State(Scope.Thread) private static class ImplementationState { - private final Implementation.JsonValidator validator; + private final Implementation.JsonValidator validator_07; + private final Implementation.JsonValidator validator_2020_12; ImplementationState(final Implementation impl) { - this.validator = buildValidator(impl); + this.validator_07 = + impl.supports(SchemaSpec.DRAFT_07) + ? impl.prepare( + TestSchemas.DRAFT_7_SCHEMA, + SchemaSpec.DRAFT_07, + new AdditionalSchemas(Map.of(), Path.of(""))) + : null; + + this.validator_2020_12 = + impl.supports(SchemaSpec.DRAFT_2020_12) + ? impl.prepare( + TestSchemas.DRAFT_2020_SCHEMA, + SchemaSpec.DRAFT_2020_12, + new AdditionalSchemas(Map.of(), Path.of(""))) + : null; + + if (validator_07 == null && validator_2020_12 == null) { + throw new UnsupportedOperationException( + "Benchmark code needs enhancing to cover this case."); + } } - public TestModel roundTrip(final ModelState model) { + public TestModel roundTrip(final ModelState model, final SchemaSpec version) { + final Implementation.JsonValidator validator = validator(version); final byte[] serialized = validator.serialize(model.model, true); return validator.deserialize(serialized); } - private static Implementation.JsonValidator buildValidator(final Implementation impl) { - if (impl.supports(SchemaSpec.DRAFT_07)) { - return impl.prepare( - TestSchemas.DRAFT_7_SCHEMA, - SchemaSpec.DRAFT_07, - new AdditionalSchemas(Map.of(), Path.of(""))); - } - - if (impl.supports(SchemaSpec.DRAFT_2020_12)) { - return impl.prepare( - TestSchemas.DRAFT_2020_SCHEMA, - SchemaSpec.DRAFT_2020_12, - new AdditionalSchemas(Map.of(), Path.of(""))); + private Implementation.JsonValidator validator(final SchemaSpec version) { + switch (version) { + case DRAFT_07: + return validator_07; + case DRAFT_2020_12: + return validator_2020_12; + default: + throw new UnsupportedOperationException( + "Benchmark code needs enhancing to cover this case."); } - - throw new UnsupportedOperationException( - "Benchmark code needs enhancing to cover this case."); } } } From f79c1f3b53e50a8287a681247040288f18c19d9d Mon Sep 17 00:00:00 2001 From: Big Andy <8012398+big-andy-coates@users.noreply.github.com> Date: Wed, 15 Nov 2023 11:02:50 +0000 Subject: [PATCH 2/3] Fix spotbugs --- README.md | 17 ++++++++++------- config/spotbugs/suppressions.xml | 6 ++++++ .../DevHarrelImplementation.java | 6 ------ .../perf/implementations/Implementation.java | 6 ------ .../implementations/JustifyImplementation.java | 6 ------ .../SchemaFriendImplementation.java | 6 ------ .../perf/performance/JsonSerdeBenchmark.java | 14 +++++++------- 7 files changed, 23 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index eba669d..bd06e21 100644 --- a/README.md +++ b/README.md @@ -22,23 +22,26 @@ Adding a new validator implementation is relatively straight forward and very we 1. First, take a look at the [micro-site][micro-site], as it gives some explanation of what is being tested. 2. Clone the repo and pull it down locally, creating your own branch to work in. 3. Add necessary dependencies to [build.gradle.kts](build.gradle.kts). -4. Add a new implementation of [Implementation](src/main/java/org/creekservice/kafka/test/perf/implementations/Implementation.java) +4. Ensure GitHub's Dependabot will update the new dependency version. + This will ensure the site updates when new versions are released. + See the [Dependabot version updates docs](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/about-dependabot-version-updates). +5. Add a new implementation of [Implementation](src/main/java/org/creekservice/kafka/test/perf/implementations/Implementation.java) to the [main implementations](src/main/java/org/creekservice/kafka/test/perf/implementations) package for the new validator library. See JavaDocs and other implementations for help. -5. Add a unit test class for your new implementation to the [test implementations](src/test/java/org/creekservice/kafka/test/perf/implementations) package. +6. Add a unit test class for your new implementation to the [test implementations](src/test/java/org/creekservice/kafka/test/perf/implementations) package. This should subtype [ImplementationTest.java](src/test/java/org/creekservice/kafka/test/perf/implementations/ImplementationTest.java). The unit test class needs to content. See other implementations for examples. Ensure tests pass! -6. Register your new Implementation type in [Implementations.java](src/main/java/org/creekservice/kafka/test/perf/implementations/Implementations.java). +7. Register your new Implementation type in [Implementations.java](src/main/java/org/creekservice/kafka/test/perf/implementations/Implementations.java). This will ensure the new implementation is included in the docs and included in the functional test -7. Manually add appropriate benchmark methods to [JsonSerdeBenchmark.java](src/main/java/org/creekservice/kafka/test/perf/performance/JsonSerdeBenchmark.java) +8. Manually add appropriate benchmark methods to [JsonSerdeBenchmark.java](src/main/java/org/creekservice/kafka/test/perf/performance/JsonSerdeBenchmark.java) and [JsonValidateBenchmark.java](src/main/java/org/creekservice/kafka/test/perf/performance/JsonValidateBenchmark.java). This is currently manual as JMH library does provide a way to generate these automatically. There should be one test per supported draft version. See JavaDocs and the other methods in these classes for examples. -8. Run `./gradlew` to format your code, perform static analysis and run the tests. +9. Run `./gradlew` to format your code, perform static analysis and run the tests. Ensure this passes! -9. Follow [these instructions](docs) to build and view the website, and ensure your new implementation data is included in tables and charts. -10. Raise a PR with your changes. +10. Follow [these instructions](docs) to build and view the website, and ensure your new implementation data is included in tables and charts. +11. Raise a PR with your changes. ### Running things locally diff --git a/config/spotbugs/suppressions.xml b/config/spotbugs/suppressions.xml index 1d71dbb..12be0f1 100644 --- a/config/spotbugs/suppressions.xml +++ b/config/spotbugs/suppressions.xml @@ -20,4 +20,10 @@ + + + + + + \ No newline at end of file diff --git a/src/main/java/org/creekservice/kafka/test/perf/implementations/DevHarrelImplementation.java b/src/main/java/org/creekservice/kafka/test/perf/implementations/DevHarrelImplementation.java index 98eb620..eacc2c1 100644 --- a/src/main/java/org/creekservice/kafka/test/perf/implementations/DevHarrelImplementation.java +++ b/src/main/java/org/creekservice/kafka/test/perf/implementations/DevHarrelImplementation.java @@ -147,10 +147,4 @@ private Validator validator(final SchemaSpec spec, final AdditionalSchemas addit throw new RuntimeException("Unsupported Spec:" + spec); } } - - // Final, empty finalize method stops spotbugs CT_CONSTRUCTOR_THROW - // Can be moved to base type after https://github.com/spotbugs/spotbugs/issues/2665 - @Override - @SuppressWarnings({"deprecation", "Finalize"}) - protected final void finalize() {} } diff --git a/src/main/java/org/creekservice/kafka/test/perf/implementations/Implementation.java b/src/main/java/org/creekservice/kafka/test/perf/implementations/Implementation.java index 9af27a8..382387b 100644 --- a/src/main/java/org/creekservice/kafka/test/perf/implementations/Implementation.java +++ b/src/main/java/org/creekservice/kafka/test/perf/implementations/Implementation.java @@ -179,12 +179,6 @@ public Set supported() { public String color() { return "rgb(" + color.getRed() + "," + color.getGreen() + "," + color.getBlue() + ")"; } - - // Final, empty finalize method stops spotbugs CT_CONSTRUCTOR_THROW - // Can be moved to base type after https://github.com/spotbugs/spotbugs/issues/2665 - @Override - @SuppressWarnings({"deprecation", "Finalize"}) - protected final void finalize() {} } /** diff --git a/src/main/java/org/creekservice/kafka/test/perf/implementations/JustifyImplementation.java b/src/main/java/org/creekservice/kafka/test/perf/implementations/JustifyImplementation.java index 46e4789..b61f0a5 100644 --- a/src/main/java/org/creekservice/kafka/test/perf/implementations/JustifyImplementation.java +++ b/src/main/java/org/creekservice/kafka/test/perf/implementations/JustifyImplementation.java @@ -149,10 +149,4 @@ private SpecVersion schemaVersion(final SchemaSpec spec) { } return ver; } - - // Final, empty finalize method stops spotbugs CT_CONSTRUCTOR_THROW - // Can be moved to base type after https://github.com/spotbugs/spotbugs/issues/2665 - @Override - @SuppressWarnings({"deprecation", "Finalize"}) - protected final void finalize() {} } diff --git a/src/main/java/org/creekservice/kafka/test/perf/implementations/SchemaFriendImplementation.java b/src/main/java/org/creekservice/kafka/test/perf/implementations/SchemaFriendImplementation.java index 6376385..eb3b28f 100644 --- a/src/main/java/org/creekservice/kafka/test/perf/implementations/SchemaFriendImplementation.java +++ b/src/main/java/org/creekservice/kafka/test/perf/implementations/SchemaFriendImplementation.java @@ -143,10 +143,4 @@ private Schema parseSchema( throw new RuntimeException(e); } } - - // Final, empty finalize method stops spotbugs CT_CONSTRUCTOR_THROW - // Can be moved to base type after https://github.com/spotbugs/spotbugs/issues/2665 - @Override - @SuppressWarnings({"deprecation", "Finalize"}) - protected final void finalize() {} } diff --git a/src/main/java/org/creekservice/kafka/test/perf/performance/JsonSerdeBenchmark.java b/src/main/java/org/creekservice/kafka/test/perf/performance/JsonSerdeBenchmark.java index b62c215..fd0120a 100644 --- a/src/main/java/org/creekservice/kafka/test/perf/performance/JsonSerdeBenchmark.java +++ b/src/main/java/org/creekservice/kafka/test/perf/performance/JsonSerdeBenchmark.java @@ -209,11 +209,11 @@ public TestModel measureDraft_2020_12_DevHarrel( @State(Scope.Thread) private static class ImplementationState { - private final Implementation.JsonValidator validator_07; - private final Implementation.JsonValidator validator_2020_12; + private final Implementation.JsonValidator validator07; + private final Implementation.JsonValidator validator2020; ImplementationState(final Implementation impl) { - this.validator_07 = + this.validator07 = impl.supports(SchemaSpec.DRAFT_07) ? impl.prepare( TestSchemas.DRAFT_7_SCHEMA, @@ -221,7 +221,7 @@ private static class ImplementationState { new AdditionalSchemas(Map.of(), Path.of(""))) : null; - this.validator_2020_12 = + this.validator2020 = impl.supports(SchemaSpec.DRAFT_2020_12) ? impl.prepare( TestSchemas.DRAFT_2020_SCHEMA, @@ -229,7 +229,7 @@ private static class ImplementationState { new AdditionalSchemas(Map.of(), Path.of(""))) : null; - if (validator_07 == null && validator_2020_12 == null) { + if (validator07 == null && validator2020 == null) { throw new UnsupportedOperationException( "Benchmark code needs enhancing to cover this case."); } @@ -244,9 +244,9 @@ public TestModel roundTrip(final ModelState model, final SchemaSpec version) { private Implementation.JsonValidator validator(final SchemaSpec version) { switch (version) { case DRAFT_07: - return validator_07; + return validator07; case DRAFT_2020_12: - return validator_2020_12; + return validator2020; default: throw new UnsupportedOperationException( "Benchmark code needs enhancing to cover this case."); From 19156ea340d99a01c2506e4141f8a432195daf4f Mon Sep 17 00:00:00 2001 From: Big Andy <8012398+big-andy-coates@users.noreply.github.com> Date: Wed, 15 Nov 2023 12:19:18 +0000 Subject: [PATCH 3/3] Fix Jackson and turn on FailOnError --- .../org/creekservice/kafka/test/perf/PerformanceMain.java | 5 ++++- .../test/perf/implementations/JacksonImplementation.java | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/creekservice/kafka/test/perf/PerformanceMain.java b/src/main/java/org/creekservice/kafka/test/perf/PerformanceMain.java index c324bde..a0f9ffb 100644 --- a/src/main/java/org/creekservice/kafka/test/perf/PerformanceMain.java +++ b/src/main/java/org/creekservice/kafka/test/perf/PerformanceMain.java @@ -44,7 +44,10 @@ private static void runBenchmarks(final String[] suppliedArgs) throws IOExceptio "json", // To a named file "-rff", - JSON_RESULTS.toString() + JSON_RESULTS.toString(), + // Fail on Error + "-foe", + "true" }; final String[] allArgs = new String[suppliedArgs.length + additionalArgs.length]; diff --git a/src/main/java/org/creekservice/kafka/test/perf/implementations/JacksonImplementation.java b/src/main/java/org/creekservice/kafka/test/perf/implementations/JacksonImplementation.java index e741450..988eb13 100644 --- a/src/main/java/org/creekservice/kafka/test/perf/implementations/JacksonImplementation.java +++ b/src/main/java/org/creekservice/kafka/test/perf/implementations/JacksonImplementation.java @@ -35,7 +35,7 @@ public class JacksonImplementation implements Implementation { "Jackson", Language.Java, Licence.Apache_v2_0, - Set.of(SchemaSpec.DRAFT_07), + Set.of(SchemaSpec.DRAFT_07, SchemaSpec.DRAFT_2020_12), "https://github.com/FasterXML/jackson-core", new Color(20, 84, 166));