From 8e2482a314c9bef4a248e1411cb7e4eeaf80d671 Mon Sep 17 00:00:00 2001 From: Andy Coates <8012398+big-andy-coates@users.noreply.github.com> Date: Mon, 9 Oct 2023 15:25:41 +0100 Subject: [PATCH 1/3] =?UTF-8?q?Functional=20and=20performance=20comparison?= =?UTF-8?q?=20of=20JSON=20serde=20and=20validation=20li=E2=80=A6=20(#22)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Functional and performance comparison of JSON serde and validation libraries. * Add dummy coveralls task to keep standard build file happy * More logging and new workflows * Get git clone working from workflow cloning from unauthenticated account. * Get schema draft loading working again. Looks like json org added redirects to https, which isn't supported in old code. * Add results to job summary * Formatting * Working dummy coveralls task --- .github/workflows/build.yml | 2 +- .github/workflows/run-func-test.yml | 27 ++++++++++ .github/workflows/run-perf-test.yml | 29 ++++++++++ build.gradle.kts | 33 +++++++++--- .../perf/testsuite/JsonTestSuiteMain.java | 5 ++ .../kafka/test/perf/testsuite/SchemaSpec.java | 53 ++++++++++++++++--- 6 files changed, 133 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/run-func-test.yml create mode 100644 .github/workflows/run-perf-test.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 01c0767..7cf226f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,7 +45,7 @@ jobs: - name: Build env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} - run: ./gradlew build coveralls + run: ./gradlew build coveralls --stacktrace - name: Publish if: github.event_name == 'push' || github.event.inputs.publish_artifacts == 'true' env: diff --git a/.github/workflows/run-func-test.yml b/.github/workflows/run-func-test.yml new file mode 100644 index 0000000..45fe2ff --- /dev/null +++ b/.github/workflows/run-func-test.yml @@ -0,0 +1,27 @@ +# This workflow run the functional test + +name: Func Test + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: gradle/wrapper-validation-action@56b90f209b02bf6d1deae490e9ef18b21a389cd4 # v1.1.0 + - name: Set up JDK + uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 # v3.12.0 + with: + java-version: '17' + distribution: 'adopt' + - name: Setup Gradle + uses: gradle/gradle-build-action@a4cf152f482c7ca97ef56ead29bf08bcd953284c # v2.7.0 + with: + gradle-home-cache-cleanup: true + - name: Run + run: ./gradlew --quiet runFunctionalTests >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/.github/workflows/run-perf-test.yml b/.github/workflows/run-perf-test.yml new file mode 100644 index 0000000..ae6df2b --- /dev/null +++ b/.github/workflows/run-perf-test.yml @@ -0,0 +1,29 @@ +# This workflow run the performance test + +name: Perf Test + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: gradle/wrapper-validation-action@56b90f209b02bf6d1deae490e9ef18b21a389cd4 # v1.1.0 + - name: Set up JDK + uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 # v3.12.0 + with: + java-version: '17' + distribution: 'adopt' + - name: Setup Gradle + uses: gradle/gradle-build-action@a4cf152f482c7ca97ef56ead29bf08bcd953284c # v2.7.0 + with: + gradle-home-cache-cleanup: true + - name: Run + run: | + ./gradlew runBenchmarks + cat benchmark_results.txt >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index 57233d9..2b7e825 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -71,7 +71,7 @@ dependencies { implementation("org.leadpony.justify:justify:3.1.0") - implementation("org.apache.logging.log4j:log4j-core:$log4jVersion"); + implementation("org.apache.logging.log4j:log4j-core:$log4jVersion") runtimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:$log4jVersion") testImplementation("org.creekservice:creek-test-hamcrest:$creekVersion") @@ -86,7 +86,6 @@ dependencies { testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:$junitVersion") } - tasks.withType { options.compilerArgs.add("-Xlint:all,-serial,-requires-automatic,-requires-transitive-automatic,-module,-processing") } @@ -101,7 +100,7 @@ val cloneTask = tasks.register("clone-json-schema-test-suite") { doLast { org.ajoberstar.grgit.Grgit.clone { dir = jsonSchemaTestSuiteDir.get().asFile - uri = "git@github.com:json-schema-org/JSON-Schema-Test-Suite.git" + uri = "https://github.com/json-schema-org/JSON-Schema-Test-Suite.git" } } } @@ -110,7 +109,6 @@ val pullTask = tasks.register("pull-json-schema-test-suite") { dependsOn(cloneTask) doLast { - println("pulling.........") org.ajoberstar.grgit.Grgit.open { dir = jsonSchemaTestSuiteDir.get().asFile }.pull() @@ -120,25 +118,46 @@ val pullTask = tasks.register("pull-json-schema-test-suite") { val runFunctionalTests = tasks.register("runFunctionalTests") { classpath = sourceSets.main.get().runtimeClasspath mainClass.set("org.creekservice.kafka.test.perf.testsuite.JsonTestSuiteMain") - args = listOf(jsonSchemaTestSuiteDir.get().asFile.absolutePath); + args = listOf(jsonSchemaTestSuiteDir.get().asFile.absolutePath) dependsOn(pullTask) } tasks.register("runBenchmarks") { classpath = sourceSets.main.get().runtimeClasspath mainClass.set("org.creekservice.kafka.test.perf.BenchmarkRunner") + args(listOf( + // Output results in text format + "-rf", "text", + // To a named file + "-rff", "benchmark_results.txt" + )) dependsOn(pullTask) } val benchmarkSmokeTest = tasks.register("runBenchmarkSmokeTest") { classpath = sourceSets.main.get().runtimeClasspath mainClass.set("org.creekservice.kafka.test.perf.BenchmarkRunner") - args(listOf("-wi", "0", "-i", "1", "-t", "1", "-r", "1s")) + args(listOf( + // No warmup: + "-wi", "0", + // Single test iteration: + "-i", "1", + // On a single thread: + "-t", "1", + // Running for 1 second + "-r", "1s", + // With forking disabled + "-f", "0" + )) dependsOn(pullTask) } +tasks.register("coveralls") { + // dummy +} + tasks.test { - dependsOn(pullTask, runFunctionalTests, benchmarkSmokeTest) + dependsOn(runFunctionalTests, benchmarkSmokeTest) } // Below is required until the following is fixed in IntelliJ: diff --git a/src/main/java/org/creekservice/kafka/test/perf/testsuite/JsonTestSuiteMain.java b/src/main/java/org/creekservice/kafka/test/perf/testsuite/JsonTestSuiteMain.java index c5e8bd8..2e79daa 100644 --- a/src/main/java/org/creekservice/kafka/test/perf/testsuite/JsonTestSuiteMain.java +++ b/src/main/java/org/creekservice/kafka/test/perf/testsuite/JsonTestSuiteMain.java @@ -37,9 +37,14 @@ import org.creekservice.kafka.test.perf.testsuite.JsonSchemaTestSuite.TestPredicate; import org.creekservice.kafka.test.perf.testsuite.output.PerDraftSummary; import org.creekservice.kafka.test.perf.testsuite.output.Summary; +import org.creekservice.kafka.test.perf.util.Logging; public final class JsonTestSuiteMain { + static { + Logging.disable(); + } + private static final List IMPLS = List.of( new EveritSerde(), diff --git a/src/main/java/org/creekservice/kafka/test/perf/testsuite/SchemaSpec.java b/src/main/java/org/creekservice/kafka/test/perf/testsuite/SchemaSpec.java index dcaedff..2324251 100644 --- a/src/main/java/org/creekservice/kafka/test/perf/testsuite/SchemaSpec.java +++ b/src/main/java/org/creekservice/kafka/test/perf/testsuite/SchemaSpec.java @@ -20,14 +20,17 @@ import static java.util.stream.Collectors.toMap; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStreamReader; import java.io.UncheckedIOException; +import java.net.HttpURLConnection; import java.net.URI; +import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Map; import java.util.Optional; -import java.util.Scanner; import java.util.Set; import java.util.function.Function; @@ -61,13 +64,13 @@ public enum SchemaSpec { private final String dirName; private final URI uri; private final String content; - private final Map additonal; + private final Map additional; SchemaSpec(final String dirName, final String uri, final Set additional) { this.dirName = requireNonNull(dirName, "dirName"); this.uri = URI.create(uri); this.content = loadContent(this.uri); - this.additonal = + this.additional = additional.stream() .map(URI::create) .collect(toMap(Function.identity(), SchemaSpec::loadContent)); @@ -97,7 +100,7 @@ private Optional getContentFromUri(final URI uri) { if (normalize(this.uri).equals(normalized)) { return Optional.of(content); } - final String content = additonal.get(normalized); + final String content = additional.get(normalized); return content == null ? Optional.empty() : Optional.of(content); } @@ -109,11 +112,45 @@ private static URI normalize(final URI uri) { return uri; } - @SuppressFBWarnings("URLCONNECTION_SSRF_FD") + @SuppressFBWarnings( + value = "URLCONNECTION_SSRF_FD", + justification = "only called with hardcoded urls") private static String loadContent(final URI uri) { - try (Scanner scanner = new Scanner(uri.toURL().openStream(), StandardCharsets.UTF_8)) { - scanner.useDelimiter("\\A"); - return scanner.hasNext() ? scanner.next() : ""; + try { + // Always load from https, as non-secure http redirect to https: + final URL url = + uri.getScheme().equals("http") + ? new URL("https" + uri.toString().substring(4)) + : uri.toURL(); + + final HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setInstanceFollowRedirects(true); + final int responseCode = connection.getResponseCode(); + + if (responseCode != HttpURLConnection.HTTP_OK) { + throw new UncheckedIOException( + new IOException( + "Failed to load content from " + uri + ", code: " + responseCode)); + } + + try (BufferedReader reader = + new BufferedReader( + new InputStreamReader( + connection.getInputStream(), StandardCharsets.UTF_8))) { + final StringBuilder builder = new StringBuilder(); + + String line; + while ((line = reader.readLine()) != null) { + builder.append(line).append(System.lineSeparator()); + } + + final String content = builder.toString(); + if (content.isBlank()) { + throw new UncheckedIOException( + new IOException("Blank content loaded from " + uri)); + } + return content; + } } catch (IOException e) { throw new UncheckedIOException(e); } From f252d17c5a4ea0c0b5d43ea71e1cec7ed9cd6b42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Oct 2023 14:31:07 +0000 Subject: [PATCH 2/3] Bump gradle/gradle-build-action from 2.7.0 to 2.9.0 (#24) Bumps [gradle/gradle-build-action](https://github.com/gradle/gradle-build-action) from 2.7.0 to 2.9.0. - [Release notes](https://github.com/gradle/gradle-build-action/releases) - [Commits](https://github.com/gradle/gradle-build-action/compare/a4cf152f482c7ca97ef56ead29bf08bcd953284c...842c587ad8aa4c68eeba24c396e15af4c2e9f30a) --- updated-dependencies: - dependency-name: gradle/gradle-build-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- .github/workflows/run-func-test.yml | 2 +- .github/workflows/run-perf-test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7cf226f..c8e07db 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -39,7 +39,7 @@ jobs: java-version: '17' distribution: 'adopt' - name: Setup Gradle - uses: gradle/gradle-build-action@a4cf152f482c7ca97ef56ead29bf08bcd953284c # v2.7.0 + uses: gradle/gradle-build-action@842c587ad8aa4c68eeba24c396e15af4c2e9f30a # v2.9.0 with: gradle-home-cache-cleanup: true - name: Build diff --git a/.github/workflows/run-func-test.yml b/.github/workflows/run-func-test.yml index 45fe2ff..db68bd8 100644 --- a/.github/workflows/run-func-test.yml +++ b/.github/workflows/run-func-test.yml @@ -20,7 +20,7 @@ jobs: java-version: '17' distribution: 'adopt' - name: Setup Gradle - uses: gradle/gradle-build-action@a4cf152f482c7ca97ef56ead29bf08bcd953284c # v2.7.0 + uses: gradle/gradle-build-action@842c587ad8aa4c68eeba24c396e15af4c2e9f30a # v2.9.0 with: gradle-home-cache-cleanup: true - name: Run diff --git a/.github/workflows/run-perf-test.yml b/.github/workflows/run-perf-test.yml index ae6df2b..b5bfd56 100644 --- a/.github/workflows/run-perf-test.yml +++ b/.github/workflows/run-perf-test.yml @@ -20,7 +20,7 @@ jobs: java-version: '17' distribution: 'adopt' - name: Setup Gradle - uses: gradle/gradle-build-action@a4cf152f482c7ca97ef56ead29bf08bcd953284c # v2.7.0 + uses: gradle/gradle-build-action@842c587ad8aa4c68eeba24c396e15af4c2e9f30a # v2.9.0 with: gradle-home-cache-cleanup: true - name: Run From bcba0371ee9a8fd8988732199e88da979f4cf5aa Mon Sep 17 00:00:00 2001 From: Andy Coates <8012398+big-andy-coates@users.noreply.github.com> Date: Mon, 9 Oct 2023 16:05:12 +0100 Subject: [PATCH 3/3] Add conclusions and graphs (#29) * Add conclusions and graphs * Add dummy tasks to get the build green. --- README.md | 48 ++++++++++++++++++--- build.gradle.kts | 11 +++-- img/Feature comparison score.svg | 2 +- img/JsonSerdeBenchmark Results.svg | 1 + img/JsonValidateBenchmark-Draft-2019-0.svg | 1 + img/JsonValidateBenchmark-Draft-2020-12.svg | 1 + img/JsonValidateBenchmark-Draft-4.svg | 1 + img/JsonValidateBenchmark-Draft-6.svg | 1 + img/JsonValidateBenchmark-Draft-7.svg | 1 + 9 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 img/JsonSerdeBenchmark Results.svg create mode 100644 img/JsonValidateBenchmark-Draft-2019-0.svg create mode 100644 img/JsonValidateBenchmark-Draft-2020-12.svg create mode 100644 img/JsonValidateBenchmark-Draft-4.svg create mode 100644 img/JsonValidateBenchmark-Draft-6.svg create mode 100644 img/JsonValidateBenchmark-Draft-7.svg diff --git a/README.md b/README.md index 7637c5a..2745f74 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,11 @@ This repo tests the following implementations of JSON schema validation: | [worldturner/medeia-validator][7] | Kotlin | draft-07, -06, -04 | Apache License 2.0 | | [erosb/json-sKema][8] | Kotlin | 2020-12 | MIT | +## Note to maintainers + +If you are the maintainer of one of the above implementations, please feel free to raise a PR if you feel your +implementation is poorly represented due to issues with the code in this repo. + ## Feature comparison To run the comparison: `./gradlew runFunctionalTests` @@ -140,7 +145,7 @@ whereas _optional_ features only account for a maximum 25% of the score. ### Feature comparison conclusions -`ScheamFriend` comes out as the clear winner of the functional test, with support for all Schema specification, at the time of writing, _and_ the highest overall score. +At the time of writing, `ScheamFriend` comes out as the clear winner of the functional test, with support for all Schema specification, at the time of writing, _and_ the highest overall score. Ignoring which implementations support which drafts for a moment, a rough ranking on functionality would be: @@ -229,6 +234,24 @@ JsonValidateBenchmark.measureDraft_7_Vertx avgt 20 2.141 ± ``` Note: results from running on 2021 Macbook Pro, M1 Max: 2.06 - 3.22 GHz, in High Power mode, JDK 17.0.6 +Each of the following graphs compares the average time it took each implementation to validate all of its **positive** +test cases. + +The following caveats apply to the results: +1. The `Snow` implementation has been removed from the graphs, as its so slow that it makes the graph unreadable when trying to compare the other implementations. +2. Comparison of time between the different drafts, i.e. between the different charts, is fairly meaningless, as the number of tests changes. Latter drafts generally have move test cases, meaning they take longer to run. +3. When comparing times a graph, remember that the time only covers each implementation's positive test cases. This means implementations with less functional coverage have less positive cases to handle. + +![JsonValidateBenchmark-Draft-4.svg](img/JsonValidateBenchmark-Draft-4.svg) + +![JsonValidateBenchmark-Draft-6.svg](img/JsonValidateBenchmark-Draft-6.svg) + +![JsonValidateBenchmark-Draft-7.svg](img/JsonValidateBenchmark-Draft-7.svg) + +![JsonValidateBenchmark-Draft-2019-0.svg](img/JsonValidateBenchmark-Draft-2019-0.svg) + +![JsonValidateBenchmark-Draft-2020-12.svg](img/JsonValidateBenchmark-Draft-2020-12.svg) + ### Schema validated JSON (de)serialization benchmark The `JsonSerdeBenchmark` benchmark measures the average time taken to serialize a simple Java object, including polymorphism, to JSON and back, @@ -254,17 +277,27 @@ JsonSerdeBenchmark.measureVertxRoundTrip avgt 20 514.517 ± ``` Note: results from running on 2021 Macbook Pro, M1 Max: 2.06 - 3.22 GHz, in High Power mode, JDK 17.0.6 +![JsonSerdeBenchmark Results.svg](img%2FJsonSerdeBenchmark%20Results.svg) + ### Performance comparison conclusions -Coming soon... +At the time of writing, `Medeia` comes as a clear winner for speed, with `Everit` not far behind. +However, these implementations look to no longer be maintained, or are deprecated, respectively. +Plus, neither of them handle the latest drafts of the JSON schema standard. +If `Medeia` and `Everit` are excluded, then the clear winner is `SchemaFriend`. -## Overall comparison +## Conclusions -Coming soon... +Hopefully this comparison is useful. The intended use-case will likely dictate which implementation(s) are suitable. -## Conclusions +If your use-case requires ultimate speed, doesn't require advanced features or support for the later draft specifications, +and you're happy with the maintenance risk associated with them, then either `Medeia` or `Everit` may be the implementation for you. +It's worth pointing out that [Confluent][confluent]'s own JSON serde internally use `Everit`, which may mean they'll be helping to support it going forward. + +Alternatively, if you're either uneasy using deprecated or unmaintained libraries, or need more functionality or support for the latest drafts, +then these tests would suggest you take a look at `SchemaFriend`: it comes out to for functionality and is only beaten on performance by the unmaintained or deprecated `Medeia` and `Everit`. -Coming soon... +Note: The author of this repository is not affiliated with any of the implementations covered by this test suite. [1]: https://github.com/eclipse-vertx/vertx-json-schema [2]: https://github.com/jimblackler/jsonschemafriend @@ -275,4 +308,5 @@ Coming soon... [7]: https://github.com/worldturner/medeia-validator [8]: https://github.com/erosb/json-sKema [JSON-Schema-Test-Suite]: https://github.com/json-schema-org/JSON-Schema-Test-Suite -[jhm]: https://github.com/openjdk/jmh \ No newline at end of file +[jhm]: https://github.com/openjdk/jmh +[confluent]: https://www.confluent.io/ \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index 2b7e825..c53053d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -152,14 +152,17 @@ val benchmarkSmokeTest = tasks.register("runBenchmarkSmokeTest") { dependsOn(pullTask) } -tasks.register("coveralls") { - // dummy -} - tasks.test { dependsOn(runFunctionalTests, benchmarkSmokeTest) } +// Dummy / empty tasks required to allow the repo to use the same standard GitHub workflows as other Creek repos: +tasks.register("coveralls") +tasks.register("cV") +tasks.register("publish") +tasks.register("closeAndReleaseStagingRepository") +tasks.register("publishPlugins") + // Below is required until the following is fixed in IntelliJ: // https://youtrack.jetbrains.com/issue/IDEA-316081/Gradle-8-toolchain-error-Toolchain-from-executable-property-does-not-match-toolchain-from-javaLauncher-property-when-different gradle.taskGraph.whenReady { diff --git a/img/Feature comparison score.svg b/img/Feature comparison score.svg index 3034d46..40dc889 100644 --- a/img/Feature comparison score.svg +++ b/img/Feature comparison score.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/img/JsonSerdeBenchmark Results.svg b/img/JsonSerdeBenchmark Results.svg new file mode 100644 index 0000000..c26ec35 --- /dev/null +++ b/img/JsonSerdeBenchmark Results.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/img/JsonValidateBenchmark-Draft-2019-0.svg b/img/JsonValidateBenchmark-Draft-2019-0.svg new file mode 100644 index 0000000..45c2cb3 --- /dev/null +++ b/img/JsonValidateBenchmark-Draft-2019-0.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/img/JsonValidateBenchmark-Draft-2020-12.svg b/img/JsonValidateBenchmark-Draft-2020-12.svg new file mode 100644 index 0000000..cc1b978 --- /dev/null +++ b/img/JsonValidateBenchmark-Draft-2020-12.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/img/JsonValidateBenchmark-Draft-4.svg b/img/JsonValidateBenchmark-Draft-4.svg new file mode 100644 index 0000000..ffcb753 --- /dev/null +++ b/img/JsonValidateBenchmark-Draft-4.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/img/JsonValidateBenchmark-Draft-6.svg b/img/JsonValidateBenchmark-Draft-6.svg new file mode 100644 index 0000000..79989ee --- /dev/null +++ b/img/JsonValidateBenchmark-Draft-6.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/img/JsonValidateBenchmark-Draft-7.svg b/img/JsonValidateBenchmark-Draft-7.svg new file mode 100644 index 0000000..935eba7 --- /dev/null +++ b/img/JsonValidateBenchmark-Draft-7.svg @@ -0,0 +1 @@ + \ No newline at end of file