Merge branch 'main' into dependabot/gradle/confluentVersion-7.5.0

.github/workflows/build.yml

@@ -39,13 +39,13 @@ jobs:
           java-version: '17'
           distribution: 'adopt'
       - name: Setup Gradle
-        uses: gradle/gradle-build-action@a4cf152f482c7ca97ef56ead29bf08bcd953284c # v2.7.0
+        uses: gradle/gradle-build-action@842c587ad8aa4c68eeba24c396e15af4c2e9f30a # v2.9.0
         with:
           gradle-home-cache-cleanup: true
       - name: Build
         env:
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
-        run: ./gradlew build coveralls
+        run: ./gradlew build coveralls --stacktrace
       - name: Publish
         if: github.event_name == 'push' || github.event.inputs.publish_artifacts == 'true'
         env:

.github/workflows/run-func-test.yml

@@ -0,0 +1,27 @@
+# This workflow run the functional test
+
+name: Func Test
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      - uses: gradle/wrapper-validation-action@56b90f209b02bf6d1deae490e9ef18b21a389cd4 # v1.1.0
+      - name: Set up JDK
+        uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 # v3.12.0
+        with:
+          java-version: '17'
+          distribution: 'adopt'
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@842c587ad8aa4c68eeba24c396e15af4c2e9f30a # v2.9.0
+        with:
+          gradle-home-cache-cleanup: true
+      - name: Run
+        run: ./gradlew --quiet runFunctionalTests >> $GITHUB_STEP_SUMMARY

.github/workflows/run-perf-test.yml

@@ -0,0 +1,29 @@
+# This workflow run the performance test
+
+name: Perf Test
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      - uses: gradle/wrapper-validation-action@56b90f209b02bf6d1deae490e9ef18b21a389cd4 # v1.1.0
+      - name: Set up JDK
+        uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 # v3.12.0
+        with:
+          java-version: '17'
+          distribution: 'adopt'
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@842c587ad8aa4c68eeba24c396e15af4c2e9f30a # v2.9.0
+        with:
+          gradle-home-cache-cleanup: true
+      - name: Run
+        run: |
+          ./gradlew runBenchmarks
+          cat benchmark_results.txt >> $GITHUB_STEP_SUMMARY

README.md

@@ -20,6 +20,11 @@ This repo tests the following implementations of JSON schema validation:
 | [worldturner/medeia-validator][7]    | Kotlin     | draft-07, -06, -04                       | Apache License 2.0                     |
 | [erosb/json-sKema][8]                | Kotlin     | 2020-12                                  | MIT                                    |
 
+## Note to maintainers
+
+If you are the maintainer of one of the above implementations, please feel free to raise a PR if you feel your
+implementation is poorly represented due to issues with the code in this repo.
+
 ## Feature comparison
 
 To run the comparison: `./gradlew runFunctionalTests`
@@ -140,7 +145,7 @@ whereas _optional_ features only account for a maximum 25% of the score.
 
 ### Feature comparison conclusions
 
-`ScheamFriend` comes out as the clear winner of the functional test, with support for all Schema specification, at the time of writing, _and_ the highest overall score.
+At the time of writing, `ScheamFriend` comes out as the clear winner of the functional test, with support for all Schema specification, at the time of writing, _and_ the highest overall score.
 
 Ignoring which implementations support which drafts for a moment, a rough ranking on functionality would be:
 
@@ -229,6 +234,24 @@ JsonValidateBenchmark.measureDraft_7_Vertx               avgt   20     2.141 ±
 ```
 Note: results from running on 2021 Macbook Pro, M1 Max: 2.06 - 3.22 GHz, in High Power mode, JDK 17.0.6
 
+Each of the following graphs compares the average time it took each implementation to validate all of its **positive**
+test cases.
+
+The following caveats apply to the results:
+1. The `Snow` implementation has been removed from the graphs, as its so slow that it makes the graph unreadable when trying to compare the other implementations.
+2. Comparison of time between the different drafts, i.e. between the different charts, is fairly meaningless, as the number of tests changes. Latter drafts generally have move test cases, meaning they take longer to run.
+3. When comparing times a graph, remember that the time only covers each implementation's positive test cases. This means implementations with less functional coverage have less positive cases to handle.   
+
+![JsonValidateBenchmark-Draft-4.svg](img/JsonValidateBenchmark-Draft-4.svg)
+
+![JsonValidateBenchmark-Draft-6.svg](img/JsonValidateBenchmark-Draft-6.svg)
+
+![JsonValidateBenchmark-Draft-7.svg](img/JsonValidateBenchmark-Draft-7.svg)
+
+![JsonValidateBenchmark-Draft-2019-0.svg](img/JsonValidateBenchmark-Draft-2019-0.svg)
+
+![JsonValidateBenchmark-Draft-2020-12.svg](img/JsonValidateBenchmark-Draft-2020-12.svg)
+
 ### Schema validated JSON (de)serialization benchmark
 
 The `JsonSerdeBenchmark` benchmark measures the average time taken to serialize a simple Java object, including polymorphism, to JSON and back,
@@ -254,17 +277,27 @@ JsonSerdeBenchmark.measureVertxRoundTrip                 avgt   20   514.517 ±
 ```
 Note: results from running on 2021 Macbook Pro, M1 Max: 2.06 - 3.22 GHz, in High Power mode, JDK 17.0.6
 
+![JsonSerdeBenchmark Results.svg](img%2FJsonSerdeBenchmark%20Results.svg)
+
 ### Performance comparison conclusions
 
-Coming soon...
+At the time of writing, `Medeia` comes as a clear winner for speed, with `Everit` not far behind.
+However, these implementations look to no longer be maintained, or are deprecated, respectively.
+Plus, neither of them handle the latest drafts of the JSON schema standard.
+If `Medeia` and `Everit` are excluded, then the clear winner is `SchemaFriend`.
 
-## Overall comparison
+## Conclusions
 
-Coming soon...
+Hopefully this comparison is useful. The intended use-case will likely dictate which implementation(s) are suitable.
 
-## Conclusions
+If your use-case requires ultimate speed, doesn't require advanced features or support for the later draft specifications, 
+and you're happy with the maintenance risk associated with them, then either `Medeia` or `Everit` may be the implementation for you.
+It's worth pointing out that [Confluent][confluent]'s own JSON serde internally use `Everit`, which may mean they'll be helping to support it going forward.
+
+Alternatively, if you're either uneasy using deprecated or unmaintained libraries, or need more functionality or support for the latest drafts, 
+then these tests would suggest you take a look at `SchemaFriend`: it comes out to for functionality and is only beaten on performance by the unmaintained or deprecated `Medeia` and `Everit`. 
 
-Coming soon...
+Note: The author of this repository is not affiliated with any of the implementations covered by this test suite.
 
 [1]: https://github.com/eclipse-vertx/vertx-json-schema
 [2]: https://github.com/jimblackler/jsonschemafriend
@@ -275,4 +308,5 @@ Coming soon...
 [7]: https://github.com/worldturner/medeia-validator
 [8]: https://github.com/erosb/json-sKema
 [JSON-Schema-Test-Suite]: https://github.com/json-schema-org/JSON-Schema-Test-Suite
-[jhm]: https://github.com/openjdk/jmh
+[jhm]: https://github.com/openjdk/jmh
+[confluent]: https://www.confluent.io/

build.gradle.kts

@@ -71,7 +71,7 @@ dependencies {
 
     implementation("org.leadpony.justify:justify:3.1.0")
 
-    implementation("org.apache.logging.log4j:log4j-core:$log4jVersion");
+    implementation("org.apache.logging.log4j:log4j-core:$log4jVersion")
     runtimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:$log4jVersion")
 
     testImplementation("org.creekservice:creek-test-hamcrest:$creekVersion")
@@ -86,7 +86,6 @@ dependencies {
     testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:$junitVersion")
 }
 
-
 tasks.withType<JavaCompile> {
     options.compilerArgs.add("-Xlint:all,-serial,-requires-automatic,-requires-transitive-automatic,-module,-processing")
 }
@@ -101,7 +100,7 @@ val cloneTask = tasks.register("clone-json-schema-test-suite") {
     doLast {
         org.ajoberstar.grgit.Grgit.clone {
             dir = jsonSchemaTestSuiteDir.get().asFile
-            uri = "git@github.com:json-schema-org/JSON-Schema-Test-Suite.git"
+            uri = "https://github.com/json-schema-org/JSON-Schema-Test-Suite.git"
         }
     }
 }
@@ -110,7 +109,6 @@ val pullTask = tasks.register("pull-json-schema-test-suite") {
     dependsOn(cloneTask)
 
     doLast {
-        println("pulling.........")
         org.ajoberstar.grgit.Grgit.open {
             dir = jsonSchemaTestSuiteDir.get().asFile
         }.pull()
@@ -120,27 +118,51 @@ val pullTask = tasks.register("pull-json-schema-test-suite") {
 val runFunctionalTests = tasks.register<JavaExec>("runFunctionalTests") {
     classpath = sourceSets.main.get().runtimeClasspath
     mainClass.set("org.creekservice.kafka.test.perf.testsuite.JsonTestSuiteMain")
-    args = listOf(jsonSchemaTestSuiteDir.get().asFile.absolutePath);
+    args = listOf(jsonSchemaTestSuiteDir.get().asFile.absolutePath)
     dependsOn(pullTask)
 }
 
 tasks.register<JavaExec>("runBenchmarks") {
     classpath = sourceSets.main.get().runtimeClasspath
     mainClass.set("org.creekservice.kafka.test.perf.BenchmarkRunner")
+    args(listOf(
+        // Output results in text format
+        "-rf", "text",
+        // To a named file
+        "-rff", "benchmark_results.txt"
+    ))
     dependsOn(pullTask)
 }
 
 val benchmarkSmokeTest = tasks.register<JavaExec>("runBenchmarkSmokeTest") {
     classpath = sourceSets.main.get().runtimeClasspath
     mainClass.set("org.creekservice.kafka.test.perf.BenchmarkRunner")
-    args(listOf("-wi", "0", "-i", "1", "-t", "1", "-r", "1s"))
+    args(listOf(
+        // No warmup:
+        "-wi", "0",
+        // Single test iteration:
+        "-i", "1",
+        // On a single thread:
+        "-t", "1",
+        // Running for 1 second
+        "-r", "1s",
+        // With forking disabled
+        "-f", "0"
+    ))
     dependsOn(pullTask)
 }
 
 tasks.test {
-    dependsOn(pullTask, runFunctionalTests, benchmarkSmokeTest)
+    dependsOn(runFunctionalTests, benchmarkSmokeTest)
 }
 
+// Dummy / empty tasks required to allow the repo to use the same standard GitHub workflows as other Creek repos:
+tasks.register("coveralls")
+tasks.register("cV")
+tasks.register("publish")
+tasks.register("closeAndReleaseStagingRepository")
+tasks.register("publishPlugins")
+
 // Below is required until the following is fixed in IntelliJ:
 // https://youtrack.jetbrains.com/issue/IDEA-316081/Gradle-8-toolchain-error-Toolchain-from-executable-property-does-not-match-toolchain-from-javaLauncher-property-when-different
 gradle.taskGraph.whenReady {

src/main/java/org/creekservice/kafka/test/perf/testsuite/JsonTestSuiteMain.java

@@ -37,9 +37,14 @@
 import org.creekservice.kafka.test.perf.testsuite.JsonSchemaTestSuite.TestPredicate;
 import org.creekservice.kafka.test.perf.testsuite.output.PerDraftSummary;
 import org.creekservice.kafka.test.perf.testsuite.output.Summary;
+import org.creekservice.kafka.test.perf.util.Logging;
 
 public final class JsonTestSuiteMain {
 
+    static {
+        Logging.disable();
+    }
+
     private static final List<SerdeImpl> IMPLS =
             List.of(
                     new EveritSerde(),

src/main/java/org/creekservice/kafka/test/perf/testsuite/SchemaSpec.java

@@ -20,14 +20,17 @@
 import static java.util.stream.Collectors.toMap;
 
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
+import java.net.HttpURLConnection;
 import java.net.URI;
+import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Scanner;
 import java.util.Set;
 import java.util.function.Function;
 
@@ -61,13 +64,13 @@ public enum SchemaSpec {
     private final String dirName;
     private final URI uri;
     private final String content;
-    private final Map<URI, String> additonal;
+    private final Map<URI, String> additional;
 
     SchemaSpec(final String dirName, final String uri, final Set<String> additional) {
         this.dirName = requireNonNull(dirName, "dirName");
         this.uri = URI.create(uri);
         this.content = loadContent(this.uri);
-        this.additonal =
+        this.additional =
                 additional.stream()
                         .map(URI::create)
                         .collect(toMap(Function.identity(), SchemaSpec::loadContent));
@@ -97,7 +100,7 @@ private Optional<String> getContentFromUri(final URI uri) {
         if (normalize(this.uri).equals(normalized)) {
             return Optional.of(content);
         }
-        final String content = additonal.get(normalized);
+        final String content = additional.get(normalized);
         return content == null ? Optional.empty() : Optional.of(content);
     }
 
@@ -109,11 +112,45 @@ private static URI normalize(final URI uri) {
         return uri;
     }
 
-    @SuppressFBWarnings("URLCONNECTION_SSRF_FD")
+    @SuppressFBWarnings(
+            value = "URLCONNECTION_SSRF_FD",
+            justification = "only called with hardcoded urls")
     private static String loadContent(final URI uri) {
-        try (Scanner scanner = new Scanner(uri.toURL().openStream(), StandardCharsets.UTF_8)) {
-            scanner.useDelimiter("\\A");
-            return scanner.hasNext() ? scanner.next() : "";
+        try {
+            // Always load from https, as non-secure http redirect to https:
+            final URL url =
+                    uri.getScheme().equals("http")
+                            ? new URL("https" + uri.toString().substring(4))
+                            : uri.toURL();
+
+            final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+            connection.setInstanceFollowRedirects(true);
+            final int responseCode = connection.getResponseCode();
+
+            if (responseCode != HttpURLConnection.HTTP_OK) {
+                throw new UncheckedIOException(
+                        new IOException(
+                                "Failed to load content from " + uri + ", code: " + responseCode));
+            }
+
+            try (BufferedReader reader =
+                    new BufferedReader(
+                            new InputStreamReader(
+                                    connection.getInputStream(), StandardCharsets.UTF_8))) {
+                final StringBuilder builder = new StringBuilder();
+
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    builder.append(line).append(System.lineSeparator());
+                }
+
+                final String content = builder.toString();
+                if (content.isBlank()) {
+                    throw new UncheckedIOException(
+                            new IOException("Blank content loaded from " + uri));
+                }
+                return content;
+            }
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }