Functional and performance comparison of JSON serde and validation li… (

#22)

* Functional and performance comparison of JSON serde and validation libraries.

* Add dummy coveralls task to keep standard build file happy

* More logging and new workflows

* Get git clone working from workflow

cloning from unauthenticated account.

* Get schema draft loading working again.

Looks like json org added redirects to https, which isn't supported in old code.

* Add results to job summary

* Formatting

* Working dummy coveralls task

.github/workflows/build.yml

@@ -45,7 +45,7 @@ jobs:
       - name: Build
         env:
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
-        run: ./gradlew build coveralls
+        run: ./gradlew build coveralls --stacktrace
       - name: Publish
         if: github.event_name == 'push' || github.event.inputs.publish_artifacts == 'true'
         env:

.github/workflows/run-func-test.yml

@@ -0,0 +1,27 @@
+# This workflow run the functional test
+
+name: Func Test
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      - uses: gradle/wrapper-validation-action@56b90f209b02bf6d1deae490e9ef18b21a389cd4 # v1.1.0
+      - name: Set up JDK
+        uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 # v3.12.0
+        with:
+          java-version: '17'
+          distribution: 'adopt'
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@a4cf152f482c7ca97ef56ead29bf08bcd953284c # v2.7.0
+        with:
+          gradle-home-cache-cleanup: true
+      - name: Run
+        run: ./gradlew --quiet runFunctionalTests >> $GITHUB_STEP_SUMMARY

.github/workflows/run-perf-test.yml

@@ -0,0 +1,29 @@
+# This workflow run the performance test
+
+name: Perf Test
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      - uses: gradle/wrapper-validation-action@56b90f209b02bf6d1deae490e9ef18b21a389cd4 # v1.1.0
+      - name: Set up JDK
+        uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 # v3.12.0
+        with:
+          java-version: '17'
+          distribution: 'adopt'
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@a4cf152f482c7ca97ef56ead29bf08bcd953284c # v2.7.0
+        with:
+          gradle-home-cache-cleanup: true
+      - name: Run
+        run: |
+          ./gradlew runBenchmarks
+          cat benchmark_results.txt >> $GITHUB_STEP_SUMMARY

build.gradle.kts

@@ -71,7 +71,7 @@ dependencies {
 
     implementation("org.leadpony.justify:justify:3.1.0")
 
-    implementation("org.apache.logging.log4j:log4j-core:$log4jVersion");
+    implementation("org.apache.logging.log4j:log4j-core:$log4jVersion")
     runtimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:$log4jVersion")
 
     testImplementation("org.creekservice:creek-test-hamcrest:$creekVersion")
@@ -86,7 +86,6 @@ dependencies {
     testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:$junitVersion")
 }
 
-
 tasks.withType<JavaCompile> {
     options.compilerArgs.add("-Xlint:all,-serial,-requires-automatic,-requires-transitive-automatic,-module,-processing")
 }
@@ -101,7 +100,7 @@ val cloneTask = tasks.register("clone-json-schema-test-suite") {
     doLast {
         org.ajoberstar.grgit.Grgit.clone {
             dir = jsonSchemaTestSuiteDir.get().asFile
-            uri = "git@github.com:json-schema-org/JSON-Schema-Test-Suite.git"
+            uri = "https://github.com/json-schema-org/JSON-Schema-Test-Suite.git"
         }
     }
 }
@@ -110,7 +109,6 @@ val pullTask = tasks.register("pull-json-schema-test-suite") {
     dependsOn(cloneTask)
 
     doLast {
-        println("pulling.........")
         org.ajoberstar.grgit.Grgit.open {
             dir = jsonSchemaTestSuiteDir.get().asFile
         }.pull()
@@ -120,25 +118,46 @@ val pullTask = tasks.register("pull-json-schema-test-suite") {
 val runFunctionalTests = tasks.register<JavaExec>("runFunctionalTests") {
     classpath = sourceSets.main.get().runtimeClasspath
     mainClass.set("org.creekservice.kafka.test.perf.testsuite.JsonTestSuiteMain")
-    args = listOf(jsonSchemaTestSuiteDir.get().asFile.absolutePath);
+    args = listOf(jsonSchemaTestSuiteDir.get().asFile.absolutePath)
     dependsOn(pullTask)
 }
 
 tasks.register<JavaExec>("runBenchmarks") {
     classpath = sourceSets.main.get().runtimeClasspath
     mainClass.set("org.creekservice.kafka.test.perf.BenchmarkRunner")
+    args(listOf(
+        // Output results in text format
+        "-rf", "text",
+        // To a named file
+        "-rff", "benchmark_results.txt"
+    ))
     dependsOn(pullTask)
 }
 
 val benchmarkSmokeTest = tasks.register<JavaExec>("runBenchmarkSmokeTest") {
     classpath = sourceSets.main.get().runtimeClasspath
     mainClass.set("org.creekservice.kafka.test.perf.BenchmarkRunner")
-    args(listOf("-wi", "0", "-i", "1", "-t", "1", "-r", "1s"))
+    args(listOf(
+        // No warmup:
+        "-wi", "0",
+        // Single test iteration:
+        "-i", "1",
+        // On a single thread:
+        "-t", "1",
+        // Running for 1 second
+        "-r", "1s",
+        // With forking disabled
+        "-f", "0"
+    ))
     dependsOn(pullTask)
 }
 
+tasks.register("coveralls") {
+    // dummy
+}
+
 tasks.test {
-    dependsOn(pullTask, runFunctionalTests, benchmarkSmokeTest)
+    dependsOn(runFunctionalTests, benchmarkSmokeTest)
 }
 
 // Below is required until the following is fixed in IntelliJ:

src/main/java/org/creekservice/kafka/test/perf/testsuite/JsonTestSuiteMain.java

@@ -37,9 +37,14 @@
 import org.creekservice.kafka.test.perf.testsuite.JsonSchemaTestSuite.TestPredicate;
 import org.creekservice.kafka.test.perf.testsuite.output.PerDraftSummary;
 import org.creekservice.kafka.test.perf.testsuite.output.Summary;
+import org.creekservice.kafka.test.perf.util.Logging;
 
 public final class JsonTestSuiteMain {
 
+    static {
+        Logging.disable();
+    }
+
     private static final List<SerdeImpl> IMPLS =
             List.of(
                     new EveritSerde(),

src/main/java/org/creekservice/kafka/test/perf/testsuite/SchemaSpec.java

@@ -20,14 +20,17 @@
 import static java.util.stream.Collectors.toMap;
 
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
+import java.net.HttpURLConnection;
 import java.net.URI;
+import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Scanner;
 import java.util.Set;
 import java.util.function.Function;
 
@@ -61,13 +64,13 @@ public enum SchemaSpec {
     private final String dirName;
     private final URI uri;
     private final String content;
-    private final Map<URI, String> additonal;
+    private final Map<URI, String> additional;
 
     SchemaSpec(final String dirName, final String uri, final Set<String> additional) {
         this.dirName = requireNonNull(dirName, "dirName");
         this.uri = URI.create(uri);
         this.content = loadContent(this.uri);
-        this.additonal =
+        this.additional =
                 additional.stream()
                         .map(URI::create)
                         .collect(toMap(Function.identity(), SchemaSpec::loadContent));
@@ -97,7 +100,7 @@ private Optional<String> getContentFromUri(final URI uri) {
         if (normalize(this.uri).equals(normalized)) {
             return Optional.of(content);
         }
-        final String content = additonal.get(normalized);
+        final String content = additional.get(normalized);
         return content == null ? Optional.empty() : Optional.of(content);
     }
 
@@ -109,11 +112,45 @@ private static URI normalize(final URI uri) {
         return uri;
     }
 
-    @SuppressFBWarnings("URLCONNECTION_SSRF_FD")
+    @SuppressFBWarnings(
+            value = "URLCONNECTION_SSRF_FD",
+            justification = "only called with hardcoded urls")
     private static String loadContent(final URI uri) {
-        try (Scanner scanner = new Scanner(uri.toURL().openStream(), StandardCharsets.UTF_8)) {
-            scanner.useDelimiter("\\A");
-            return scanner.hasNext() ? scanner.next() : "";
+        try {
+            // Always load from https, as non-secure http redirect to https:
+            final URL url =
+                    uri.getScheme().equals("http")
+                            ? new URL("https" + uri.toString().substring(4))
+                            : uri.toURL();
+
+            final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+            connection.setInstanceFollowRedirects(true);
+            final int responseCode = connection.getResponseCode();
+
+            if (responseCode != HttpURLConnection.HTTP_OK) {
+                throw new UncheckedIOException(
+                        new IOException(
+                                "Failed to load content from " + uri + ", code: " + responseCode));
+            }
+
+            try (BufferedReader reader =
+                    new BufferedReader(
+                            new InputStreamReader(
+                                    connection.getInputStream(), StandardCharsets.UTF_8))) {
+                final StringBuilder builder = new StringBuilder();
+
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    builder.append(line).append(System.lineSeparator());
+                }
+
+                final String content = builder.toString();
+                if (content.isBlank()) {
+                    throw new UncheckedIOException(
+                            new IOException("Blank content loaded from " + uri));
+                }
+                return content;
+            }
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }