From 8e2482a314c9bef4a248e1411cb7e4eeaf80d671 Mon Sep 17 00:00:00 2001 From: Andy Coates <8012398+big-andy-coates@users.noreply.github.com> Date: Mon, 9 Oct 2023 15:25:41 +0100 Subject: [PATCH] =?UTF-8?q?Functional=20and=20performance=20comparison=20o?= =?UTF-8?q?f=20JSON=20serde=20and=20validation=20li=E2=80=A6=20(#22)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Functional and performance comparison of JSON serde and validation libraries. * Add dummy coveralls task to keep standard build file happy * More logging and new workflows * Get git clone working from workflow cloning from unauthenticated account. * Get schema draft loading working again. Looks like json org added redirects to https, which isn't supported in old code. * Add results to job summary * Formatting * Working dummy coveralls task --- .github/workflows/build.yml | 2 +- .github/workflows/run-func-test.yml | 27 ++++++++++ .github/workflows/run-perf-test.yml | 29 ++++++++++ build.gradle.kts | 33 +++++++++--- .../perf/testsuite/JsonTestSuiteMain.java | 5 ++ .../kafka/test/perf/testsuite/SchemaSpec.java | 53 ++++++++++++++++--- 6 files changed, 133 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/run-func-test.yml create mode 100644 .github/workflows/run-perf-test.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 01c0767..7cf226f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,7 +45,7 @@ jobs: - name: Build env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} - run: ./gradlew build coveralls + run: ./gradlew build coveralls --stacktrace - name: Publish if: github.event_name == 'push' || github.event.inputs.publish_artifacts == 'true' env: diff --git a/.github/workflows/run-func-test.yml b/.github/workflows/run-func-test.yml new file mode 100644 index 0000000..45fe2ff --- /dev/null +++ b/.github/workflows/run-func-test.yml @@ -0,0 +1,27 @@ +# This workflow run the functional test + +name: Func Test + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: gradle/wrapper-validation-action@56b90f209b02bf6d1deae490e9ef18b21a389cd4 # v1.1.0 + - name: Set up JDK + uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 # v3.12.0 + with: + java-version: '17' + distribution: 'adopt' + - name: Setup Gradle + uses: gradle/gradle-build-action@a4cf152f482c7ca97ef56ead29bf08bcd953284c # v2.7.0 + with: + gradle-home-cache-cleanup: true + - name: Run + run: ./gradlew --quiet runFunctionalTests >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/.github/workflows/run-perf-test.yml b/.github/workflows/run-perf-test.yml new file mode 100644 index 0000000..ae6df2b --- /dev/null +++ b/.github/workflows/run-perf-test.yml @@ -0,0 +1,29 @@ +# This workflow run the performance test + +name: Perf Test + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: gradle/wrapper-validation-action@56b90f209b02bf6d1deae490e9ef18b21a389cd4 # v1.1.0 + - name: Set up JDK + uses: actions/setup-java@cd89f46ac9d01407894225f350157564c9c7cee2 # v3.12.0 + with: + java-version: '17' + distribution: 'adopt' + - name: Setup Gradle + uses: gradle/gradle-build-action@a4cf152f482c7ca97ef56ead29bf08bcd953284c # v2.7.0 + with: + gradle-home-cache-cleanup: true + - name: Run + run: | + ./gradlew runBenchmarks + cat benchmark_results.txt >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index 57233d9..2b7e825 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -71,7 +71,7 @@ dependencies { implementation("org.leadpony.justify:justify:3.1.0") - implementation("org.apache.logging.log4j:log4j-core:$log4jVersion"); + implementation("org.apache.logging.log4j:log4j-core:$log4jVersion") runtimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:$log4jVersion") testImplementation("org.creekservice:creek-test-hamcrest:$creekVersion") @@ -86,7 +86,6 @@ dependencies { testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:$junitVersion") } - tasks.withType { options.compilerArgs.add("-Xlint:all,-serial,-requires-automatic,-requires-transitive-automatic,-module,-processing") } @@ -101,7 +100,7 @@ val cloneTask = tasks.register("clone-json-schema-test-suite") { doLast { org.ajoberstar.grgit.Grgit.clone { dir = jsonSchemaTestSuiteDir.get().asFile - uri = "git@github.com:json-schema-org/JSON-Schema-Test-Suite.git" + uri = "https://github.com/json-schema-org/JSON-Schema-Test-Suite.git" } } } @@ -110,7 +109,6 @@ val pullTask = tasks.register("pull-json-schema-test-suite") { dependsOn(cloneTask) doLast { - println("pulling.........") org.ajoberstar.grgit.Grgit.open { dir = jsonSchemaTestSuiteDir.get().asFile }.pull() @@ -120,25 +118,46 @@ val pullTask = tasks.register("pull-json-schema-test-suite") { val runFunctionalTests = tasks.register("runFunctionalTests") { classpath = sourceSets.main.get().runtimeClasspath mainClass.set("org.creekservice.kafka.test.perf.testsuite.JsonTestSuiteMain") - args = listOf(jsonSchemaTestSuiteDir.get().asFile.absolutePath); + args = listOf(jsonSchemaTestSuiteDir.get().asFile.absolutePath) dependsOn(pullTask) } tasks.register("runBenchmarks") { classpath = sourceSets.main.get().runtimeClasspath mainClass.set("org.creekservice.kafka.test.perf.BenchmarkRunner") + args(listOf( + // Output results in text format + "-rf", "text", + // To a named file + "-rff", "benchmark_results.txt" + )) dependsOn(pullTask) } val benchmarkSmokeTest = tasks.register("runBenchmarkSmokeTest") { classpath = sourceSets.main.get().runtimeClasspath mainClass.set("org.creekservice.kafka.test.perf.BenchmarkRunner") - args(listOf("-wi", "0", "-i", "1", "-t", "1", "-r", "1s")) + args(listOf( + // No warmup: + "-wi", "0", + // Single test iteration: + "-i", "1", + // On a single thread: + "-t", "1", + // Running for 1 second + "-r", "1s", + // With forking disabled + "-f", "0" + )) dependsOn(pullTask) } +tasks.register("coveralls") { + // dummy +} + tasks.test { - dependsOn(pullTask, runFunctionalTests, benchmarkSmokeTest) + dependsOn(runFunctionalTests, benchmarkSmokeTest) } // Below is required until the following is fixed in IntelliJ: diff --git a/src/main/java/org/creekservice/kafka/test/perf/testsuite/JsonTestSuiteMain.java b/src/main/java/org/creekservice/kafka/test/perf/testsuite/JsonTestSuiteMain.java index c5e8bd8..2e79daa 100644 --- a/src/main/java/org/creekservice/kafka/test/perf/testsuite/JsonTestSuiteMain.java +++ b/src/main/java/org/creekservice/kafka/test/perf/testsuite/JsonTestSuiteMain.java @@ -37,9 +37,14 @@ import org.creekservice.kafka.test.perf.testsuite.JsonSchemaTestSuite.TestPredicate; import org.creekservice.kafka.test.perf.testsuite.output.PerDraftSummary; import org.creekservice.kafka.test.perf.testsuite.output.Summary; +import org.creekservice.kafka.test.perf.util.Logging; public final class JsonTestSuiteMain { + static { + Logging.disable(); + } + private static final List IMPLS = List.of( new EveritSerde(), diff --git a/src/main/java/org/creekservice/kafka/test/perf/testsuite/SchemaSpec.java b/src/main/java/org/creekservice/kafka/test/perf/testsuite/SchemaSpec.java index dcaedff..2324251 100644 --- a/src/main/java/org/creekservice/kafka/test/perf/testsuite/SchemaSpec.java +++ b/src/main/java/org/creekservice/kafka/test/perf/testsuite/SchemaSpec.java @@ -20,14 +20,17 @@ import static java.util.stream.Collectors.toMap; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStreamReader; import java.io.UncheckedIOException; +import java.net.HttpURLConnection; import java.net.URI; +import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Map; import java.util.Optional; -import java.util.Scanner; import java.util.Set; import java.util.function.Function; @@ -61,13 +64,13 @@ public enum SchemaSpec { private final String dirName; private final URI uri; private final String content; - private final Map additonal; + private final Map additional; SchemaSpec(final String dirName, final String uri, final Set additional) { this.dirName = requireNonNull(dirName, "dirName"); this.uri = URI.create(uri); this.content = loadContent(this.uri); - this.additonal = + this.additional = additional.stream() .map(URI::create) .collect(toMap(Function.identity(), SchemaSpec::loadContent)); @@ -97,7 +100,7 @@ private Optional getContentFromUri(final URI uri) { if (normalize(this.uri).equals(normalized)) { return Optional.of(content); } - final String content = additonal.get(normalized); + final String content = additional.get(normalized); return content == null ? Optional.empty() : Optional.of(content); } @@ -109,11 +112,45 @@ private static URI normalize(final URI uri) { return uri; } - @SuppressFBWarnings("URLCONNECTION_SSRF_FD") + @SuppressFBWarnings( + value = "URLCONNECTION_SSRF_FD", + justification = "only called with hardcoded urls") private static String loadContent(final URI uri) { - try (Scanner scanner = new Scanner(uri.toURL().openStream(), StandardCharsets.UTF_8)) { - scanner.useDelimiter("\\A"); - return scanner.hasNext() ? scanner.next() : ""; + try { + // Always load from https, as non-secure http redirect to https: + final URL url = + uri.getScheme().equals("http") + ? new URL("https" + uri.toString().substring(4)) + : uri.toURL(); + + final HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setInstanceFollowRedirects(true); + final int responseCode = connection.getResponseCode(); + + if (responseCode != HttpURLConnection.HTTP_OK) { + throw new UncheckedIOException( + new IOException( + "Failed to load content from " + uri + ", code: " + responseCode)); + } + + try (BufferedReader reader = + new BufferedReader( + new InputStreamReader( + connection.getInputStream(), StandardCharsets.UTF_8))) { + final StringBuilder builder = new StringBuilder(); + + String line; + while ((line = reader.readLine()) != null) { + builder.append(line).append(System.lineSeparator()); + } + + final String content = builder.toString(); + if (content.isBlank()) { + throw new UncheckedIOException( + new IOException("Blank content loaded from " + uri)); + } + return content; + } } catch (IOException e) { throw new UncheckedIOException(e); }