Data driven performance results

fixes: #53
creek-service · Nov 14, 2023 · f23216e · f23216e
1 parent 1612852
commit f23216e
Show file tree

Hide file tree

Showing 33 changed files with 1,350 additions and 364 deletions.
diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml
@@ -33,14 +33,14 @@ jobs:
         with:
           gradle-home-cache-cleanup: true
       - name: Extract implementation info
-        run: ./gradlew --quiet extractImplementations > implementations.json
+        run: ./gradlew --quiet extractImplementations
       - name: Add results to step summary
-        run:  cat implementations.json >> $GITHUB_STEP_SUMMARY
+        run:  cat docs/_includes/implementations.json >> $GITHUB_STEP_SUMMARY
       - name: Upload Implementations
         uses: actions/upload-artifact@v3
         with:
           name: implementations
-          path: implementations.json
+          path: docs/_includes/*
           retention-days: 1
 
   run_functional:
@@ -61,14 +61,42 @@ jobs:
       - name: Add results to step summary    
         run: |
           echo "# Overall comparison" >> $GITHUB_STEP_SUMMARY 
-          cat build/reports/creek/functional-summary.md >> $GITHUB_STEP_SUMMARY
+          cat docs/_includes/functional-summary.md >> $GITHUB_STEP_SUMMARY
           echo "# Specific Draft & Implementation results" >>  $GITHUB_STEP_SUMMARY
-          cat build/reports/creek/per-draft.md >> $GITHUB_STEP_SUMMARY
+          cat docs/_includes/per-draft.md >> $GITHUB_STEP_SUMMARY
       - name: Upload Implementations
         uses: actions/upload-artifact@v3
         with:
-          name: functional-summary
-          path: build/reports/creek/*
+          name: functional
+          path: docs/_includes/*
+          retention-days: 1
+
+  run_performance:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Set up JDK
+        uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 # v3.13.0
+        with:
+          java-version: '17'
+          distribution: 'adopt'
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@842c587ad8aa4c68eeba24c396e15af4c2e9f30a # v2.9.0
+        with:
+          gradle-home-cache-cleanup: true
+      - name: Run performance benchmarks
+        run: ./gradlew --quiet runBenchmarks
+      - name: Add results to step summary
+        run: |
+          echo "# Json Validator Benchmark Results" >> $GITHUB_STEP_SUMMARY
+          cat docs/_includes/JsonValidateBenchmark.md >> $GITHUB_STEP_SUMMARY
+          echo "# Json Serde Benchmark Results" >> $GITHUB_STEP_SUMMARY
+          cat docs/_includes/JsonSerdeBenchmark.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload Implementations
+        uses: actions/upload-artifact@v3
+        with:
+          name: performance
+          path: docs/_includes/*
           retention-days: 1
 
   build_pages:
@@ -90,22 +118,11 @@ jobs:
       - name: Setup Pages
         id: pages
         uses: actions/configure-pages@f156874f8191504dae5b037505266ed5dda6c382 # v3.0.6
-      - name: Download Implementations JSON
-        uses: actions/download-artifact@v3
-        with:
-          name: implementations
-      - name: Inject Implementations JSON into site
-        run: sed -i $'/IMPLEMENTATIONS_JSON/ { r implementations.json\nd }' docs/_docs/*
-      - name: Download Functional JSON
+      - name: Download all build artefacts
         uses: actions/download-artifact@v3
         with:
-          name: functional-summary
-      - name: Inject Functional JSON into site
-        run: | 
-          sed -i $'/FUNCTIONAL_SUMMARY_JSON/ { r functional-summary.json\nd }' docs/_docs/*
-          cat per-draft.md >> "docs/_docs/2. functional.md"
+          path: docs/_includes/
       - name: Build with Jekyll
-        # Outputs to the './docs/_site' directory by default
         run: (cd docs && bundle exec jekyll build --baseurl "${{ steps.pages.outputs.base_path }}")
         env:
           JEKYLL_ENV: production

diff --git a/README.md b/README.md
@@ -312,7 +312,8 @@ Adding a new validator implementation is relatively straight forward and very we
    There should be one test per supported draft version. See the other methods in these classes for examples.
 7. Run `./gradlew` to format your code, perform static analysis and run the tests. 
    Ensure this passes!
-8. Raise a PR with your changes.
+8. Follow [these instructions](docs) to build and view the website, and ensure your new implementation data is included in tables and charts.
+9. Raise a PR with your changes.
 
 
 [1]: https://github.com/eclipse-vertx/vertx-json-schema

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -127,12 +127,6 @@ val runFunctionalTests = tasks.register<JavaExec>("runFunctionalTests") {
 tasks.register<JavaExec>("runBenchmarks") {
     classpath = sourceSets.main.get().runtimeClasspath
     mainClass.set("org.creekservice.kafka.test.perf.PerformanceMain")
-    args(listOf(
-        // Output results in csv format
-        "-rf", "csv",
-        // To a named file
-        "-rff", "benchmark_results.csv"
-    ))
     dependsOn(pullTask)
 }
 
@@ -163,6 +157,11 @@ tasks.check {
     dependsOn(runFunctionalTests, runBenchmarkSmokeTest, extractImplementations)
 }
 
+tasks.register("buildTestIncludes") {
+    description = "Build include files needed to generate the Jekyll website";
+    dependsOn(runFunctionalTests, runBenchmarkSmokeTest, extractImplementations)
+}
+
 // Dummy / empty tasks required to allow the repo to use the same standard GitHub workflows as other Creek repos:
 tasks.register("coveralls")
 tasks.register("cV")

diff --git a/docs/.gitignore b/docs/.gitignore
@@ -12,4 +12,7 @@ Gemfile.lock
 .jekyll-metadata
 .sass-cache
 _asset_bundler_cache
-_site
+_site
+
+# Includes generated by the GitHub pages build:
+_includes/**
diff --git a/docs/README.md b/docs/README.md
@@ -32,11 +32,35 @@ git commit -m "updating gems"
 git push --set-upstream origin gems-update
 ```
 
-#### 3. Run the local server
+#### 3. Generate includes
+
+For the site to render correctly certain include files need to be generated.
+
+Includes are stored in the `docs/_includes` directory and will be ignored by git.
+
+These include:
+
+| Include details                                                           | Gradle task name                       | Filename                |
+|---------------------------------------------------------------------------|----------------------------------------|-------------------------|
+| A JSON document containing the details of all implementations under test. | extractImplementations                 | implementations.json    |
+| A JSON document containing the summary of the functional testing          | runFunctionalTests                     | functional-summary.json |
+| A Markdown document containing the per-draft functional testing results   | runFunctionalTests                     | per-draft.md            |
+| A JSON document containing the results of the performance benchmarking    | runBenchmarkSmokeTest or runBenchmarks | benchmark_results.json  |
+
+Generate these locally by running:
+
+```shell
+./graldew buildTestIncludes
+```
+
+Note: this will not run the full performance benchmarking as this takes many hours. 
+Instead, it will run the smoke benchmarks will generate inaccurate data go enough for testing the rendering of the website. 
+
+#### 4. Run the local server
 
 ```shell
 (cd docs && bundle exec jekyll serve --livereload --baseurl /json-schema-validation-comparison)
 ```
 
 This will launch a web server so that you can work on the site locally.
-Check it out on [http://localhost:4000/json-schema-validation-comparison](http://localhost:4000/json-schema-validation-comparison).
+Check it out on [http://localhost:4000/json-schema-validation-comparison](http://localhost:4000/json-schema-validation-comparison).
diff --git a/docs/_docs/1. implementations.md b/docs/_docs/1. implementations.md
@@ -25,9 +25,7 @@ against the underlying [<i class="fab fa-fw fa-github"/>&nbsp; GitHub Repo](http
 <script src="https://cdn.jsdelivr.net/npm/[email protected]" type="text/javascript"></script>
 
 <script>
-    const implData = 
-      IMPLEMENTATIONS_JSON
-    ;
+    const implData = {% include implementations.json %};
 
     new simpleDatatables.DataTable("#implsTable", {
         paging: false,
@@ -40,7 +38,7 @@ against the underlying [<i class="fab fa-fw fa-github"/>&nbsp; GitHub Repo](http
                 "Licence",
                 "Supported Schema Versions",
             ],
-            "data": implData.map(row => [ 
+            "data": implData.filter(row => row.shortName !== "Jackson").map(row => [ 
                 "<a href='" + row.url + "'>" + row.longName + "</a>", 
                 row.shortName, 
                 row.language,

diff --git a/docs/_docs/2. functional.md b/docs/_docs/2. functional.md
@@ -8,15 +8,24 @@ toc: true
 classes: wide
 ---
 
-## Test setup
+## Introduction
 
-Each validator implementation is run through a suite a functional tests.
+The purpose of this test is to determine how functionally complete each validator implementation is.
+
+Each implementation is run through a suite a functional tests.
 The tests cover almost every aspect of the different drafts of the JSON Schema specification.
 The tests cover both positive cases, where the validation should succeed, and negative cases, where validation should fail.
 The tests cover both _required_ features and those deemed _optional_ by the specifications.
 
 The tests are curated by the JSON community and are available [<i class="fab fa-fw fa-github"/>&nbsp; on GitHub][JSON-Schema-Test-Suite]{: .btn .btn--success}{:target="_blank"}.
 
+**Note:**
+The [Bowtie Project <i class="fas fa-external-link-alt"></i>][Bowtie]{:target="_blank"} aims to provide similar feature 
+comparison for _all_ validator implementations.
+At the time of writing it does not cover all JVM implementations and does not cover optional schema features.
+Though, these are being worked on.  Ultimately, this project should replace this comparison.
+{: .notice--warning}
+
 ## Results
 
 For each schema specification an implementation supports, the number of test cases that pass and fail is tracked, 
@@ -51,6 +60,13 @@ Each populated cell also contains a **score**, indicating the functional complet
 The **score** weights test results of _required_ features at triple _optional_ features, meaning 75% of the score is reserved for _required_ features,
 whereas _optional_ features only account for a maximum 25% of the score.
 
+### Detailed results
+
+Below is a more details set of results for each specification draft an implementation supports.
+Each table details the number of test cases that pass and fail for each test file in the JSON schema test suite.
+
+{% include per-draft.md %}
+
 [//]: # (Chart scripts: https://www.chartjs.org/docs/latest/)
 <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
 
@@ -59,13 +75,11 @@ whereas _optional_ features only account for a maximum 25% of the score.
 <script src="https://cdn.jsdelivr.net/npm/[email protected]" type="text/javascript"></script>
 
 <script>
-    const implData = 
-        IMPLEMENTATIONS_JSON
-    ;
+    const implData = {% include implementations.json %};
 
-    const summaryData = 
-        FUNCTIONAL_SUMMARY_JSON  
-    ;
+    const summaryData = {% include functional-summary.json %};
+
+    let filteredRows = summaryData.rows.filter(row => row[0] !== "Jackson");
 
     function createRow(row){
       function formatResult(result){
@@ -92,19 +106,19 @@ whereas _optional_ features only account for a maximum 25% of the score.
         searchable: false,
         data: {
             "headings": summaryData.headings.map(h => h + '\u00A0'.repeat(Math.max(0, 25 - h.length))),
-            "data": summaryData.rows.map(row => createRow(row))
+            "data": filteredRows.map(row => createRow(row))
         }
     });
 
     new Chart(document.getElementById('summaryChart'), 
         {
           type: 'bar',
           data: {
-          labels: summaryData.rows.map(row => row[0]),
+          labels: filteredRows.map(row => row[0]),
           datasets: [{
-             data: summaryData.rows.map(row => row[1].score),
-             borderColor: summaryData.rows.map(row => implData.find(impl => impl.shortName === row[0]).color),
-            backgroundColor: summaryData.rows.map(row => implData.find(impl => impl.shortName === row[0]).color.replace('rgb', 'rgba').replace(')', ',0.2)')),
+             data: filteredRows.map(row => row[1].score),
+             borderColor: filteredRows.map(row => implData.find(impl => impl.shortName === row[0]).color),
+             backgroundColor: filteredRows.map(row => implData.find(impl => impl.shortName === row[0]).color.replace('rgb', 'rgba').replace(')', ',0.2)')),
              borderWidth: 1
            }]
           },
@@ -130,10 +144,5 @@ whereas _optional_ features only account for a maximum 25% of the score.
 </script>
 
 [JSON-Schema-Test-Suite]: https://github.com/json-schema-org/JSON-Schema-Test-Suite
+[Bowtie]: https://github.com/bowtie-json-schema/bowtie
 
-### Detailed results
-
-Below is a more details set of results for each specification draft an implementation supports.
-Each table details the number of test cases that pass and fail for each test file in the JSON schema test suite.
-
-[//]: # (Do not add content below this line, or delete the line following this comment, as the build appends data to this file)