Minor doc improvements

docs/_docs/1. implementations.md

@@ -14,6 +14,9 @@ See below for an up-to-date list of the JVM based JSON Validator implementations
  <table id="implsTable"></table>
 </div>
 
+**Note:** &gt;&gt;&gt; scroll to the right for more columns on the table.
+{: .notice--warning}
+
 ## Note to maintainers
 
 If you are the maintainer of one of the above implementations, and you feel your implementation is poorly represented,

docs/_docs/2. functional.md

@@ -37,75 +37,7 @@ The charts below illustrate the percentage of required and optional test cases t
 and aggregate/overall graphs that show the pass rate across all the drafts an implementation supports.
 Results are broken down into separate _required_ and _optional_ graphs.
 
-#### Overall
-
-<div>
-    <canvas id="overallRequiredChart"></canvas>
-</div>
-&nbsp;
-<div>
-    <canvas id="overallOptionalChart"></canvas>
-</div>
-
-#### Draft_2020_12
-
-<div>
-    <canvas id="draft2020RequiredChart"></canvas>
-</div>
-&nbsp;
-<div>
-    <canvas id="draft2020OptionalChart"></canvas>
-</div>
-
-#### Draft_2019_09
-
-<div>
-    <canvas id="draft2019RequiredChart"></canvas>
-</div>
-&nbsp;
-<div>
-    <canvas id="draft2019OptionalChart"></canvas>
-</div>
-
-#### Draft_07
-
-<div>
-    <canvas id="draft07RequiredChart"></canvas>
-</div>
-&nbsp;
-<div>
-    <canvas id="draft07OptionalChart"></canvas>
-</div>
-
-#### Draft_06
-
-<div>
-    <canvas id="draft06RequiredChart"></canvas>
-</div>
-&nbsp;
-<div>
-    <canvas id="draft06OptionalChart"></canvas>
-</div>
-
-#### Draft_04
-
-<div>
-    <canvas id="draft04RequiredChart"></canvas>
-</div>
-&nbsp;
-<div>
-    <canvas id="draft04OptionalChart"></canvas>
-</div>
-
-#### Draft_03
-
-<div>
-    <canvas id="draft03RequiredChart"></canvas>
-</div>
-&nbsp;
-<div>
-    <canvas id="draft03OptionalChart"></canvas>
-</div>
+<div id="functionalCharts"></div>
 
 #### Summary results table
 
@@ -123,10 +55,6 @@ Unpopulated cells indicate the implementations does not support that specific sc
 
 Populated cells details the number (and percentage) of **r**equired and **o**ptional test cases that **pass** and **fail**.
 
-Each populated cell also contains a **score**, indicating the functional completeness of the implementation, out of 100.
-The **score** weights test results of _required_ features at triple _optional_ features, meaning 75% of the score is reserved for _required_ features,
-whereas _optional_ features only account for a maximum 25% of the score.
-
 ### Detailed results
 
 Below is a more details set of results for each specification draft an implementation supports.
@@ -146,16 +74,16 @@ Each table details the number of test cases that pass and fail for each test fil
 
     const summaryData = {% include functional-summary.json %};
 
-    let filteredRows = summaryData.rows.filter(row => row[0] !== "Jackson");
+    const headings = summaryData.headings;
+    const filteredRows = summaryData.rows.filter(row => row[0] !== "Jackson");
 
     function createRow(row){
       function formatResult(result){
-        if (result.score === 0.0) {
+        if (result.requiredPass === 0 && result.requiredFail === 0) {
           return "";
         }
 
-        return  "Score: " + result.score
-           + "<br>pass:"
+        return  "pass:"
            + "<br>r: " + result.requiredPass + "(" + result.requiredPassPct + "%)"
            + "<br>o: " + result.optionalPass + "(" + result.optionalPassPct + "%)"
            + "<br>fail:"
@@ -172,20 +100,22 @@ Each table details the number of test cases that pass and fail for each test fil
         paging: false,
         searchable: false,
         data: {
-            "headings": summaryData.headings.map(h => h + '\u00A0'.repeat(Math.max(0, 25 - h.length))),
+            "headings": headings.map(h => h + '\u00A0'.repeat(Math.max(0, 25 - h.length))),
             "data": filteredRows.map(row => createRow(row))
         }
     });
 
-    const chartContainer = document.getElementById('complianceCharts');
+    const chartContainer = document.getElementById('functionalCharts');
 
-    const createChart = (id, column, title, field, suggestedMin) => {
-        const name = summaryData.headings[column];
-        const chartData = summaryData.rows
-            .filter(row => row[0] !== 'Jackson')
+    const createChart = (column, title, field, suggestedMin) => {
+        const canvas = document.createElement('canvas');
+        chartContainer.append(canvas);
+
+        const name = headings[column];
+        const chartData = filteredRows
             .filter(row => row[column][field] !== 0)
             .toSorted((r1, r2) => r2[column][field] - r1[column][field]);
-        new Chart(document.getElementById(id), 
+        new Chart(canvas, 
             {
               type: 'bar',
               data: {
@@ -201,7 +131,12 @@ Each table details the number of test cases that pass and fail for each test fil
                 plugins: {
                     title: {
                         display: true,
-                        text: `[${name}] ${title}`
+                        text: `${name} ${title} Functionality (higher is better)`,
+                        align: 'start',
+                        padding: {
+                          top: 50,
+                          bottom: 30
+                        }
                     },
                     legend: {
                         display: false
@@ -211,27 +146,24 @@ Each table details the number of test cases that pass and fail for each test fil
                   y: {
                     beginAtZero: false,
                     suggestedMin: suggestedMin,
-                    suggestedMax: 100
+                    suggestedMax: 100,
+                    title: {
+                      display: true,
+                      text: '% of test cases passed'
+                    }
                   }
                 }
               },
             })
         };
 
-        createChart('overallRequiredChart', 1, 'REQUIRED tests pass percentage (higher is better)', 'requiredPassPct', 80);
-        createChart('overallOptionalChart', 1, 'OPTIONAL tests pass percentage (higher is better)', 'optionalPassPct', 50);
-        createChart('draft2020RequiredChart', 7, 'REQUIRED tests pass percentage (higher is better)', 'requiredPassPct', 80);
-        createChart('draft2020OptionalChart', 7, 'OPTIONAL tests pass percentage (higher is better)', 'optionalPassPct', 50);
-        createChart('draft2019RequiredChart', 6, 'REQUIRED tests pass percentage (higher is better)', 'requiredPassPct', 80);
-        createChart('draft2019OptionalChart', 6, 'OPTIONAL tests pass percentage (higher is better)', 'optionalPassPct', 50);
-        createChart('draft07RequiredChart', 5, 'REQUIRED tests pass percentage (higher is better)', 'requiredPassPct', 80);
-        createChart('draft07OptionalChart', 5, 'OPTIONAL tests pass percentage (higher is better)', 'optionalPassPct', 50);
-        createChart('draft06RequiredChart', 4, 'REQUIRED tests pass percentage (higher is better)', 'requiredPassPct', 80);
-        createChart('draft06OptionalChart', 4, 'OPTIONAL tests pass percentage (higher is better)', 'optionalPassPct', 50);
-        createChart('draft04RequiredChart', 3, 'REQUIRED tests pass percentage (higher is better)', 'requiredPassPct', 80);
-        createChart('draft04OptionalChart', 3, 'OPTIONAL tests pass percentage (higher is better)', 'optionalPassPct', 50);
-        createChart('draft03RequiredChart', 2, 'REQUIRED tests pass percentage (higher is better)', 'requiredPassPct', 80);
-        createChart('draft03OptionalChart', 2, 'OPTIONAL tests pass percentage (higher is better)', 'optionalPassPct', 50);
+        createChart(1, 'Required', 'requiredPassPct', 80);
+        createChart(1, 'Optional', 'optionalPassPct', 50);
+
+        for(let i = headings.length - 1; i > 1; i--) {
+            createChart(i, 'Required', 'requiredPassPct', 80);
+            createChart(i, 'Optional', 'optionalPassPct', 50);
+        }
 </script>
 
 [JSON-Schema-Test-Suite]: https://github.com/json-schema-org/JSON-Schema-Test-Suite

docs/_docs/3. performance.md

@@ -23,7 +23,7 @@ as a serialization format.  Combined, these should give a good comparison of per
 
 **Note:**
 The benchmarks are run on [GitHub's own infrastructure](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners). 
-These may not be dedicated machines, which can influence performance results.   
+These may not be dedicated machines, which can influence the stability of performance results.   
 {: .notice--warning}
 
 ### JSON schema test suite benchmark
@@ -108,10 +108,7 @@ Comparison of different implementations across specification versions may be mis
       const chartContainer = document.getElementById(benchmarkType + 'Charts');
 
       drafts.forEach(function(draft) {
-        const title = document.createElement('h4');
         const canvas = document.createElement('canvas');
-        title.textContent = draft + ' Results';
-        chartContainer.append(title);
         chartContainer.append(canvas);
 
         let draftData = resultData.filter(r => r.benchmark.includes(draft)).sort(function(a, b) {
@@ -135,7 +132,12 @@ Comparison of different implementations across specification versions may be mis
               plugins: {
                   title: {
                       display: true,
-                      text: draft + ' ' + benchmarkType + ' Performance (lower is better)'
+                      text: draft + ' ' + benchmarkType + ' Performance (lower is better)',
+                      align: 'start',
+                      padding: {
+                        top: 50,
+                        bottom: 30
+                      }
                   },
                   legend: {
                       display: false
@@ -155,8 +157,8 @@ Comparison of different implementations across specification versions may be mis
       });
   }
 
-  buildCharts(validateResults, 'Validate', ["Draft_04", "Draft_06", "Draft_07", "Draft_2019_09", "Draft_2020_12"]);
-  buildCharts(serdeResults.filter(r => !r.benchmark.includes('_Snow')), 'Serde', ["Draft_07", "Draft_2020_12"]);
+  buildCharts(validateResults.filter(r => !r.benchmark.includes('_Snow')), 'Validate', ["Draft_2020_12", "Draft_2019_09", "Draft_07", "Draft_06", "Draft_04", "Draft_03"]);
+  buildCharts(serdeResults, 'Serde', ["Draft_2020_12", "Draft_07"]);
 </script>
 
 

docs/_docs/4. other considerations.md

@@ -13,14 +13,16 @@ Things to also consider are:
 
 - Is the project in active development? When was its last release?
   The `Project activity` column on the [Libraries under test table]({% link _docs/1. implementations.md %}) attempts to show this, 
-  but relies on someone updating the site if a project beings inactive. 
+  but relies on someone updating the site if a project becomes inactive. 
   Projects that aren't active come with their own set of issues, especially around bug & security fixes, or dependency updates.
 - What dependencies does it bring in? Less being more. 
-  For example, Vert.x brings in Netty as a dependency, which seems unnecessary.
+  For example, `Vertx` brings in Netty as a dependency, which seems unnecessary.
+  Whereas, `DevHarrel` is very light on dependendies. 
 - Size of the library's jar file and its dependencies.
   The `Jar size` column on the [Libraries under test table]({% link _docs/1. implementations.md %}) shows the size of the library's primary jar, 
   but does not _yet_ include the size of any other dependencies this brings in. 
 - Is the implementation fit for purpose. 
-  For example, the `Snow` implementation documents itself as a reference implementation.
-  (This may go some way to explain its poor performance).
-  Another example is the `Vertx` implementation, which doesn't seem to provide anyway to control how remote references are loaded.
+  For example:
+    - The `Snow` implementation documents itself as a reference implementation. (This may go some way to explain its poor performance).
+    - The `Vertx` implementation doesn't seem to provide a way to control how remote references are loaded.
+    - The `Justify` implementation supports validating while streaming large JSON documents, rather than having to load the entire document into memory.

src/main/java/org/creekservice/kafka/test/perf/FunctionalMain.java

@@ -25,7 +25,6 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Function;
@@ -98,7 +97,7 @@ private static void writeOutput(final String content, final Path path) {
             if (parent != null) {
                 Files.createDirectories(parent);
             }
-            Files.write(path, content.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
+            Files.write(path, content.getBytes(StandardCharsets.UTF_8));
         } catch (IOException e) {
             throw new RuntimeException(e);
         }

src/main/java/org/creekservice/kafka/test/perf/testsuite/output/Summary.java

@@ -26,7 +26,6 @@
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -37,9 +36,6 @@
 
 public final class Summary {
 
-    /** How much weight to put in required features vs optional. */
-    private static final int REQUIRED_WEIGHT = 3;
-
     private static final String COL_IMPL = "Implementations";
     private static final String COL_OVERALL = "Overall";
 
@@ -125,14 +121,8 @@ private static Table buildTable(
             final List<String> specColumns,
             final List<String> headers) {
         final Table table = new Table(headers);
-
-        counts.entrySet().stream()
-                .sorted(
-                        Comparator.<Map.Entry<String, Map<String, Counts>>>comparingDouble(
-                                        e1 -> e1.getValue().get(COL_OVERALL).score())
-                                .reversed())
-                .forEach(e -> populateRow(table.addRow(), e.getKey(), e.getValue(), specColumns));
-
+        counts.forEach(
+                (impl, specCounts) -> populateRow(table.addRow(), impl, specCounts, specColumns));
         return table;
     }
 
@@ -206,25 +196,12 @@ BigDecimal optFailPct() {
             return percentage(optFail(), optTotal);
         }
 
-        @JsonProperty("score")
-        BigDecimal formattedScore() {
-            return BigDecimal.valueOf(score()).setScale(1, RoundingMode.HALF_EVEN);
-        }
-
-        double score() {
-            final double reqPct = reqTotal == 0 ? 0 : ((double) reqPassed / reqTotal);
-            final double optPct = optTotal == 0 ? 0 : ((double) optPassed / optTotal);
-            return 100 * ((reqPct * REQUIRED_WEIGHT) + optPct) / (REQUIRED_WEIGHT + 1);
-        }
-
         @Override
         public String toString() {
             if (totalTotal() == 0) {
                 return "";
             }
-            return "score: "
-                    + formattedScore()
-                    + "<br>pass: r:"
+            return "pass: r:"
                     + reqPassed
                     + " ("
                     + reqPassPct()