From 7bc5ffe2e6e8059190e37ef5f24a849f92a58523 Mon Sep 17 00:00:00 2001 From: Big Andy <8012398+big-andy-coates@users.noreply.github.com> Date: Fri, 17 Nov 2023 21:13:56 +0000 Subject: [PATCH] Minor doc improvements fi xes: #73 Plus some reformatting and removing the, now superfluous, 'score' --- docs/_docs/1. implementations.md | 3 + docs/_docs/2. functional.md | 132 +++++------------- docs/_docs/3. performance.md | 16 ++- docs/_docs/4. other considerations.md | 12 +- .../kafka/test/perf/FunctionalMain.java | 3 +- .../test/perf/testsuite/output/Summary.java | 29 +--- 6 files changed, 55 insertions(+), 140 deletions(-) diff --git a/docs/_docs/1. implementations.md b/docs/_docs/1. implementations.md index 2e1ff92..9e98a14 100644 --- a/docs/_docs/1. implementations.md +++ b/docs/_docs/1. implementations.md @@ -14,6 +14,9 @@ See below for an up-to-date list of the JVM based JSON Validator implementations
+**Note:** >>> scroll to the right for more columns on the table. +{: .notice--warning} + ## Note to maintainers If you are the maintainer of one of the above implementations, and you feel your implementation is poorly represented, diff --git a/docs/_docs/2. functional.md b/docs/_docs/2. functional.md index eb9df45..a166a65 100644 --- a/docs/_docs/2. functional.md +++ b/docs/_docs/2. functional.md @@ -37,75 +37,7 @@ The charts below illustrate the percentage of required and optional test cases t and aggregate/overall graphs that show the pass rate across all the drafts an implementation supports. Results are broken down into separate _required_ and _optional_ graphs. -#### Overall - -
- -
-  -
- -
- -#### Draft_2020_12 - -
- -
-  -
- -
- -#### Draft_2019_09 - -
- -
-  -
- -
- -#### Draft_07 - -
- -
-  -
- -
- -#### Draft_06 - -
- -
-  -
- -
- -#### Draft_04 - -
- -
-  -
- -
- -#### Draft_03 - -
- -
-  -
- -
+
#### Summary results table @@ -123,10 +55,6 @@ Unpopulated cells indicate the implementations does not support that specific sc Populated cells details the number (and percentage) of **r**equired and **o**ptional test cases that **pass** and **fail**. -Each populated cell also contains a **score**, indicating the functional completeness of the implementation, out of 100. -The **score** weights test results of _required_ features at triple _optional_ features, meaning 75% of the score is reserved for _required_ features, -whereas _optional_ features only account for a maximum 25% of the score. - ### Detailed results Below is a more details set of results for each specification draft an implementation supports. @@ -146,16 +74,16 @@ Each table details the number of test cases that pass and fail for each test fil const summaryData = {% include functional-summary.json %}; - let filteredRows = summaryData.rows.filter(row => row[0] !== "Jackson"); + const headings = summaryData.headings; + const filteredRows = summaryData.rows.filter(row => row[0] !== "Jackson"); function createRow(row){ function formatResult(result){ - if (result.score === 0.0) { + if (result.requiredPass === 0 && result.requiredFail === 0) { return ""; } - return "Score: " + result.score - + "
pass:" + return "pass:" + "
r: " + result.requiredPass + "(" + result.requiredPassPct + "%)" + "
o: " + result.optionalPass + "(" + result.optionalPassPct + "%)" + "
fail:" @@ -172,20 +100,22 @@ Each table details the number of test cases that pass and fail for each test fil paging: false, searchable: false, data: { - "headings": summaryData.headings.map(h => h + '\u00A0'.repeat(Math.max(0, 25 - h.length))), + "headings": headings.map(h => h + '\u00A0'.repeat(Math.max(0, 25 - h.length))), "data": filteredRows.map(row => createRow(row)) } }); - const chartContainer = document.getElementById('complianceCharts'); + const chartContainer = document.getElementById('functionalCharts'); - const createChart = (id, column, title, field, suggestedMin) => { - const name = summaryData.headings[column]; - const chartData = summaryData.rows - .filter(row => row[0] !== 'Jackson') + const createChart = (column, title, field, suggestedMin) => { + const canvas = document.createElement('canvas'); + chartContainer.append(canvas); + + const name = headings[column]; + const chartData = filteredRows .filter(row => row[column][field] !== 0) .toSorted((r1, r2) => r2[column][field] - r1[column][field]); - new Chart(document.getElementById(id), + new Chart(canvas, { type: 'bar', data: { @@ -201,7 +131,12 @@ Each table details the number of test cases that pass and fail for each test fil plugins: { title: { display: true, - text: `[${name}] ${title}` + text: `${name} ${title} Functionality (higher is better)`, + align: 'start', + padding: { + top: 50, + bottom: 30 + } }, legend: { display: false @@ -211,27 +146,24 @@ Each table details the number of test cases that pass and fail for each test fil y: { beginAtZero: false, suggestedMin: suggestedMin, - suggestedMax: 100 + suggestedMax: 100, + title: { + display: true, + text: '% of test cases passed' + } } } }, }) }; - createChart('overallRequiredChart', 1, 'REQUIRED tests pass percentage (higher is better)', 'requiredPassPct', 80); - createChart('overallOptionalChart', 1, 'OPTIONAL tests pass percentage (higher is better)', 'optionalPassPct', 50); - createChart('draft2020RequiredChart', 7, 'REQUIRED tests pass percentage (higher is better)', 'requiredPassPct', 80); - createChart('draft2020OptionalChart', 7, 'OPTIONAL tests pass percentage (higher is better)', 'optionalPassPct', 50); - createChart('draft2019RequiredChart', 6, 'REQUIRED tests pass percentage (higher is better)', 'requiredPassPct', 80); - createChart('draft2019OptionalChart', 6, 'OPTIONAL tests pass percentage (higher is better)', 'optionalPassPct', 50); - createChart('draft07RequiredChart', 5, 'REQUIRED tests pass percentage (higher is better)', 'requiredPassPct', 80); - createChart('draft07OptionalChart', 5, 'OPTIONAL tests pass percentage (higher is better)', 'optionalPassPct', 50); - createChart('draft06RequiredChart', 4, 'REQUIRED tests pass percentage (higher is better)', 'requiredPassPct', 80); - createChart('draft06OptionalChart', 4, 'OPTIONAL tests pass percentage (higher is better)', 'optionalPassPct', 50); - createChart('draft04RequiredChart', 3, 'REQUIRED tests pass percentage (higher is better)', 'requiredPassPct', 80); - createChart('draft04OptionalChart', 3, 'OPTIONAL tests pass percentage (higher is better)', 'optionalPassPct', 50); - createChart('draft03RequiredChart', 2, 'REQUIRED tests pass percentage (higher is better)', 'requiredPassPct', 80); - createChart('draft03OptionalChart', 2, 'OPTIONAL tests pass percentage (higher is better)', 'optionalPassPct', 50); + createChart(1, 'Required', 'requiredPassPct', 80); + createChart(1, 'Optional', 'optionalPassPct', 50); + + for(let i = headings.length - 1; i > 1; i--) { + createChart(i, 'Required', 'requiredPassPct', 80); + createChart(i, 'Optional', 'optionalPassPct', 50); + } [JSON-Schema-Test-Suite]: https://github.com/json-schema-org/JSON-Schema-Test-Suite diff --git a/docs/_docs/3. performance.md b/docs/_docs/3. performance.md index f2f52df..a80219d 100644 --- a/docs/_docs/3. performance.md +++ b/docs/_docs/3. performance.md @@ -23,7 +23,7 @@ as a serialization format. Combined, these should give a good comparison of per **Note:** The benchmarks are run on [GitHub's own infrastructure](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners). -These may not be dedicated machines, which can influence performance results. +These may not be dedicated machines, which can influence the stability of performance results. {: .notice--warning} ### JSON schema test suite benchmark @@ -108,10 +108,7 @@ Comparison of different implementations across specification versions may be mis const chartContainer = document.getElementById(benchmarkType + 'Charts'); drafts.forEach(function(draft) { - const title = document.createElement('h4'); const canvas = document.createElement('canvas'); - title.textContent = draft + ' Results'; - chartContainer.append(title); chartContainer.append(canvas); let draftData = resultData.filter(r => r.benchmark.includes(draft)).sort(function(a, b) { @@ -135,7 +132,12 @@ Comparison of different implementations across specification versions may be mis plugins: { title: { display: true, - text: draft + ' ' + benchmarkType + ' Performance (lower is better)' + text: draft + ' ' + benchmarkType + ' Performance (lower is better)', + align: 'start', + padding: { + top: 50, + bottom: 30 + } }, legend: { display: false @@ -155,8 +157,8 @@ Comparison of different implementations across specification versions may be mis }); } - buildCharts(validateResults, 'Validate', ["Draft_04", "Draft_06", "Draft_07", "Draft_2019_09", "Draft_2020_12"]); - buildCharts(serdeResults.filter(r => !r.benchmark.includes('_Snow')), 'Serde', ["Draft_07", "Draft_2020_12"]); + buildCharts(validateResults.filter(r => !r.benchmark.includes('_Snow')), 'Validate', ["Draft_2020_12", "Draft_2019_09", "Draft_07", "Draft_06", "Draft_04", "Draft_03"]); + buildCharts(serdeResults, 'Serde', ["Draft_2020_12", "Draft_07"]); diff --git a/docs/_docs/4. other considerations.md b/docs/_docs/4. other considerations.md index de387d6..f90e64b 100644 --- a/docs/_docs/4. other considerations.md +++ b/docs/_docs/4. other considerations.md @@ -13,14 +13,16 @@ Things to also consider are: - Is the project in active development? When was its last release? The `Project activity` column on the [Libraries under test table]({% link _docs/1. implementations.md %}) attempts to show this, - but relies on someone updating the site if a project beings inactive. + but relies on someone updating the site if a project becomes inactive. Projects that aren't active come with their own set of issues, especially around bug & security fixes, or dependency updates. - What dependencies does it bring in? Less being more. - For example, Vert.x brings in Netty as a dependency, which seems unnecessary. + For example, `Vertx` brings in Netty as a dependency, which seems unnecessary. + Whereas, `DevHarrel` is very light on dependendies. - Size of the library's jar file and its dependencies. The `Jar size` column on the [Libraries under test table]({% link _docs/1. implementations.md %}) shows the size of the library's primary jar, but does not _yet_ include the size of any other dependencies this brings in. - Is the implementation fit for purpose. - For example, the `Snow` implementation documents itself as a reference implementation. - (This may go some way to explain its poor performance). - Another example is the `Vertx` implementation, which doesn't seem to provide anyway to control how remote references are loaded. + For example: + - The `Snow` implementation documents itself as a reference implementation. (This may go some way to explain its poor performance). + - The `Vertx` implementation doesn't seem to provide a way to control how remote references are loaded. + - The `Justify` implementation supports validating while streaming large JSON documents, rather than having to load the entire document into memory. diff --git a/src/main/java/org/creekservice/kafka/test/perf/FunctionalMain.java b/src/main/java/org/creekservice/kafka/test/perf/FunctionalMain.java index 389bc9b..5832e03 100644 --- a/src/main/java/org/creekservice/kafka/test/perf/FunctionalMain.java +++ b/src/main/java/org/creekservice/kafka/test/perf/FunctionalMain.java @@ -25,7 +25,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; import java.util.HashMap; import java.util.Map; import java.util.function.Function; @@ -98,7 +97,7 @@ private static void writeOutput(final String content, final Path path) { if (parent != null) { Files.createDirectories(parent); } - Files.write(path, content.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); + Files.write(path, content.getBytes(StandardCharsets.UTF_8)); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/main/java/org/creekservice/kafka/test/perf/testsuite/output/Summary.java b/src/main/java/org/creekservice/kafka/test/perf/testsuite/output/Summary.java index 91fd6be..989298a 100644 --- a/src/main/java/org/creekservice/kafka/test/perf/testsuite/output/Summary.java +++ b/src/main/java/org/creekservice/kafka/test/perf/testsuite/output/Summary.java @@ -26,7 +26,6 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -37,9 +36,6 @@ public final class Summary { - /** How much weight to put in required features vs optional. */ - private static final int REQUIRED_WEIGHT = 3; - private static final String COL_IMPL = "Implementations"; private static final String COL_OVERALL = "Overall"; @@ -125,14 +121,8 @@ private static Table buildTable( final List specColumns, final List headers) { final Table table = new Table(headers); - - counts.entrySet().stream() - .sorted( - Comparator.>>comparingDouble( - e1 -> e1.getValue().get(COL_OVERALL).score()) - .reversed()) - .forEach(e -> populateRow(table.addRow(), e.getKey(), e.getValue(), specColumns)); - + counts.forEach( + (impl, specCounts) -> populateRow(table.addRow(), impl, specCounts, specColumns)); return table; } @@ -206,25 +196,12 @@ BigDecimal optFailPct() { return percentage(optFail(), optTotal); } - @JsonProperty("score") - BigDecimal formattedScore() { - return BigDecimal.valueOf(score()).setScale(1, RoundingMode.HALF_EVEN); - } - - double score() { - final double reqPct = reqTotal == 0 ? 0 : ((double) reqPassed / reqTotal); - final double optPct = optTotal == 0 ? 0 : ((double) optPassed / optTotal); - return 100 * ((reqPct * REQUIRED_WEIGHT) + optPct) / (REQUIRED_WEIGHT + 1); - } - @Override public String toString() { if (totalTotal() == 0) { return ""; } - return "score: " - + formattedScore() - + "
pass: r:" + return "pass: r:" + reqPassed + " (" + reqPassPct()