feat: generate the Dashboard - base structure

.github/actions/test/action.yml

@@ -8,6 +8,11 @@ inputs:
     description: "The Subdomain URL of the IPFS Gateway implementation to be tested."
     default: "http://example.com"
     required: false
+  accept-test-failure:
+    # see https://github.com/orgs/community/discussions/15452
+    description: "When set to `true`, the action will not fail (become red) if the tests fail. Use the reports to determine the outcome of the tests."
+    default: "false"
+    required: false
   json:
     description: "The path where the JSON test report should be generated."
     required: true
@@ -21,6 +26,9 @@ inputs:
   markdown:
     description: "The path where the summary Markdown test report should be generated."
     required: false
+  report:
+    description: "The path where the summary JSON test report should be generated."
+    required: false
   specs:
     description: "A comma-separated list of specs to be tested. Accepts a spec (test only this spec), a +spec (test also this immature spec), or a -spec (do not test this mature spec)."
     required: false
@@ -45,6 +53,7 @@ runs:
         repository: ${{ steps.github.outputs.action_repository }}
         ref: ${{ steps.github.outputs.action_sha || steps.github.outputs.action_ref }}
         dockerfile: Dockerfile
+        allow-exit-codes: ${{ inputs.accept-test-failure == 'false' && '0' || '0,1' }}
         opts: --network=host
         args: test --url="$URL" --json="$JSON" --specs="$SPECS" --subdomain-url="$SUBDOMAIN" --job-url="$JOB_URL" -- ${{ inputs.args }}
         build-args: |
@@ -69,3 +78,13 @@ runs:
         mode: summary
         input: ${{ inputs.xml }}
         output: ${{ inputs.markdown }}
+    - name: Create the JSON Report
+      if: inputs.report && (failure() || success())
+      shell: bash
+      env:
+        JSON: ${{ inputs.json }}
+        REPORT: ${{ inputs.report }}
+      run: |
+        # TODO: checkout here.
+        wget 'https://raw.githubusercontent.com/singulargarden/gateway-conformance/main/munge.js' -O munge.js
+        cat "${JSON}" | node munge.js > "${REPORT}"

.github/workflows/deploy-pages.yml

@@ -86,29 +86,33 @@ jobs:
         with:
           path: artifacts
       - name: Generate Data Aggregates
-        working-directory: ./artifacts
+        working-directory: ./
         run: |
-          mkdir ../aggregates
+          npm ci
+          mkdir ./munged
 
           # download-artifact downloads artifacts in a directory named after the artifact
           # details: https://github.com/actions/download-artifact#download-all-artifacts
-          for folder in ./conformance-*.json; do
+          for folder in ./artifacts/conformance-*.json; do
             file="${folder}/output.json"
-            new_file="../aggregates/${folder#.\/conformance-}" # drop the ./conformance- prefix
-            jq -ns 'inputs' "$file" | node ../aggregate.js 1 > "${new_file}"
+            new_file="./munged/${folder#.\/artifacts\/conformance-}" # drop the ./artifacts/conformance- prefix
+            cat "$file" | node ./munge.js > "${new_file}"
           done
+
+          # generate the sqlite database from our munged files
+          node ./munge_sql.js ./aggregates.db ./munged/*.json
       - name: Upload Data Aggregates
         # will be very useful for local debugging
         if: (failure() || success())
         uses: actions/upload-artifact@v3
         with:
           name: dashboard-aggregates
-          path: ./aggregates
+          path: |
+            ./munged
+            ./aggregates.db
       - name: Generate Content
         run: |
-          node ./aggregate-into-table.js ./aggregates/*.json > ./table.md
-          cp ./table.md ./www/data/table.md
-          cat ./table.md >> $GITHUB_STEP_SUMMARY
+          node ./munge_aggregates.js ./aggregates.db ./www
       - name: Build with Hugo
         run: |
           hugo \

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `--version` flag shows the current version
 - Metadata logging used to associate tests with custom data like versions, specs identifiers, etc.
 - Output Github's workflow URL with metadata. [PR](https://github.com/ipfs/gateway-conformance/pull/145)
+- Basic Dashboard Output with content generation. [PR](https://github.com/ipfs/gateway-conformance/pull/152)
 
 ## [0.3.0] - 2023-07-31
 ### Added

munge.js

@@ -0,0 +1,129 @@
+/**
+ * This file loads a test2json output from stdin,
+ * and generate a test result structure.
+ * 
+ * The output is a map of test names to test results.
+ * 
+ * A test result is an object with the following fields:
+ * - path: the test path (["Test Something", "Sub Test", ...])
+ * - output: the test stdout
+ * - outcome: "pass" | "fail" | "skip" | "unknown"
+ * - time: the test finish time
+ * - meta: test metadata such as "version", "ipip", etc.
+ */
+const fs = require("fs");
+
+// # read jsonlines from stdin:
+let lines = fs.readFileSync(0, "utf-8");
+
+// # clean input 
+lines = lines
+    .split("\n")
+    // ## remove empty lines
+    .filter((line) => line !== "")
+    // ## extract json
+    .map((line) => {
+        try {
+            return JSON.parse(line);
+        } catch (e) {
+            throw new Error(`Failed to parse line: ${line}: ${e}`);
+        }
+    })
+    // ## Drop lines that are not about a Test
+    .filter((line) => {
+        const { Test } = line;
+        return Test !== undefined;
+    });
+
+// # extract test metadata
+//   For now we look for the '--- META:' marker in test logs.
+//   see details in https://github.com/ipfs/gateway-conformance/pull/125
+const extractMetadata = (line) => {
+    const { Action, Output } = line;
+
+    if (Action !== "output") {
+        return null;
+    }
+
+    const match = Output.match(/.* --- META: (.*)/);
+
+    if (!match) {
+        return null;
+    }
+
+    const metadata = match[1];
+    return JSON.parse(metadata);
+}
+
+// # Group all lines by Test name, and Action
+const groups = {};
+lines.forEach((line) => {
+    const { Test, Action } = line;
+
+    if (!Test || !Action) {
+        throw new Error(`Missing Test field in line: ${JSON.stringify(line)}`);
+    }
+
+    if (!groups[Test]) {
+        groups[Test] = {};
+    }
+
+    if (!groups[Test][Action]) {
+        groups[Test][Action] = [];
+    }
+
+    groups[Test][Action].push(line);
+
+    // Add metadata while we're at it
+    const metadata = extractMetadata(line);
+
+    if (metadata) {
+        if (!groups[Test]["meta"]) {
+            groups[Test]["meta"] = [];
+        }
+        groups[Test]["meta"].push(metadata);
+    }
+});
+
+// # Now that we grouped test results,
+//   merge test results into logical aggregates, like stdouts, metadata, etc.
+const groupTest = (test) => {
+    const { run, output, pass, fail, skip, meta } = test;
+
+    const path = run[0]["Test"].split("/").map((name) => {
+        return name.replace(/_/g, " ");
+    });
+
+    const outputMerged = output.reduce((acc, line) => {
+        const { Output } = line;
+        return acc + Output;
+    }, "");
+
+    const metaMerged = meta ? meta.reduce((acc, line) => {
+        // fail in case of duplicate keys
+        if (Object.keys(acc).some((key) => line[key] !== undefined)) {
+            throw new Error(`Duplicate metadata key: ${JSON.stringify(line)}`);
+        }
+        return { ...acc, ...line };
+    }) : undefined;
+
+    const outcomeLine = (pass || fail || skip || [{ Action: "Unknown" }])[0];
+    const time = outcomeLine["Time"];
+
+    return {
+        path,
+        output: outputMerged,
+        outcome: outcomeLine["Action"],
+        time,
+        meta: metaMerged
+    }
+}
+
+const merged = {}
+Object.entries(groups).forEach(([test, group]) => {
+    merged[test] = groupTest(group);
+})
+
+// output result to stdout
+const result = merged
+fs.writeFileSync(1, JSON.stringify(result, null, 2));