Lyche link-checking

.github/workflows/pages.yaml

@@ -4,28 +4,62 @@ on:
     branches:
       - main
   pull_request: {}
+  workflow_dispatch:
+    inputs:
+      ignore_linkcheck:
+        description: Ignore link checker results when deploying
+        required: false
+        default: false
+        type: boolean
+  schedule:
+    - cron: "0 0 * * MON"
+    - cron: "0 0 * * THU"
+permissions:
+  pull-requests: write
+env:
+  LYCHEE_VERSION: "0.13.0"
+  HUGO_VERSION: "0.118.2"
+  JAVA_VERSION: "17"
+  JAVA_DISTRIBUTION: "temurin"
 jobs:
   deploy:
     runs-on: ubuntu-22.04
     steps:
+      #
+      # Environment setup
+      #
       - uses: actions/checkout@v3
         with:
           submodules: true
           fetch-depth: 0
       - uses: actions/setup-java@v3
         with:
-          distribution: "temurin"
-          java-version: "17"
+          distribution: "${{ env.JAVA_DISTRIBUTION }}"
+          java-version: "${{ env.JAVA_VERSION }}"
       - name: Setup Hugo
         uses: peaceiris/actions-hugo@v2
         with:
-          hugo-version: "0.118.2"
+          hugo-version: "${{ env.HUGO_VERSION }}"
           extended: true
+      # lifted from https://github.com/lycheeverse/lychee-action/blob/master/action.yml
+      - name: Setup Lychee
+        run: |
+          # Cleanup artifacts from previous run in case it crashed
+          rm -rf "lychee-v${{ env.LYCHEE_VERSION }}-x86_64-unknown-linux-gnu.tar.gz" lychee
+          curl -sLO "https://github.com/lycheeverse/lychee/releases/download/v${{ env.LYCHEE_VERSION }}/lychee-v${{ env.LYCHEE_VERSION }}-x86_64-unknown-linux-gnu.tar.gz"
+          tar -xvzf "lychee-v${{ env.LYCHEE_VERSION }}-x86_64-unknown-linux-gnu.tar.gz"
+          rm "lychee-v${{ env.LYCHEE_VERSION }}-x86_64-unknown-linux-gnu.tar.gz"
+          install -t "$HOME/.local/bin" -D lychee 
+          rm lychee
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
       - name: Setup swap space
         # The Hugo build can require a significant amount of memory
         uses: pierotofy/set-swap-space@49819abfb41bd9b44fb781159c033dba90353a7c
         with:
-          swap-size-gb: 10
+          swap-size-gb: "10"
+      #
+      # Set up cache
+      #
       - name: Get the list of tagged revisions (for cache)
         id: get-revisions
         run: |
@@ -38,21 +72,53 @@ jobs:
           path: |
             site/content/models/v*/
             site/data/models/v*/
-          key: ${{ hashFiles('site/archetypes/**') }}-${{ hashFiles('support/*.sh') }}-${{ steps.get-revisions.outputs.revisions_hash }}
+          key: cache-models-${{ hashFiles('site/archetypes/**') }}-${{ hashFiles('support/*.sh') }}-${{ steps.get-revisions.outputs.revisions_hash }}
           # A new tagged revision will invalidate the primary cache key
           # See https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
           restore-keys: |
-            ${{ hashFiles('site/archetypes/**') }}-${{ hashFiles('support/*.sh') }}
+            cache-models-${{ hashFiles('site/archetypes/**') }}-${{ hashFiles('support/*.sh') }}
+      # Cache lychee results (e.g. to avoid hitting rate limits)
+      - name: Restore lychee cache
+        uses: actions/cache@v3
+        with:
+          path: .lycheecache
+          key: cache-lychee-${{ github.sha }}
+          restore-keys: cache-lychee-
+      #
+      # Build
+      #
       - name: Build
         # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
         # GHA runners have 2 CPUs
         run: make site -j2
+      #
+      # Checks (and check plumbing)
+      #
+      - name: Link Check
+        id: linkcheck
+        run: make linkcheck LYCHEE_EXTRA_FLAGS='--github-token ${{ secrets.GITHUB_TOKEN }}'
+      - name: Upload linkcheck report
+        uses: actions/upload-artifact@v3
+        with:
+          name: linkcheck-report
+          path: lychee_report.md
+          retention-days: 5
+      - name: Comment broken links
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          path: lychee_report.md
+          skip_unchanged: true
+        if: github.event_name == 'pull_request'
+      #
+      # Deployment
+      #
       - name: Deploy
         uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305
-        if: github.ref == 'refs/heads/main'
+        # Deploy if on main branch and EITHER the linkcheck succeeds or ignore_linkcheck has been flipped
+        #   crucially ignore periodic checks
+        if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (steps.linkcheck.outcome == 'success' || github.event.inputs.ignore_linkcheck)
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          # personal_token: ${{ secrets.COMMIT_TOKEN }}
           enable_jekyll: false
           publish_dir: ./site/public
           publish_branch: nist-pages

.gitignore

@@ -0,0 +1,2 @@
+lychee_report.md
+.lycheecache

Makefile

@@ -6,37 +6,40 @@ SHELL:=/usr/bin/env bash
 help: ## Show this help message
 	@grep -E '^[a-zA-Z_-]+:.*?##.*$$' $(MAKEFILE_LIST) | awk 'BEGIN { \
 	 FS = ":.*?## "; \
-	 printf "\033[1m%-30s\033[0m %s\n", "TARGET", "DESCRIPTION" \
+	 printf "\033[1m%-30s %s\033[0m\n", "TARGET", "DESCRIPTION" \
 	} \
 	{ printf "\033[32m%-30s\033[0m %s\n", $$1, $$2 }'
 
 .PHONY: clean
-clean: clean-modeldoc clean-site clean-release-assets ## Clean all
+clean: clean-modeldoc clean-site clean-release-assets clean-linkcheck ## Clean all
 
 #
 # Website generation / hugo
 #
 
+# Override REVISIONS to build a subset of the site or a special branch
+#   (e.g. `make site REVISIONS='v1.1.0 my-special-branch'`)
 REVISIONS:=develop $(shell ./support/list_revisions.sh)
+
 MODELDOC_CONTENT_DIR:=site/content/models
 MODELDOC_REVISION_CONTENT_DIR:=$(patsubst %,$(MODELDOC_CONTENT_DIR)/%/,$(REVISIONS))
 MODELDOC_DATA_DIR:=site/data/models
 MODELDOC_REVISION_DATA_DIR:=$(patsubst %,$(MODELDOC_DATA_DIR)/%/,$(REVISIONS))
-SITE_OUTPUT:=site/public
+SITE_OUTPUT_DIR:=site/public
 
 .PHONY: serve
 serve: modeldoc release-assets ## Spin up a static web server for local dev
 	cd site; hugo serve
 
 .PHONY: site
-site: $(SITE_OUTPUT) ## Build the site
+site: $(SITE_OUTPUT_DIR) ## Build the site
 
-$(SITE_OUTPUT): modeldoc release-assets
+$(SITE_OUTPUT_DIR): $(MODELDOC_REVISION_CONTENT_DIR) $(RELEASE_ASSET_REDIRECTS_DIR)
 	cd site; hugo --minify
 
 .PHONY: clean-site
 clean-site: ## Clean the site
-	rm -fr $(SITE_OUTPUT)
+	rm -fr $(SITE_OUTPUT_DIR)
 
 #
 # Model documentation
@@ -68,4 +71,30 @@ $(RELEASE_ASSET_REDIRECTS_DIR):
 
 .PHONY: clean-release-assets
 clean-release-assets: ## Clean release redirects
-	rm -fr site/content/release-assets/latest/
+	rm -fr $(RELEASE_ASSET_REDIRECTS_DIR)
+
+#
+# Checks
+#
+
+LYCHEE_OUTPUT_FILE:=lychee_report.md
+LYCHEE_IGNORE_FILE:=./support/lychee_ignore.txt
+LYCHEE_CONFIG_FILE:=./support/lychee.toml
+# Flags that currently cannot be configured via the configuration file
+LYCHEE_FLAGS:=--verbose --format markdown
+# Extra flags for the user to override (used to set github token in GHA workflow)
+LYCHEE_EXTRA_FLAGS:=
+
+.PHONY: linkcheck
+linkcheck: $(LYCHEE_OUTPUT_FILE) ## Generate a report of all site links
+
+$(LYCHEE_OUTPUT_FILE): $(SITE_OUTPUT_DIR)
+	lychee \
+		--exclude-file '$(LYCHEE_IGNORE_FILE)' \
+		--config '$(LYCHEE_CONFIG_FILE)' \
+		--output $(LYCHEE_OUTPUT_FILE) \
+		$(LYCHEE_FLAGS) $(LYCHEE_EXTRA_FLAGS) \
+		'$(SITE_OUTPUT_DIR)/**/*.html'
+
+clean-linkcheck: ## Clean the linkcheck report
+	rm -f $(LYCHEE_OUTPUT_FILE)

README.md

@@ -22,7 +22,22 @@ To see other common operations, refer to the [`Makefile`](./Makefile) or simply
 
 ### Local development
 
-The target `make serve` runs builds the site dependencies and runs `hugo serve`.
+The target `make serve` runs builds the site and runs `hugo serve`.
+
+### Link-checking
+
+This repository uses [Lychee](https://lychee.cli.rs/#/) to crawl the built site for bad external and internal links.
+To perform link-checking, run the target `make linkcheck`.
+The link checker produces a report, nominally located at `./lychee_report.md`.
+
+Lychee is configured to ignore URL patterns present in the [`./support/lychee_ignore.txt`](./support/lychee_ignore.txt).
+
+[Additional configuration](https://lychee.cli.rs/#/usage/cli) can be specified using the Makefile variable `LYCHEE_EXTRA_FLAGS`:
+
+```sh
+# Cache server responses to save bandwidth on repeated runs
+make linkcheck LYCHEE_EXTRA_FLAGS='--cache'
+```
 
 ### Speeding up the build
 

site/.gitignore

@@ -5,4 +5,7 @@ public
 # all directories here are generated on the fly
 content/models/*/
 data/models/*/
-content/release-assets/latest/*
+content/release-assets/latest/*
+
+# Aliased to Metaschema datatypes page
+!content/models/datatypes/

site/archetypes/complete-reference/json-definitions.md

@@ -14,7 +14,7 @@ aliases:
 {{ end }}
 ---
 
-The following is a reference for the JSON object definitions derived from this model's [metaschema](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/src/metaschema/oscal_{{ getenv "HUGO_SCHEMA_ID" }}_metaschema.xml), which imports the metaschemas for all of the OSCAL models.
+The following is a reference for the JSON object definitions derived from this model's [metaschema](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/src/metaschema/oscal_{{ getenv "HUGO_MODEL_RAWNAME" }}_metaschema.xml), which imports the metaschemas for all of the OSCAL models.
 
 {{< rawhtml >}}
 {{ os.ReadFile (printf "%s/%s" (getenv "HUGO_MODEL_DATA_DIR") "json-definitions.html") }}

site/archetypes/complete-reference/json-index.md

@@ -15,7 +15,7 @@ aliases:
 ---
 
 The following is an index of each JSON property used in the JSON format, which represents the combination of all OSCAL models.
-The combined schema can be {{ if eq (getenv "HUGO_REF_TYPE") "tag" }}found [here](https://github.com/usnistgov/OSCAL/releases/download/{{ getenv "HUGO_REF_BRANCH" }}/oscal_{{ getenv "HUGO_SCHEMA_ID" }}_schema.json){{ else }}built using [the following instructions](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/build/README.md#artifact-generation){{ end }}.
+The combined schema can be {{ if eq (getenv "HUGO_REF_TYPE") "tag" }}found [here](https://github.com/usnistgov/OSCAL/releases/download/{{ getenv "HUGO_REF_BRANCH" }}/oscal_{{ getenv "HUGO_MODEL_RAWNAME" }}_schema.json){{ else }}built using [the following instructions](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/build/README.md#artifact-generation){{ end }}.
 Each entry in the index lists all uses of the given property in the format, which is linked to the corresponding entry in the [JSON Format Reference](../json-reference/).
 Each entry also lists the formal name for the definition which is linked to the corresponding JSON definition in the [JSON Format Metaschema Reference](../json-definitions/).
 

site/archetypes/complete-reference/json-outline.md

@@ -14,7 +14,7 @@ aliases:
 ---
 
 The following outline is a representation of the JSON format for the combination of all OSCAL models.
-The combined schema can be {{ if eq (getenv "HUGO_REF_TYPE") "tag" }}found [here](https://github.com/usnistgov/OSCAL/releases/download/{{ getenv "HUGO_REF_BRANCH" }}/oscal_{{ getenv "HUGO_SCHEMA_ID" }}_schema.json){{ else }}built using [the following instructions](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/build/README.md#artifact-generation){{ end }}.
+The combined schema can be {{ if eq (getenv "HUGO_REF_TYPE") "tag" }}found [here](https://github.com/usnistgov/OSCAL/releases/download/{{ getenv "HUGO_REF_BRANCH" }}/oscal_{{ getenv "HUGO_MODEL_RAWNAME" }}_schema.json){{ else }}built using [the following instructions](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/build/README.md#artifact-generation){{ end }}.
 For each property, the name links to the corresponding entry in the [JSON Format Reference](../json-reference/).
 The cardinality and data type are also provided for each property where appropriate.
 

site/archetypes/complete-reference/xml-definitions.md

@@ -14,7 +14,7 @@ aliases:
 {{ end }}
 ---
 
-The following is a reference for the XML element and attribute types derived from this model's [metaschema](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/src/metaschema/oscal_{{ getenv "HUGO_SCHEMA_ID" }}_metaschema.xml).
+The following is a reference for the XML element and attribute types derived from this model's [metaschema](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/src/metaschema/oscal_{{ getenv "HUGO_MODEL_RAWNAME" }}_metaschema.xml).
 
 {{< rawhtml >}}
 {{ os.ReadFile (printf "%s/%s" (getenv "HUGO_MODEL_DATA_DIR") "xml-definitions.html") }}

site/archetypes/complete-reference/xml-index.md

@@ -16,7 +16,7 @@ aliases:
 
 The following is an index of each XML element and attribute used in the XML format for this model.
 Each entry in the index lists all uses of the given element or attribute in the format, which is linked to the corresponding entry in the [XML Format Reference](../xml-reference/).
-The combined schema can be {{ if eq (getenv "HUGO_REF_TYPE") "tag" }}found [here](https://github.com/usnistgov/OSCAL/releases/download/{{ getenv "HUGO_REF_BRANCH" }}/oscal_{{ getenv "HUGO_SCHEMA_ID" }}_schema.xsd){{ else }}built using [the following instructions](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/build/README.md#artifact-generation){{ end }}.
+The combined schema can be {{ if eq (getenv "HUGO_REF_TYPE") "tag" }}found [here](https://github.com/usnistgov/OSCAL/releases/download/{{ getenv "HUGO_REF_BRANCH" }}/oscal_{{ getenv "HUGO_MODEL_RAWNAME" }}_schema.xsd){{ else }}built using [the following instructions](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/build/README.md#artifact-generation){{ end }}.
 Each entry also lists the formal name for the given element or attribute which is linked to the corresponding XML type in the [XML Format Metaschema Reference](../xml-definitions/).
 
 {{< rawhtml >}}

site/archetypes/complete-reference/xml-outline.md

@@ -14,7 +14,7 @@ aliases:
 ---
 
 The following outline is a representation of the XML format for the combination of all OSCAL models.
-The combined schema can be {{ if eq (getenv "HUGO_REF_TYPE") "tag" }}found [here](https://github.com/usnistgov/OSCAL/releases/download/{{ getenv "HUGO_REF_BRANCH" }}/oscal_{{ getenv "HUGO_SCHEMA_ID" }}_schema.xsd){{ else }}built using [the following instructions](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/build/README.md#artifact-generation){{ end }}.
+The combined schema can be {{ if eq (getenv "HUGO_REF_TYPE") "tag" }}found [here](https://github.com/usnistgov/OSCAL/releases/download/{{ getenv "HUGO_REF_BRANCH" }}/oscal_{{ getenv "HUGO_MODEL_RAWNAME" }}_schema.xsd){{ else }}built using [the following instructions](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/build/README.md#artifact-generation){{ end }}.
 For each element or attribute, the name links to the corresponding entry in the [XML Format Reference](../xml-reference/).
 The cardinality and data type are also provided for each element or attribute where appropriate.
 

site/archetypes/model-reference/_index.md

@@ -23,4 +23,4 @@ The following reference documentation is available for the OSCAL {{ getenv "HUGO
 - **Outline ([JSON/YAML](json-outline/), [XML](xml-outline/)):** Provides a brief listing of the model's information items organized hierarchically. Useful for understanding the structure and basic syntax of the model in a given format.
 - **Reference ([JSON/YAML](json-reference/), [XML](xml-reference/)):** Provides a detailed description of the structure and syntax for the model's information items organized hierarchically. Useful for understanding how to use information items within the model in the given format. 
 - **Index ([JSON/YAML](json-index/), [XML](xml-index/)):** Provides a listing of the model's information items organized by where they are used.
-- **Definitions ([JSON/YAML](json-definitions/), [XML](xml-definitions/)):** Provides a reference for the XML Schema types and JSON Schema definitions derived from this model's [metaschema](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/src/metaschema/oscal_{{ getenv "HUGO_SCHEMA_ID" }}). A metaschema is used to [express the model](https://pages.nist.gov/OSCAL/concepts/layer/overview/#modeling-approach) in a format agnostic way, that is then used to generate the XML and JSON Schema, XML <-> JSON content converters, and this reference documentation.
+- **Definitions ([JSON/YAML](json-definitions/), [XML](xml-definitions/)):** Provides a reference for the XML Schema types and JSON Schema definitions derived from this model's [metaschema](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/src/metaschema/oscal_{{ getenv "HUGO_MODEL_RAWNAME" }}_metaschema.xml). A metaschema is used to [express the model](https://pages.nist.gov/OSCAL/concepts/layer/overview/#modeling-approach) in a format agnostic way, that is then used to generate the XML and JSON Schema, XML <-> JSON content converters, and this reference documentation.

site/archetypes/model-reference/json-definitions.md

@@ -14,7 +14,7 @@ aliases:
 {{ end }}
 ---
 
-The following is a reference for the JSON object definitions derived from the [metaschema](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}//src/metaschema/oscal_{{ getenv "HUGO_SCHEMA_ID" }}_metaschema.xml) for this [model](https://pages.nist.gov/OSCAL/concepts/layer/{{ getenv "HUGO_LAYER_ID" }}/{{ getenv "HUGO_SCHEMA_ID" }}/).
+The following is a reference for the JSON object definitions derived from the [metaschema](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/src/metaschema/oscal_{{ getenv "HUGO_MODEL_RAWNAME" }}_metaschema.xml) for this [model](https://pages.nist.gov/OSCAL/concepts/layer/{{ getenv "HUGO_LAYER_ID" }}/{{ getenv "HUGO_SCHEMA_ID" }}/).
 
 {{< rawhtml >}}
 {{ os.ReadFile (printf "%s/%s" (getenv "HUGO_MODEL_DATA_DIR") "json-definitions.html") }}

site/archetypes/model-reference/json-index.md

@@ -15,7 +15,7 @@ aliases:
 ---
 
 The following is an index of each JSON property used in the JSON format for this [model](https://pages.nist.gov/OSCAL/concepts/layer/{{ getenv "HUGO_LAYER_ID" }}/{{ getenv "HUGO_SCHEMA_ID" }}/),
-whose schema can be {{ if eq (getenv "HUGO_REF_TYPE") "tag" }}found [here](https://github.com/usnistgov/OSCAL/releases/download/{{ getenv "HUGO_REF_BRANCH" }}/oscal_{{ getenv "HUGO_SCHEMA_ID" }}_schema.json){{ else }}built using [the following instructions](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/build/README.md#artifact-generation){{ end }}.
+whose schema can be {{ if eq (getenv "HUGO_REF_TYPE") "tag" }}found [here](https://github.com/usnistgov/OSCAL/releases/download/{{ getenv "HUGO_REF_BRANCH" }}/oscal_{{ getenv "HUGO_MODEL_RAWNAME" }}_schema.json){{ else }}built using [the following instructions](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/build/README.md#artifact-generation){{ end }}.
 Each entry in the index lists all uses of the given property in the format, which is linked to the corresponding entry in the [JSON Format Reference](../json-reference/).
 Each entry also lists the formal name for the definition which is linked to the corresponding JSON definition in the [JSON Format Metaschema Reference](../json-definitions/).
 

site/archetypes/model-reference/json-outline.md

@@ -14,7 +14,7 @@ aliases:
 ---
 
 The following outline is a representation of the JSON format for this [model](https://pages.nist.gov/OSCAL/concepts/layer/{{ getenv "HUGO_LAYER_ID" }}/{{ getenv "HUGO_SCHEMA_ID" }}/),
-whose schema can be {{ if eq (getenv "HUGO_REF_TYPE") "tag" }}found [here](https://github.com/usnistgov/OSCAL/releases/download/{{ getenv "HUGO_REF_BRANCH" }}/oscal_{{ getenv "HUGO_SCHEMA_ID" }}_schema.json){{ else }}built using [the following instructions](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/build/README.md#artifact-generation){{ end }}.
+whose schema can be {{ if eq (getenv "HUGO_REF_TYPE") "tag" }}found [here](https://github.com/usnistgov/OSCAL/releases/download/{{ getenv "HUGO_REF_BRANCH" }}/oscal_{{ getenv "HUGO_MODEL_RAWNAME" }}_schema.json){{ else }}built using [the following instructions](https://github.com/usnistgov/OSCAL/blob/{{ getenv "HUGO_REF_BRANCH" }}/build/README.md#artifact-generation){{ end }}.
 For each property, the name links to the corresponding entry in the [JSON Format Reference](../json-reference/).
 The cardinality and data type are also provided for each property where appropriate.