Automatically generate contracts documentation (#3534)

We are creating GH Actions workflows which automatically generates the
contracts documentation in Markdown, based on the functions and the
NatSpec-format comments in the Solidity files stored in the `contracts`
folders in `keep-network/keep-core/solidity/random-beacon` and
`keep-network/keep-core/solidity/ecdsa`. For certain workflow triggers,
the generated documentation will be pushed to the
`threshold-network/threshold` repository, under
`./docs/app-development/random-beacon/random-beacon-api` /
`./docs/app-development/ecdsa/ecdsa-api`. As the `.docs` folder is
synched with Threshold docs GitBook space, the pushed docs will be
displayed in HTML under `docs.threshold.network` domain.

There are two main jobs in the workflows:

* `contracts-docs-publish-preview`
* `contracts-docs-publish`

Both call a reusable workflow `reusable-solidity-docs.yml` which resides
in the `keep-network/ci` repository under `.github/workflows`. The jobs
differ in parameters with witch the reusable action is called. The
common part of the jobs is the beginning stage, where the Solidity files
are being prepared for Markdown generation. During that stage the jobs
will remove the spaces chars between the NatSpec comments markers
(`///`) and the text of the comment. This is done to ensure proper
rendering of the lists in the output docs and is a default action made
by `reusable-solidity-docs.yml`.
Once files are ready, the jobs use Docgen tool to generate the Markdown
docs. The tool needs to be installed in the project and configured in
the `hardhat.config.ts`. The configuration that we use specifies that
the docs should be generated to the `geerated-docs` subfolder, to one
single `index.md` file. The `.sol` files in the
`./solidity/contracts/test` folder will be ignored during generation
(those are test/stub contracts which are not used on Mainnet). A custom
template will be used during docs generations. The template was created
based on the default

https://github.com/OpenZeppelin/solidity-docgen/blob/master/src/themes/markdown/common.hbs
template, but it removes the cursive from the `@dev` type comments
(because the cursive didn't work well with formatting of the lists).
Once the Docgen is run and the `index.md` file is generated, the jobs
will add a Table of Contents to the file, to improve the navigation. The
TOC will be added by the `markdown-toc` tool and will be generated using
`--maxdepth 2` param, which results in listing all the contract names,
but not the functions. Once the TOC is added, we'll start to see the
difference in the behavior of the jobs.

The `contracts-docs-publish-preview` job is triggered for:
* `pull_request` events, if the PR modifies either files in
`./solidity/<project>/contracts` or the workflow files itself,
* `push` events, when the push is made to a branch which name starts
with `releases/mainnet/solidity/` and project's files are modified,
* `workflow_dispatch` events. The job publishes the generated artifacts
as artifacts of the GH Actions workflow run. For PRs, link to the run
with the artifacts will be posted as a PR comment.

The `contracts-docs-publish` job is triggered for:
* published releases, which names start with `refs/tags/solidity/`. The
job pushes the generated file to the
`./docs/app-development/random-beacon/random-beacon-api` /
`./docs/app-development/ecdsa/ecdsa-api`folder in the
`threshold-network/threshold` repo, to the `main` branch. The `.docs`
folder is synched with the GitBook Threshold documentation, so thanks to
the job, the changes in the contracts should be reflected on the
docs.threshold.network. In order for the job to work, we need to
configure a user which will create the commit The user needs to have
right to commit to the `threshold-network/threshold` repo. We also need
to configure following secrets:
- [x] `THRESHOLD_DOCS_GITHUB_TOKEN`
- [x] `THRESHOLD_DOCS_GPG_PRIVATE_KEY_BASE64`
- [x] `THRESHOLD_DOCS_GPG_PASSPHRASE ` We're using a GPG key to be able
to do verified commits.

.github/workflows/contracts-ecdsa-docs.yml

@@ -0,0 +1,86 @@
+name: Solidity ECDSA docs
+
+on:
+  pull_request:
+  push:
+    branches:
+      - releases/mainnet/solidity/**
+    paths:
+      - "solidity/ecdsa/**"
+  release:
+    types:
+      - "published"
+  workflow_dispatch:
+
+jobs:
+  docs-detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      path-filter: ${{ steps.filter.outputs.path-filter }}
+    steps:
+      - uses: actions/checkout@v3
+        if: github.event_name == 'pull_request'
+      - uses: dorny/paths-filter@v2
+        if: github.event_name == 'pull_request'
+        id: filter
+        with:
+          filters: |
+            path-filter:
+              - './solidity/ecdsa/contracts/**'
+              - './.github/workflows/contracts-ecdsa-docs.yml'
+
+  # This job will be triggered for PRs which modify contracts. It will generate
+  # the archive with contracts documentation in Markdown and attatch it to the
+  # workflow run results. Link to the archive will be posted in a PR comment.
+  # The job will also be run after manual triggering and after pushes to the
+  # `releases/mainnet/solidity/**` branches if there were changes to the
+  # `ecdsa` project.
+  contracts-docs-publish-preview:
+    name: Publish preview of contracts documentation
+    needs: docs-detect-changes
+    if: |
+      github.event_name == 'pull_request'
+        || github.event_name == 'push'
+        || github.event_name == 'workflow_dispatch'
+    uses: keep-network/ci/.github/workflows/reusable-solidity-docs.yml@main
+    with:
+      projectDir: /solidity/ecdsa
+      publish: false
+      commentPR: true
+      exportAsGHArtifacts: true
+
+  # This job is needed to avoid a clash of `contracts-docs-publish` jobs for
+  # `random-beacon` and `ecdsa` projects (if both are run and pull the code at
+  # the same time and try to push to the same branch, one of them will fail). 
+  contracts-docs-prepublish-wait:
+    name: Wait for contracts docs to be published
+    needs: docs-detect-changes
+    if: github.event_name == 'release' && startsWith(github.ref, 'refs/tags/solidity/')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Wait 4 minutes
+        run: sleep 240
+
+  # This job will be triggered for releases which name starts with
+  # `refs/tags/solidity/`. It will generate contracts documentation in
+  # Markdown and sync it with a specific path of
+  # `threshold-network/threshold` repository. If changes will be detected,
+  # a PR updating the docs will be created in the destination repository. The
+  # commit pushing the changes will be verified using GPG key.
+  contracts-docs-publish:
+    name: Publish contracts documentation
+    needs: contracts-docs-prepublish-wait
+    uses: keep-network/ci/.github/workflows/reusable-solidity-docs.yml@main
+    with:
+      projectDir: /solidity/ecdsa
+      publish: true
+      verifyCommits: true
+      destinationRepo: threshold-network/threshold
+      destinationFolder: ./docs/app-development/tbtc-v2/ecdsa-api
+      destinationBaseBranch: main
+      userEmail: [email protected]
+      userName: Valkyrie
+    secrets:
+      githubToken: ${{ secrets.THRESHOLD_DOCS_GITHUB_TOKEN }}
+      gpgPrivateKey: ${{ secrets.THRESHOLD_DOCS_GPG_PRIVATE_KEY_BASE64 }}
+      gpgPassphrase: ${{ secrets.THRESHOLD_DOCS_GPG_PASSPHRASE }}

.github/workflows/contracts-random-beacon-docs.yml

@@ -0,0 +1,75 @@
+name: Solidity Random Beacon docs
+
+on:
+  pull_request:
+  push:
+    branches:
+      - releases/mainnet/solidity/**
+    paths:
+      - "solidity/random-beacon/**"
+  release:
+    types:
+      - "published"
+  workflow_dispatch:
+
+jobs:
+  docs-detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      path-filter: ${{ steps.filter.outputs.path-filter }}
+    steps:
+      - uses: actions/checkout@v3
+        if: github.event_name == 'pull_request'
+      - uses: dorny/paths-filter@v2
+        if: github.event_name == 'pull_request'
+        id: filter
+        with:
+          filters: |
+            path-filter:
+              - './solidity/random-beacon/contracts/**'
+              - './.github/workflows/contracts-random-beacon-docs.yml'
+
+  # This job will be triggered for PRs which modify contracts. It will generate
+  # the archive with contracts documentation in Markdown and attatch it to the
+  # workflow run results. Link to the archive will be posted in a PR comment.
+  # The job will also be run after manual triggering and after pushes to the
+  # `releases/mainnet/solidity/**` branches if there were changes to the
+  # `random-beacon` project.
+  contracts-docs-publish-preview:
+    name: Publish preview of contracts documentation
+    needs: docs-detect-changes
+    if: |
+      github.event_name == 'pull_request'
+        || github.event_name == 'push'
+        || github.event_name == 'workflow_dispatch'
+    uses: keep-network/ci/.github/workflows/reusable-solidity-docs.yml@main
+    with:
+      projectDir: /solidity/random-beacon
+      publish: false
+      commentPR: true
+      exportAsGHArtifacts: true
+
+  # This job will be triggered for releases which name starts with
+  # `refs/tags/solidity/`. It will generate contracts documentation in
+  # Markdown and sync it with a specific path of
+  # `threshold-network/threshold` repository. If changes will be detected,
+  # a PR updating the docs will be created in the destination repository. The
+  # commit pushing the changes will be verified using GPG key.
+  contracts-docs-publish:
+    name: Publish contracts documentation
+    needs: docs-detect-changes
+    if: github.event_name == 'release' && startsWith(github.ref, 'refs/tags/solidity/')
+    uses: keep-network/ci/.github/workflows/reusable-solidity-docs.yml@main
+    with:
+      projectDir: /solidity/random-beacon
+      publish: true
+      verifyCommits: true
+      destinationRepo: threshold-network/threshold
+      destinationFolder: ./docs/app-development/random-beacon/random-beacon-api
+      destinationBaseBranch: main
+      userEmail: [email protected]
+      userName: Valkyrie
+    secrets:
+      githubToken: ${{ secrets.THRESHOLD_DOCS_GITHUB_TOKEN }}
+      gpgPrivateKey: ${{ secrets.THRESHOLD_DOCS_GPG_PRIVATE_KEY_BASE64 }}
+      gpgPassphrase: ${{ secrets.THRESHOLD_DOCS_GPG_PASSPHRASE }}

solidity/ecdsa/.prettierignore

@@ -6,4 +6,5 @@ export/
 external/
 hardhat-dependency-compiler/
 typechain/
+docgen-templates/
 export.json

solidity/ecdsa/docgen-templates/common.hbs

@@ -0,0 +1,33 @@
+{{h}} {{name}}
+
+{{#if signature}}
+```solidity
+{{{signature}}}
+```
+{{/if}}
+
+{{{natspec.notice}}}
+
+{{#if natspec.dev}}
+{{{natspec.dev}}}
+{{/if}}
+
+{{#if natspec.params}}
+{{h 2}} Parameters
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+{{#each params}}
+| {{name}} | {{type}} | {{{joinLines natspec}}} |
+{{/each}}
+{{/if}}
+
+{{#if natspec.returns}}
+{{h 2}} Return Values
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+{{#each returns}}
+| {{#if name}}{{name}}{{else}}[{{@index}}]{{/if}} | {{type}} | {{{joinLines natspec}}} |
+{{/each}}
+{{/if}}

solidity/ecdsa/hardhat.config.ts

@@ -9,6 +9,7 @@ import "@tenderly/hardhat-tenderly"
 import "hardhat-contract-sizer"
 import "hardhat-dependency-compiler"
 import "hardhat-gas-reporter"
+import "solidity-docgen"
 
 import "./tasks"
 import { task } from "hardhat/config"
@@ -221,6 +222,12 @@ const config: HardhatUserConfig = {
   typechain: {
     outDir: "typechain",
   },
+  docgen: {
+    outputDir: "generated-docs",
+    templates: "docgen-templates",
+    pages: "single", // `single`, `items` or `files`
+    exclude: ["./test"],
+  },
 }
 
 task(TASK_TEST, "Runs mocha tests").setAction(async (args, hre, runSuper) => {

solidity/ecdsa/package.json

@@ -64,6 +64,7 @@
     "prettier-plugin-solidity": "^1.0.0-beta.19",
     "solhint": "^3.3.6",
     "solhint-config-keep": "github:keep-network/solhint-config-keep",
+    "solidity-docgen": "^0.6.0-beta.35",
     "ts-node": "^10.4.0",
     "typechain": "^6.1.0",
     "typescript": "^4.5.4"

solidity/ecdsa/yarn.lock

@@ -6082,6 +6082,18 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/growl/-/growl-1.10.5.tgz#f2735dc2283674fa67478b10181059355c369e5e"
   integrity sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==
 
+handlebars@^4.7.7:
+  version "4.7.7"
+  resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.7.tgz#9ce33416aad02dbd6c8fafa8240d5d98004945a1"
+  integrity sha512-aAcXm5OAfE/8IXkcZvCepKU3VzW1/39Fb5ZuqMtgI/hT8X2YgoMvBY5dLhq/cpOvw7Lk1nK/UF71aLG/ZnVYRA==
+  dependencies:
+    minimist "^1.2.5"
+    neo-async "^2.6.0"
+    source-map "^0.6.1"
+    wordwrap "^1.0.0"
+  optionalDependencies:
+    uglify-js "^3.1.4"
+
 har-schema@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92"
@@ -8079,6 +8091,11 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd"
   integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==
 
+neo-async@^2.6.0:
+  version "2.6.2"
+  resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f"
+  integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==
+
 next-tick@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.1.0.tgz#1836ee30ad56d67ef281b22bd199f709449b35eb"
@@ -9828,11 +9845,24 @@ solidity-ast@^0.4.15:
   resolved "https://registry.yarnpkg.com/solidity-ast/-/solidity-ast-0.4.31.tgz#c63e42f894cd047826a05dbb8d1e1dfc17282d39"
   integrity sha512-kX6o4XE4ihaqENuRRTMJfwQNHoqWusPENZUlX4oVb19gQdfi7IswFWnThONHSW/61umgfWdKtCBgW45iuOTryQ==
 
+solidity-ast@^0.4.38:
+  version "0.4.46"
+  resolved "https://registry.yarnpkg.com/solidity-ast/-/solidity-ast-0.4.46.tgz#d0745172dced937741d07464043564e35b147c59"
+  integrity sha512-MlPZQfPhjWXqh7YxWcBGDXaPZIfMYCOHYoLEhGDWulNwEPIQQZuB7mA9eP17CU0jY/bGR4avCEUVVpvHtT2gbA==
+
 solidity-comments-extractor@^0.0.7:
   version "0.0.7"
   resolved "https://registry.yarnpkg.com/solidity-comments-extractor/-/solidity-comments-extractor-0.0.7.tgz#99d8f1361438f84019795d928b931f4e5c39ca19"
   integrity sha512-wciNMLg/Irp8OKGrh3S2tfvZiZ0NEyILfcRCXCD4mp7SgK/i9gzLfhY2hY7VMCQJ3kH9UB9BzNdibIVMchzyYw==
 
+solidity-docgen@^0.6.0-beta.35:
+  version "0.6.0-beta.35"
+  resolved "https://registry.yarnpkg.com/solidity-docgen/-/solidity-docgen-0.6.0-beta.35.tgz#174d7fe54efa8b10f7d3cbe0dfc40e52e11bf867"
+  integrity sha512-9QdwK1THk/MWIdq1PEW/6dvtND0pUqpFTsbKwwU9YQIMYuRhH1lek9SsgnsGGYtdJ0VTrXXcVT30q20a8Y610A==
+  dependencies:
+    handlebars "^4.7.7"
+    solidity-ast "^0.4.38"
+
 source-map-resolve@^0.5.0:
   version "0.5.3"
   resolved "https://registry.yarnpkg.com/source-map-resolve/-/source-map-resolve-0.5.3.tgz#190866bece7553e1f8f267a2ee82c606b5509a1a"
@@ -9877,7 +9907,7 @@ source-map@^0.5.6, source-map@^0.5.7:
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc"
   integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=
 
-source-map@^0.6.0:
+source-map@^0.6.0, source-map@^0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
   integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
@@ -10716,6 +10746,11 @@ typical@^5.2.0:
   resolved "https://registry.yarnpkg.com/typical/-/typical-5.2.0.tgz#4daaac4f2b5315460804f0acf6cb69c52bb93066"
   integrity sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==
 
+uglify-js@^3.1.4:
+  version "3.17.4"
+  resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.17.4.tgz#61678cf5fa3f5b7eb789bb345df29afb8257c22c"
+  integrity sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==
+
 ultron@~1.1.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/ultron/-/ultron-1.1.1.tgz#9fe1536a10a664a65266a1e3ccf85fd36302bc9c"
@@ -11548,6 +11583,11 @@ word-wrap@^1.2.3, word-wrap@~1.2.3:
   resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"
   integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==
 
+wordwrap@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb"
+  integrity sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==
+
 wordwrapjs@^4.0.0:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/wordwrapjs/-/wordwrapjs-4.0.1.tgz#d9790bccfb110a0fc7836b5ebce0937b37a8b98f"

solidity/random-beacon/.prettierignore

@@ -5,4 +5,5 @@ deployments/
 external/
 hardhat-dependency-compiler/
 typechain/
+docgen-templates/
 export.json

solidity/random-beacon/docgen-templates/common.hbs

@@ -0,0 +1,33 @@
+{{h}} {{name}}
+
+{{#if signature}}
+```solidity
+{{{signature}}}
+```
+{{/if}}
+
+{{{natspec.notice}}}
+
+{{#if natspec.dev}}
+{{{natspec.dev}}}
+{{/if}}
+
+{{#if natspec.params}}
+{{h 2}} Parameters
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+{{#each params}}
+| {{name}} | {{type}} | {{{joinLines natspec}}} |
+{{/each}}
+{{/if}}
+
+{{#if natspec.returns}}
+{{h 2}} Return Values
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+{{#each returns}}
+| {{#if name}}{{name}}{{else}}[{{@index}}]{{/if}} | {{type}} | {{{joinLines natspec}}} |
+{{/each}}
+{{/if}}

solidity/random-beacon/hardhat.config.ts

@@ -10,6 +10,7 @@ import "hardhat-contract-sizer"
 import "@typechain/hardhat"
 import "hardhat-dependency-compiler"
 import "./tasks"
+import "solidity-docgen"
 
 import { task } from "hardhat/config"
 
@@ -192,6 +193,12 @@ const config: HardhatUserConfig = {
     currency: "USD",
     coinmarketcap: process.env.COINMARKETCAP_API_KEY,
   },
+  docgen: {
+    outputDir: "generated-docs",
+    templates: "docgen-templates",
+    pages: "single", // `single`, `items` or `files`
+    exclude: ["./test"],
+  },
 }
 
 task("check-accounts-count", "Checks accounts count").setAction(async () => {

solidity/random-beacon/package.json

@@ -66,6 +66,7 @@
     "prettier-plugin-solidity": "^1.0.0-beta.18",
     "solhint": "^3.3.6",
     "solhint-config-keep": "github:keep-network/solhint-config-keep",
+    "solidity-docgen": "^0.6.0-beta.35",
     "ts-node": "^10.2.1",
     "typechain": "^7.0.0",
     "typescript": "^4.4.3"