diff --git a/.github/workflows/sync-and-process-files.yml b/.github/workflows/sync-and-process-files.yml index 690b779dc1e..ecd6d71e723 100644 --- a/.github/workflows/sync-and-process-files.yml +++ b/.github/workflows/sync-and-process-files.yml @@ -2,48 +2,87 @@ name: sync and process files from another repo on: repository_dispatch: types: [sync-to-docs] + workflow_dispatch: + inputs: + repo: + description: Repository to source documentation from + required: true + type: string + ref: + description: Ref name in the source repo + required: true + type: string + sha: + description: SHA in the source repo, should correspond to ref + required: true + type: string + jobs: sync-and-process-files: env: + SOURCE_REPO: ${{ github.event.client_payload.repo || inputs.repo }} + SOURCE_REF: ${{ github.event.client_payload.ref || inputs.ref }} + SOURCE_SHA: ${{ github.event.client_payload.sha || inputs.sha }} + # The body text of the PR requests that will be created - BODY: "Automated changes to pull in and process updates from repo: ${{ github.event.client_payload.repo }} ref: ${{ github.event.client_payload.ref }}" + BODY: | + Automated changes to pull in and process updates from repo: ${{ github.event.client_payload.repo || inputs.repo }} ref: ${{ github.event.client_payload.ref || inputs.ref }} - # The name of the branch that will be created - BRANCH_NAME: automatic_docs_update/repo_${{ github.event.client_payload.repo }}/ref_${{ github.event.client_payload.ref }} + ## Reviewing + - Look for formatting that may not work as intended + - Watch out for local changes (factual corrections, copy edits, link fixes) that may be overwritten + - You may need to resolve conflicts before merging - check the upstream repo for context when this isn't obvious - # The users that should be assigned to the PR as a comma separated list of github usernames. - REVIEWERS: + # The name of the branch that will be created + BRANCH_NAME: automatic_docs_update/repo_${{ github.event.client_payload.repo || inputs.repo }}/ref_${{ github.event.client_payload.ref || inputs.ref }} # The title of the PR request that will be created - TITLE: "Process changes to docs from: repo: ${{ github.event.client_payload.repo }} ref: ${{ github.event.client_payload.ref }}" + TITLE: "Process changes to docs from: repo: ${{ github.event.client_payload.repo || inputs.repo }} ref: ${{ github.event.client_payload.ref || inputs.ref }}" - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: + - name: Check inputs + if: ${{ !env.SOURCE_REPO || !env.SOURCE_REF || !env.SOURCE_SHA }} + run: | + echo "::error title=missing inputs::must provide source repo, source ref and source SHA" + exit 1 + - name: Checkout destination uses: actions/checkout@v4 with: path: destination + lfs: true - name: Checkout source repo uses: actions/checkout@v4 with: - ref: ${{ github.event.client_payload.sha }} - repository: ${{ github.event.client_payload.repo }} + ref: ${{ env.SOURCE_SHA }} + repository: ${{ env.SOURCE_REPO }} token: ${{ secrets.SYNC_FILES_TOKEN }} path: source - name: setup node uses: actions/setup-node@v4 with: - node-version: "14" + node-version: "18" - name: update npm - run: npm install -g npm@7 + run: npm install -g npm@10 - name: Process changes - run: ${{ github.workspace }}/destination/scripts/source/dispatch_product.py ${{ github.event.client_payload.repo }} ${{ github.workspace }} + id: changes + run: ${{ github.workspace }}/destination/scripts/source/dispatch_product.py ${{env.SOURCE_REPO }} ${{ github.workspace }} working-directory: source + - name: Update PR body + if: ${{ steps.changes.outputs.new-tag }} + run: | + echo 'BODY<> $GITHUB_ENV + echo "$BODY" >> $GITHUB_ENV + echo '## After merging' >> $GITHUB_ENV + echo 'Create a tag named `${{ steps.changes.outputs.new-tag }}` that points to the merge commit' >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + - name: Create pull request if: ${{ !env.ACT }} uses: peter-evans/create-pull-request@v6 @@ -51,6 +90,6 @@ jobs: body: ${{ env.BODY }} branch: ${{ env.BRANCH_NAME }} path: destination/ - reviewers: ${{ env.REVIEWERS }} title: ${{ env.TITLE }} token: ${{ secrets.GITHUB_TOKEN }} + diff --git a/scripts/fileProcessor/processors/cnp/add-frontmatters.mjs b/scripts/fileProcessor/processors/cnp/add-frontmatters.mjs index 42471ea7ea3..28d80f81c35 100644 --- a/scripts/fileProcessor/processors/cnp/add-frontmatters.mjs +++ b/scripts/fileProcessor/processors/cnp/add-frontmatters.mjs @@ -51,7 +51,9 @@ navigation: const mkdocsYaml = yaml.load( await fs.readFile("mkdocs.yml", { encoding: "utf8" }), ); - mkdocsYaml.nav.forEach((line) => { + // handle nested / labeled entries (by flattening) + const nav = mkdocsYaml.nav.flatMap(l => l.slice ? l : Object.values(l).flatMap(nl => nl)) + nav.forEach((line) => { // make sure file extensions are stripped off. modifiedFrontmatter = `${modifiedFrontmatter} - ${line.slice(0, -3)}\n`; diff --git a/scripts/fileProcessor/processors/cnp/update-yaml-links.mjs b/scripts/fileProcessor/processors/cnp/update-yaml-links.mjs index 582513f4d08..8a81e80b818 100644 --- a/scripts/fileProcessor/processors/cnp/update-yaml-links.mjs +++ b/scripts/fileProcessor/processors/cnp/update-yaml-links.mjs @@ -6,7 +6,7 @@ import remarkFrontmatter from "remark-frontmatter"; import remarkStringify from "remark-stringify"; import admonitions from "remark-admonitions"; import visit from "unist-util-visit"; -import isAbsoluteUrl from "is-absolute-url"; +import path from "path"; export const process = async (filename, content) => { const processor = unified() @@ -41,8 +41,11 @@ function linkRewriter() { // link rewriter: // - make relative to parent (because gatsby URL paths are always directories) visit(tree, "link", (node) => { - if (isAbsoluteUrl(node.url) || node.url[0] === "/") return; - if (!node.url.includes(".yaml")) return; + let url = new URL(node.url, "fake://do/x/"); + // don't mess with absolute URLs or absolute paths + if (!url.href.startsWith("fake://do/x/")) return; + // only rewrite yaml file links (for historical reasons, these are resolved differently from ordinary paths) + if (!['.yaml', '.yml'].includes(path.extname(url.pathname))) return; node.url = node.url.replace(/^\/?/, "../"); }); }; diff --git a/scripts/source/dispatch_product.py b/scripts/source/dispatch_product.py index 3874bdb67ff..37060f5f30f 100755 --- a/scripts/source/dispatch_product.py +++ b/scripts/source/dispatch_product.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 import argparse -import os +import subprocess import sys parser = argparse.ArgumentParser() @@ -10,8 +10,8 @@ args = parser.parse_args() commands = { - "EnterpriseDB/cloud-native-postgres": f"{args.workspace}/destination/scripts/source/process-cnp-docs.sh {args.workspace}/source {args.workspace }/destination", - "EnterpriseDB/pg4k-pgd": f"{args.workspace}/destination/scripts/source/process-pgd4k-docs.sh {args.workspace}/source {args.workspace }/destination", + "EnterpriseDB/cloud-native-postgres": f"{args.workspace}/destination/scripts/source/process-cnp-docs.sh {args.workspace}/source {args.workspace}/destination", + "EnterpriseDB/pg4k-pgd": f"{args.workspace}/destination/scripts/source/process-pgd4k-docs.sh {args.workspace}/source {args.workspace}/destination", "EnterpriseDB/fe": f"mkdir -p {args.workspace}/destination/icons-pkg && \ cp -fr utils/icons-placeholder/output/* {args.workspace}/destination/icons-pkg/", "EnterpriseDB/LiveCompare": f"node {args.workspace}/destination/scripts/source/livecompare.js {args.workspace}/source {args.workspace}/destination --unhandled-rejections=strict", @@ -21,9 +21,9 @@ "EnterpriseDB/tpa": f"{args.workspace}/destination/scripts/source/process-tpa-docs.sh {args.workspace}/source {args.workspace}/destination", } -ret = os.system( +ret = subprocess.call( f"cd {args.workspace}/destination/scripts/source && \ - npm ci" + npm ci", shell=True ) if ret != 0: @@ -31,7 +31,7 @@ if args.repo in commands: cmd = commands[args.repo] - ret = os.system(cmd) + ret = subprocess.call(cmd, shell=True) else: print( f"The workflow has not been configured for the {args.repo} repo", diff --git a/scripts/source/process-cnp-docs.sh b/scripts/source/process-cnp-docs.sh index a8729484066..27620cb8507 100755 --- a/scripts/source/process-cnp-docs.sh +++ b/scripts/source/process-cnp-docs.sh @@ -1,5 +1,12 @@ #!/bin/bash +# Process: +# - establish previous tag +# - establish new tag +# - create new branch with new content based on previous tag +# (allow conflicts to be resolved when merging with latest mainline branch) +# - identify new tag + if [ -z $1 ] || [ -z $2 ] then echo "the path to the source and destination checkouts must be provided" @@ -12,67 +19,146 @@ CWD=`pwd` SOURCE_CHECKOUT=`cd $1 && pwd` DESTINATION_CHECKOUT=`cd $2 && pwd` +function do_import { + local source=$1 + local dest=$2 + local version=$3 + local cwd=`pwd` + + cd $dest/product_docs/docs/postgres_for_kubernetes/1/ + node "$DESTINATION_CHECKOUT/scripts/source/files-to-ignore.mjs" \ + "$dest/product_docs/docs/postgres_for_kubernetes/1/" \ + > $source/files-to-ignore.txt + + cd $source/docs + + # grab key bit of source for use in docs + cp $source/config/manager/default-monitoring.yaml $source/docs/src/ + + node "$DESTINATION_CHECKOUT/scripts/fileProcessor/main.mjs" \ + -f "src/**/*.md" \ + -p "cnp/add-frontmatters" \ + -p "cnp/flatten-appendices" \ + -p "cnp/replace-github-urls" \ + -p "cnp/update-links" \ + -p "cnp/update-yaml-links" \ + -p "cnp/rewrite-mdextra-anchors" \ + -p "cnp/cleanup-html" \ + -p "cnp/rename-to-mdx" \ + > /dev/null + + node "$DESTINATION_CHECKOUT/scripts/source/merge-indexes.mjs" \ + "$source/docs/src/index.mdx" \ + "$dest/product_docs/docs/postgres_for_kubernetes/1/index.mdx" \ + "$source/docs/src/index.mdx" \ + >> $source/files-to-ignore.txt + + rsync -av --delete --exclude-from=$source/files-to-ignore.txt src/ $dest/product_docs/docs/postgres_for_kubernetes/1/ > /dev/null + + # Archive API docs + local api_ref_dir="$dest/product_docs/docs/postgres_for_kubernetes/1/pg4k.v1" + local current_api_ref="$api_ref_dir/$version.mdx" + mkdir -p "$api_ref_dir" + mv "$dest/product_docs/docs/postgres_for_kubernetes/1/pg4k.v1.mdx" "$current_api_ref" + # TODO: just install yq + node "$DESTINATION_CHECKOUT/scripts/source/update-yaml.mjs" "$current_api_ref" \ + title="API Reference - $version" \ + navTitle="$version" \ + pdfExclude=true + if [ $version == $LATEST_TAG ] + then + local api_ref_index="$api_ref_dir/index.mdx" + cp "$current_api_ref" "$api_ref_index" + node "$DESTINATION_CHECKOUT/scripts/source/update-yaml.mjs" "$api_ref_index" \ + navTitle="API Reference" \ + pdfExclude=null \ + navigation=[`ls "$api_ref_dir" | grep ^v | sed -e 's/\.mdx$//' | sort -V -r | paste -sd "," - `] + fi + node "$DESTINATION_CHECKOUT/scripts/source/update-yaml.mjs" "$current_api_ref" \ + originalFilePath=null + + cd $cwd +} + cd $DESTINATION_CHECKOUT/scripts/fileProcessor npm ci +# grab some information about what we're importing cd $SOURCE_CHECKOUT -# grab some information about what we're importing -CURRENT_TAG=`git describe --exact-match --tags` +git fetch --tags + LATEST_TAG=`git tag | sort -V -r | head -n 1` +CURRENT_TAG=`git describe --exact-match --tags || echo "$LATEST_TAG-next"` +CURRENT_TAG_INDEX=`git tag | sort -V -r | grep -n $CURRENT_TAG | cut -d : -f 1 || echo 0` +PREVIOUS_TAG=`git tag | sort -V -r | head -n $(($CURRENT_TAG_INDEX+1)) | tail -n 1` -# create a temporary worktree to avoid messing up source repo (for local work; CI doesn't care) -git worktree add --detach ./docs-import +cd $DESTINATION_CHECKOUT -cd $DESTINATION_CHECKOUT/product_docs/docs/postgres_for_kubernetes/1/ -node $DESTINATION_CHECKOUT/scripts/source/files-to-ignore.mjs \ - "$DESTINATION_CHECKOUT/product_docs/docs/postgres_for_kubernetes/1/" \ - > $SOURCE_CHECKOUT/docs-import/files-to-ignore.txt - -cd $SOURCE_CHECKOUT/docs-import/docs - -# grab key bit of source for use in docs -cp $SOURCE_CHECKOUT/docs-import/config/manager/default-monitoring.yaml $SOURCE_CHECKOUT/docs-import/docs/src/ - -node $DESTINATION_CHECKOUT/scripts/fileProcessor/main.mjs \ - -f "src/**/*.md" \ - -p "cnp/add-frontmatters" \ - -p "cnp/flatten-appendices" \ - -p "cnp/replace-github-urls" \ - -p "cnp/update-links" \ - -p "cnp/update-yaml-links" \ - -p "cnp/rewrite-mdextra-anchors" \ - -p "cnp/cleanup-html" \ - -p "cnp/rename-to-mdx" - -node $DESTINATION_CHECKOUT/scripts/source/merge-indexes.mjs \ - "$SOURCE_CHECKOUT/docs-import/docs/src/index.mdx" \ - "$DESTINATION_CHECKOUT/product_docs/docs/postgres_for_kubernetes/1/index.mdx" \ - "$SOURCE_CHECKOUT/docs-import/docs/src/index.mdx" \ - >> $SOURCE_CHECKOUT/docs-import/files-to-ignore.txt - -rsync -av --delete --exclude-from=$SOURCE_CHECKOUT/docs-import/files-to-ignore.txt src/ $DESTINATION_CHECKOUT/product_docs/docs/postgres_for_kubernetes/1/ - -# Archive API docs -API_REF_DIR="$DESTINATION_CHECKOUT/product_docs/docs/postgres_for_kubernetes/1/pg4k.v1" -CURRENT_API_REF="$API_REF_DIR/$CURRENT_TAG.mdx" -mv "$DESTINATION_CHECKOUT/product_docs/docs/postgres_for_kubernetes/1/pg4k.v1.mdx" "$CURRENT_API_REF" -# TODO: just install yq -node "$DESTINATION_CHECKOUT/scripts/source/update-yaml.mjs" "$CURRENT_API_REF" \ - title="API Reference - $CURRENT_TAG" \ - navTitle="$CURRENT_TAG" -if [ $CURRENT_TAG == $LATEST_TAG ] +PREVIOUS_COMMIT=`git rev-list -n 1 product/pg4k/$PREVIOUS_TAG || git rev-list -n 1 HEAD` +PREVIOUS_COMMIT_DESC=`git rev-list --format=oneline -n 1 $PREVIOUS_COMMIT` + +if [[ -z "$CURRENT_TAG" || -z "$PREVIOUS_TAG" || -z "$PREVIOUS_COMMIT" || -z "$LATEST_TAG" ]] then - API_REF_INDEX="$API_REF_DIR/index.mdx" - cp "$CURRENT_API_REF" "$API_REF_INDEX" - node "$DESTINATION_CHECKOUT/scripts/source/update-yaml.mjs" "$API_REF_INDEX" \ - navTitle="API Reference" \ - navigation=[`ls "$API_REF_DIR" | grep ^v | sed -e 's/\.mdx$//' | sort -V -r | paste -sd "," - `] + echo "Missing context: CURRENT_TAG=$CURRENT_TAG PREVIOUS_TAG=$PREVIOUS_TAG PREVIOUS_COMMIT=$PREVIOUS_COMMIT LATEST_TAG=$LATEST_TAG" + exit 1 fi -node "$DESTINATION_CHECKOUT/scripts/source/update-yaml.mjs" "$CURRENT_API_REF" \ - originalFilePath=null + +echo "Applying diff between $PREVIOUS_TAG and $CURRENT_TAG, including local changes since $PREVIOUS_COMMIT_DESC" +set -e + +git config user.name "github-actions[bot]" +git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + +# create a temporary worktree to avoid messing up source repo (for local work; CI doesn't care) +cd $SOURCE_CHECKOUT +git worktree add --detach ./docs-import + +cd $DESTINATION_CHECKOUT + +git worktree add --detach ./docs-import + +cd $DESTINATION_CHECKOUT/docs-import +git checkout $PREVIOUS_COMMIT +cd $SOURCE_CHECKOUT/docs-import +git checkout $PREVIOUS_TAG + +do_import $SOURCE_CHECKOUT/docs-import $DESTINATION_CHECKOUT/docs-import $PREVIOUS_TAG + +cd $DESTINATION_CHECKOUT/docs-import +git checkout -b temp/docs-import-$PREVIOUS_TAG +git add product_docs/docs/postgres_for_kubernetes +git commit -m "PG4K import for $PREVIOUS_TAG" + +cd $SOURCE_CHECKOUT +git worktree remove --force ./docs-import +git worktree add --detach ./docs-import + +do_import $SOURCE_CHECKOUT/docs-import $DESTINATION_CHECKOUT/docs-import $CURRENT_TAG + +cd $DESTINATION_CHECKOUT/docs-import + +git checkout -b temp/docs-import-$CURRENT_TAG +git add product_docs/docs/postgres_for_kubernetes +git commit -m "PG4K import for $CURRENT_TAG" + +cd $DESTINATION_CHECKOUT + +git checkout $PREVIOUS_COMMIT + +git cherry-pick --no-commit --strategy=ort -X theirs temp/docs-import-$CURRENT_TAG # cleanup: remove worktree cd $SOURCE_CHECKOUT git worktree remove --force ./docs-import + +cd $DESTINATION_CHECKOUT + +git worktree remove --force ./docs-import + +git branch -D temp/docs-import-$CURRENT_TAG +git branch -D temp/docs-import-$PREVIOUS_TAG + cd $CWD + +echo "new-tag=product/pg4k/$CURRENT_TAG" >> $GITHUB_OUTPUT