Merging import for cnp

.github/workflows/sync-and-process-files.yml

@@ -2,55 +2,94 @@ name: sync and process files from another repo
 on:
   repository_dispatch:
     types: [sync-to-docs]
+  workflow_dispatch:
+    inputs:
+      repo:
+        description: Repository to source documentation from
+        required: true
+        type: string
+      ref: 
+        description: Ref name in the source repo
+        required: true
+        type: string
+      sha:
+        description: SHA in the source repo, should correspond to ref
+        required: true
+        type: string
+
 jobs:
   sync-and-process-files:
     env:
+      SOURCE_REPO: ${{ github.event.client_payload.repo || inputs.repo }}
+      SOURCE_REF: ${{ github.event.client_payload.ref || inputs.ref }}
+      SOURCE_SHA: ${{ github.event.client_payload.sha || inputs.sha }}
+
       # The body text of the PR requests that will be created
-      BODY: "Automated changes to pull in and process updates from repo: ${{ github.event.client_payload.repo }} ref: ${{ github.event.client_payload.ref }}"
+      BODY: |
+        Automated changes to pull in and process updates from repo: ${{ github.event.client_payload.repo || inputs.repo }} ref: ${{ github.event.client_payload.ref || inputs.ref }}
 
-      # The name of the branch that will be created
-      BRANCH_NAME: automatic_docs_update/repo_${{ github.event.client_payload.repo }}/ref_${{ github.event.client_payload.ref }}
+        ## Reviewing
+        - Look for formatting that may not work as intended
+        - Watch out for local changes (factual corrections, copy edits, link fixes) that may be overwritten 
+        - You may need to resolve conflicts before merging - check the upstream repo for context when this isn't obvious
 
-      # The users that should be assigned to the PR as a comma separated list of github usernames.
-      REVIEWERS:
+      # The name of the branch that will be created
+      BRANCH_NAME: automatic_docs_update/repo_${{ github.event.client_payload.repo || inputs.repo }}/ref_${{ github.event.client_payload.ref || inputs.ref }}
 
       # The title of the PR request that will be created
-      TITLE: "Process changes to docs from: repo: ${{ github.event.client_payload.repo }} ref: ${{ github.event.client_payload.ref }}"
+      TITLE: "Process changes to docs from: repo: ${{ github.event.client_payload.repo || inputs.repo }} ref: ${{ github.event.client_payload.ref || inputs.ref }}"
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
+      - name: Check inputs
+        if: ${{ !env.SOURCE_REPO || !env.SOURCE_REF || !env.SOURCE_SHA }}
+        run: |
+          echo "::error title=missing inputs::must provide source repo, source ref and source SHA"
+          exit 1
+
       - name: Checkout destination
         uses: actions/checkout@v4
         with:
           path: destination
+          lfs: true
 
       - name: Checkout source repo
         uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.client_payload.sha }}
-          repository: ${{ github.event.client_payload.repo }}
+          ref: ${{ env.SOURCE_SHA }}
+          repository: ${{ env.SOURCE_REPO }}
           token: ${{ secrets.SYNC_FILES_TOKEN }}
           path: source
 
       - name: setup node
         uses: actions/setup-node@v4
         with:
-          node-version: "14"
+          node-version: "18"
 
       - name: update npm
-        run: npm install -g npm@7
+        run: npm install -g npm@10
 
       - name: Process changes
-        run: ${{ github.workspace }}/destination/scripts/source/dispatch_product.py ${{ github.event.client_payload.repo }}  ${{ github.workspace }}
+        id: changes
+        run: ${{ github.workspace }}/destination/scripts/source/dispatch_product.py ${{env.SOURCE_REPO }}  ${{ github.workspace }}
         working-directory: source
 
+      - name: Update PR body
+        if: ${{ steps.changes.outputs.new-tag }}
+        run: |
+          echo 'BODY<<EOF' >> $GITHUB_ENV
+          echo "$BODY" >> $GITHUB_ENV
+          echo '## After merging' >> $GITHUB_ENV
+          echo 'Create a tag named `${{ steps.changes.outputs.new-tag }}` that points to the merge commit' >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV          
+
       - name: Create pull request
         if: ${{ !env.ACT }}
         uses: peter-evans/create-pull-request@v6
         with:
           body: ${{ env.BODY }}
           branch: ${{ env.BRANCH_NAME }}
           path: destination/
-          reviewers: ${{ env.REVIEWERS }}
           title: ${{ env.TITLE }}
           token: ${{ secrets.GITHUB_TOKEN }}
+

scripts/fileProcessor/processors/cnp/add-frontmatters.mjs

@@ -51,7 +51,9 @@ navigation:
   const mkdocsYaml = yaml.load(
     await fs.readFile("mkdocs.yml", { encoding: "utf8" }),
   );
-  mkdocsYaml.nav.forEach((line) => {
+  // handle nested / labeled entries (by flattening)
+  const nav = mkdocsYaml.nav.flatMap(l => l.slice ? l : Object.values(l).flatMap(nl => nl))
+  nav.forEach((line) => {
     // make sure file extensions are stripped off.
     modifiedFrontmatter = `${modifiedFrontmatter}  - ${line.slice(0, -3)}\n`;
 

scripts/fileProcessor/processors/cnp/update-yaml-links.mjs

@@ -6,7 +6,7 @@ import remarkFrontmatter from "remark-frontmatter";
 import remarkStringify from "remark-stringify";
 import admonitions from "remark-admonitions";
 import visit from "unist-util-visit";
-import isAbsoluteUrl from "is-absolute-url";
+import path from "path";
 
 export const process = async (filename, content) => {
   const processor = unified()
@@ -41,8 +41,11 @@ function linkRewriter() {
     // link rewriter:
     // - make relative to parent (because gatsby URL paths are always directories)
     visit(tree, "link", (node) => {
-      if (isAbsoluteUrl(node.url) || node.url[0] === "/") return;
-      if (!node.url.includes(".yaml")) return;
+      let url = new URL(node.url, "fake://do/x/");
+      // don't mess with absolute URLs or absolute paths
+      if (!url.href.startsWith("fake://do/x/")) return;
+      // only rewrite yaml file links (for historical reasons, these are resolved differently from ordinary paths)
+      if (!['.yaml', '.yml'].includes(path.extname(url.pathname))) return;
       node.url = node.url.replace(/^\/?/, "../");
     });
   };

scripts/source/dispatch_product.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 import argparse
-import os
+import subprocess
 import sys
 
 parser = argparse.ArgumentParser()
@@ -10,8 +10,8 @@
 args = parser.parse_args()
 
 commands = {
-    "EnterpriseDB/cloud-native-postgres": f"{args.workspace}/destination/scripts/source/process-cnp-docs.sh {args.workspace}/source {args.workspace }/destination",
-    "EnterpriseDB/pg4k-pgd": f"{args.workspace}/destination/scripts/source/process-pgd4k-docs.sh {args.workspace}/source {args.workspace }/destination",
+    "EnterpriseDB/cloud-native-postgres": f"{args.workspace}/destination/scripts/source/process-cnp-docs.sh {args.workspace}/source {args.workspace}/destination",
+    "EnterpriseDB/pg4k-pgd": f"{args.workspace}/destination/scripts/source/process-pgd4k-docs.sh {args.workspace}/source {args.workspace}/destination",
     "EnterpriseDB/fe": f"mkdir -p {args.workspace}/destination/icons-pkg && \
               cp -fr utils/icons-placeholder/output/* {args.workspace}/destination/icons-pkg/",
     "EnterpriseDB/LiveCompare": f"node {args.workspace}/destination/scripts/source/livecompare.js {args.workspace}/source {args.workspace}/destination --unhandled-rejections=strict",
@@ -21,17 +21,17 @@
     "EnterpriseDB/tpa": f"{args.workspace}/destination/scripts/source/process-tpa-docs.sh {args.workspace}/source {args.workspace}/destination",
 }
 
-ret = os.system(
+ret = subprocess.call(
     f"cd {args.workspace}/destination/scripts/source && \
-                  npm ci"
+                  npm ci", shell=True
 )
 
 if ret != 0:
     print(f"npm install failed: {ret}", file=sys.stderr)
 
 if args.repo in commands:
     cmd = commands[args.repo]
-    ret = os.system(cmd)
+    ret = subprocess.call(cmd, shell=True)
 else:
     print(
         f"The workflow has not been configured for the {args.repo} repo",

scripts/source/process-cnp-docs.sh

@@ -1,5 +1,12 @@
 #!/bin/bash
 
+# Process:
+# - establish previous tag
+# - establish new tag
+# - create new branch with new content based on previous tag
+#   (allow conflicts to be resolved when merging with latest mainline branch)
+# - identify new tag
+
 if [ -z $1 ] || [ -z $2 ]
 then
   echo "the path to the source and destination checkouts must be provided"
@@ -12,67 +19,146 @@ CWD=`pwd`
 SOURCE_CHECKOUT=`cd $1 && pwd`
 DESTINATION_CHECKOUT=`cd $2 && pwd`
 
+function do_import {
+  local source=$1
+  local dest=$2
+  local version=$3
+  local cwd=`pwd`
+
+  cd $dest/product_docs/docs/postgres_for_kubernetes/1/
+  node "$DESTINATION_CHECKOUT/scripts/source/files-to-ignore.mjs" \
+    "$dest/product_docs/docs/postgres_for_kubernetes/1/" \
+    > $source/files-to-ignore.txt
+
+  cd $source/docs
+
+  # grab key bit of source for use in docs
+  cp $source/config/manager/default-monitoring.yaml $source/docs/src/
+
+  node "$DESTINATION_CHECKOUT/scripts/fileProcessor/main.mjs" \
+    -f "src/**/*.md" \
+    -p "cnp/add-frontmatters" \
+    -p "cnp/flatten-appendices" \
+    -p "cnp/replace-github-urls" \
+    -p "cnp/update-links" \
+    -p "cnp/update-yaml-links" \
+    -p "cnp/rewrite-mdextra-anchors" \
+    -p "cnp/cleanup-html" \
+    -p "cnp/rename-to-mdx" \
+    > /dev/null
+
+  node "$DESTINATION_CHECKOUT/scripts/source/merge-indexes.mjs" \
+    "$source/docs/src/index.mdx" \
+    "$dest/product_docs/docs/postgres_for_kubernetes/1/index.mdx" \
+    "$source/docs/src/index.mdx" \
+    >> $source/files-to-ignore.txt
+
+  rsync -av --delete --exclude-from=$source/files-to-ignore.txt src/ $dest/product_docs/docs/postgres_for_kubernetes/1/ > /dev/null
+
+  # Archive API docs
+  local api_ref_dir="$dest/product_docs/docs/postgres_for_kubernetes/1/pg4k.v1"
+  local current_api_ref="$api_ref_dir/$version.mdx"
+  mkdir -p "$api_ref_dir"
+  mv "$dest/product_docs/docs/postgres_for_kubernetes/1/pg4k.v1.mdx" "$current_api_ref"
+  # TODO: just install yq
+  node "$DESTINATION_CHECKOUT/scripts/source/update-yaml.mjs" "$current_api_ref" \
+    title="API Reference - $version" \
+    navTitle="$version" \
+    pdfExclude=true
+  if [ $version == $LATEST_TAG ]
+  then
+    local api_ref_index="$api_ref_dir/index.mdx"
+    cp "$current_api_ref" "$api_ref_index"
+    node "$DESTINATION_CHECKOUT/scripts/source/update-yaml.mjs" "$api_ref_index" \
+      navTitle="API Reference" \
+      pdfExclude=null \
+      navigation=[`ls "$api_ref_dir" | grep ^v | sed -e 's/\.mdx$//' | sort -V -r |  paste -sd "," - `]
+  fi 
+  node "$DESTINATION_CHECKOUT/scripts/source/update-yaml.mjs" "$current_api_ref" \
+    originalFilePath=null
+
+  cd $cwd
+}
+
 cd $DESTINATION_CHECKOUT/scripts/fileProcessor
 npm ci
 
+# grab some information about what we're importing
 cd $SOURCE_CHECKOUT
 
-# grab some information about what we're importing
-CURRENT_TAG=`git describe --exact-match --tags`
+git fetch --tags
+
 LATEST_TAG=`git tag | sort -V -r | head -n 1`
+CURRENT_TAG=`git describe --exact-match --tags || echo "$LATEST_TAG-next"`
+CURRENT_TAG_INDEX=`git tag | sort -V -r | grep -n $CURRENT_TAG | cut -d : -f 1 || echo 0`
+PREVIOUS_TAG=`git tag | sort -V -r | head -n $(($CURRENT_TAG_INDEX+1)) | tail -n 1`
 
-# create a temporary worktree to avoid messing up source repo (for local work; CI doesn't care)
-git worktree add --detach ./docs-import
+cd $DESTINATION_CHECKOUT
 
-cd $DESTINATION_CHECKOUT/product_docs/docs/postgres_for_kubernetes/1/
-node $DESTINATION_CHECKOUT/scripts/source/files-to-ignore.mjs \
-  "$DESTINATION_CHECKOUT/product_docs/docs/postgres_for_kubernetes/1/" \
-  > $SOURCE_CHECKOUT/docs-import/files-to-ignore.txt
-
-cd $SOURCE_CHECKOUT/docs-import/docs
-
-# grab key bit of source for use in docs
-cp $SOURCE_CHECKOUT/docs-import/config/manager/default-monitoring.yaml $SOURCE_CHECKOUT/docs-import/docs/src/
-
-node $DESTINATION_CHECKOUT/scripts/fileProcessor/main.mjs \
-  -f "src/**/*.md" \
-  -p "cnp/add-frontmatters" \
-  -p "cnp/flatten-appendices" \
-  -p "cnp/replace-github-urls" \
-  -p "cnp/update-links" \
-  -p "cnp/update-yaml-links" \
-  -p "cnp/rewrite-mdextra-anchors" \
-  -p "cnp/cleanup-html" \
-  -p "cnp/rename-to-mdx"
-
-node $DESTINATION_CHECKOUT/scripts/source/merge-indexes.mjs \
-  "$SOURCE_CHECKOUT/docs-import/docs/src/index.mdx" \
-  "$DESTINATION_CHECKOUT/product_docs/docs/postgres_for_kubernetes/1/index.mdx" \
-  "$SOURCE_CHECKOUT/docs-import/docs/src/index.mdx" \
-  >> $SOURCE_CHECKOUT/docs-import/files-to-ignore.txt
-
-rsync -av --delete --exclude-from=$SOURCE_CHECKOUT/docs-import/files-to-ignore.txt src/ $DESTINATION_CHECKOUT/product_docs/docs/postgres_for_kubernetes/1/
-
-# Archive API docs
-API_REF_DIR="$DESTINATION_CHECKOUT/product_docs/docs/postgres_for_kubernetes/1/pg4k.v1"
-CURRENT_API_REF="$API_REF_DIR/$CURRENT_TAG.mdx"
-mv "$DESTINATION_CHECKOUT/product_docs/docs/postgres_for_kubernetes/1/pg4k.v1.mdx" "$CURRENT_API_REF"
-# TODO: just install yq
-node "$DESTINATION_CHECKOUT/scripts/source/update-yaml.mjs" "$CURRENT_API_REF" \
-  title="API Reference - $CURRENT_TAG" \
-  navTitle="$CURRENT_TAG"
-if [ $CURRENT_TAG == $LATEST_TAG ]
+PREVIOUS_COMMIT=`git rev-list -n 1 product/pg4k/$PREVIOUS_TAG || git rev-list -n 1 HEAD`
+PREVIOUS_COMMIT_DESC=`git rev-list --format=oneline -n 1 $PREVIOUS_COMMIT`
+
+if [[ -z "$CURRENT_TAG" || -z "$PREVIOUS_TAG" || -z "$PREVIOUS_COMMIT" || -z "$LATEST_TAG" ]] 
 then
-  API_REF_INDEX="$API_REF_DIR/index.mdx"
-  cp "$CURRENT_API_REF" "$API_REF_INDEX"
-  node "$DESTINATION_CHECKOUT/scripts/source/update-yaml.mjs" "$API_REF_INDEX" \
-    navTitle="API Reference" \
-    navigation=[`ls "$API_REF_DIR" | grep ^v | sed -e 's/\.mdx$//' | sort -V -r |  paste -sd "," - `]
+  echo "Missing context: CURRENT_TAG=$CURRENT_TAG PREVIOUS_TAG=$PREVIOUS_TAG PREVIOUS_COMMIT=$PREVIOUS_COMMIT LATEST_TAG=$LATEST_TAG"
+  exit 1
 fi 
-node "$DESTINATION_CHECKOUT/scripts/source/update-yaml.mjs" "$CURRENT_API_REF" \
-  originalFilePath=null
+
+echo "Applying diff between $PREVIOUS_TAG and $CURRENT_TAG, including local changes since $PREVIOUS_COMMIT_DESC" 
+set -e
+
+git config user.name "github-actions[bot]"
+git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+# create a temporary worktree to avoid messing up source repo (for local work; CI doesn't care)
+cd $SOURCE_CHECKOUT
+git worktree add --detach ./docs-import
+
+cd $DESTINATION_CHECKOUT
+
+git worktree add --detach ./docs-import
+
+cd $DESTINATION_CHECKOUT/docs-import
+git checkout $PREVIOUS_COMMIT
+cd $SOURCE_CHECKOUT/docs-import
+git checkout $PREVIOUS_TAG
+
+do_import $SOURCE_CHECKOUT/docs-import $DESTINATION_CHECKOUT/docs-import $PREVIOUS_TAG
+
+cd $DESTINATION_CHECKOUT/docs-import
+git checkout -b temp/docs-import-$PREVIOUS_TAG
+git add product_docs/docs/postgres_for_kubernetes
+git commit -m "PG4K import for $PREVIOUS_TAG"
+
+cd $SOURCE_CHECKOUT
+git worktree remove --force ./docs-import
+git worktree add --detach ./docs-import
+
+do_import $SOURCE_CHECKOUT/docs-import $DESTINATION_CHECKOUT/docs-import $CURRENT_TAG
+
+cd $DESTINATION_CHECKOUT/docs-import
+
+git checkout -b temp/docs-import-$CURRENT_TAG
+git add product_docs/docs/postgres_for_kubernetes
+git commit -m "PG4K import for $CURRENT_TAG"
+
+cd $DESTINATION_CHECKOUT
+
+git checkout $PREVIOUS_COMMIT
+
+git cherry-pick --no-commit --strategy=ort -X theirs temp/docs-import-$CURRENT_TAG
 
 # cleanup: remove worktree
 cd $SOURCE_CHECKOUT
 git worktree remove --force ./docs-import
+
+cd $DESTINATION_CHECKOUT
+
+git worktree remove --force ./docs-import
+
+git branch -D temp/docs-import-$CURRENT_TAG
+git branch -D temp/docs-import-$PREVIOUS_TAG
+
 cd $CWD
+
+echo "new-tag=product/pg4k/$CURRENT_TAG" >> $GITHUB_OUTPUT