From f46cb70686207ae9dea50927507fe5c78f391210 Mon Sep 17 00:00:00 2001 From: Josh Heyer Date: Fri, 11 Mar 2022 20:45:26 +0000 Subject: [PATCH] Alter BDR, etc. source scripts to sync w/ deletion --- scripts/source/bdr.js | 23 ++++++++++++++--------- scripts/source/dispatch_product.py | 2 +- scripts/source/livecompare.js | 20 ++++++++++++-------- scripts/source/pglogical.js | 19 ++++++++++++------- scripts/source/test/sync-bdr.json | 9 ++++----- 5 files changed, 43 insertions(+), 30 deletions(-) diff --git a/scripts/source/bdr.js b/scripts/source/bdr.js index 134e4522272..6ca15284cbc 100644 --- a/scripts/source/bdr.js +++ b/scripts/source/bdr.js @@ -49,7 +49,7 @@ const destination = path.resolve(args[1]); file.path = destFilepath; try { await fs.mkdir(path.dirname(file.path)); - } catch {} + } catch { } await write(file); }; @@ -76,7 +76,7 @@ const destination = path.resolve(args[1]); const fileAbsolutePath = path.resolve(basePath, dirEntry[navTitle]); const filename = path.relative(basePath, fileAbsolutePath); const destFilepath = path.resolve( - destPath, + basePath, filename.replace(/\//g, "_") + "x", ); @@ -90,7 +90,7 @@ const destination = path.resolve(args[1]); ); if (filename === indexFilename) continue; - process(fileAbsolutePath, filename, destFilepath); + await process(fileAbsolutePath, filename, destFilepath); } } @@ -102,14 +102,19 @@ const destination = path.resolve(args[1]); description: "BDR (Bi-Directional Replication) is a ground-breaking multi-master replication capability for PostgreSQL clusters that has been in full production status since 2014.", }; - process( + await process( path.resolve(basePath, indexFilename), indexFilename, - path.resolve(destPath, indexFilename + "x"), + path.resolve(basePath, indexFilename + "x"), ); // copy images - exec(`rsync -a --delete ${imgPath} ${destPath}`); + await exec(`rsync -am --delete --include="*/" --include="*.png" --include="*.jpg" --include="*.jpeg" --include="*.svg" --exclude="*" ${imgPath} ${destPath}`); + + // copy select files, removing those that have been deleted + await exec(`rsync --archive --recursive --delete --include="*/" --include="*.mdx" --exclude="*" ${basePath}/ ${destPath}/`); + // uncomment this if BDR ever includes images in the normal place + //await exec(`rsync --archive --recursive --delete --include="*/" --include="*.mdx" --include="*/" --include="*.png" --include="*.jpg" --include="*.jpeg" --include="*.svg" --exclude="*" ${basePath}/ ${destPath}/`); })(); // GPP leaves the files littered with these; they alter parsing by flipping sections to HTML context @@ -186,7 +191,7 @@ function bdrTransformer() { if (node.value.trim()) console.warn( `${file.path}:${node.position.start.line}:${node.position.start.column} Stripping HTML content:\n\t ` + - node.value, + node.value, ); parent.children.splice(index, 1); @@ -215,8 +220,8 @@ function bdrTransformer() { const blockTypes = ["root", "paragraph", "listItem", "blockquote"]; for ( let i = ancestors.length - 1, - parent = ancestors[ancestors.length - 1], - child = node; + parent = ancestors[ancestors.length - 1], + child = node; i >= 0; --i, child = parent, parent = ancestors[i] ) { diff --git a/scripts/source/dispatch_product.py b/scripts/source/dispatch_product.py index 41f1decd291..de3d89ca7ba 100755 --- a/scripts/source/dispatch_product.py +++ b/scripts/source/dispatch_product.py @@ -16,7 +16,7 @@ "EnterpriseDB/LiveCompare": f"node {args.workspace}/destination/scripts/source/livecompare.js {args.workspace}/source {args.workspace}/destination --unhandled-rejections=strict", "EnterpriseDB/bdr": f"node {args.workspace}/destination/scripts/source/bdr.js {args.workspace}/source {args.workspace}/destination --unhandled-rejections=strict", "EnterpriseDB/pglogical": f"node {args.workspace}/destination/scripts/source/pglogical.js {args.workspace}/source {args.workspace}/destination --unhandled-rejections=strict", - "EnterpriseDB/harp": f"rsync -a --delete {args.workspace}/source/docs/user_guide/* {args.workspace}/destination/product_docs/docs/harp/2.0", + "EnterpriseDB/harp": f"rsync -a --delete {args.workspace}/source/docs/user_guide/ {args.workspace}/destination/product_docs/docs/harp/2.0/", } ret = os.system( diff --git a/scripts/source/livecompare.js b/scripts/source/livecompare.js index 81c75f7c058..e725c9e72d0 100644 --- a/scripts/source/livecompare.js +++ b/scripts/source/livecompare.js @@ -15,6 +15,7 @@ const yaml = require("js-yaml"); const visit = require("unist-util-visit"); const visitAncestors = require("unist-util-visit-parents"); const mdast2string = require("mdast-util-to-string"); +const { exec } = require("child_process"); const isAbsoluteUrl = require("is-absolute-url"); const fileToMetadata = {}; @@ -38,7 +39,7 @@ const destination = path.resolve(args[1]); file.path = destFilepath; try { await fs.mkdir(path.dirname(file.path), { recursive: true }); - } catch {} + } catch { } await write(file); }; @@ -65,7 +66,7 @@ const destination = path.resolve(args[1]); const fileAbsolutePath = path.resolve(basePath, dirEntry[navTitle]); const filename = path.relative(basePath, fileAbsolutePath); const destFilepath = path.resolve( - destPath, + basePath, filename.replace(/\//g, "_") + "x", ); @@ -79,16 +80,19 @@ const destination = path.resolve(args[1]); ); if (filename === indexFilename) continue; - process(fileAbsolutePath, filename, destFilepath); + await process(fileAbsolutePath, filename, destFilepath); } } // write out index w/ navigation tree - process( + await process( path.resolve(basePath, indexFilename), indexFilename, - path.resolve(destPath, indexFilename + "x"), + path.resolve(basePath, indexFilename + "x"), ); + + // copy select files, removing those that have been deleted + await exec(`rsync --archive --recursive --delete --include="*/" --include="*.mdx" --include="*.png" --include="*.jpg" --include="*.jpeg" --include="*.svg" --exclude="*" ${basePath}/ ${destPath}/`); })(); // GPP leaves the files littered with these; they alter parsing by flipping sections to HTML context @@ -153,7 +157,7 @@ function livecompareTransformer() { if (node.value.trim()) console.warn( `${file.path}:${node.position.start.line}:${node.position.start.column} Stripping HTML content:\n\t ` + - node.value, + node.value, ); parent.children.splice(index, 1); @@ -182,8 +186,8 @@ function livecompareTransformer() { const blockTypes = ["root", "paragraph", "listItem", "blockquote"]; for ( let i = ancestors.length - 1, - parent = ancestors[ancestors.length - 1], - child = node; + parent = ancestors[ancestors.length - 1], + child = node; i >= 0; --i, child = parent, parent = ancestors[i] ) { diff --git a/scripts/source/pglogical.js b/scripts/source/pglogical.js index 84e1b8e837d..95122f3b084 100644 --- a/scripts/source/pglogical.js +++ b/scripts/source/pglogical.js @@ -15,11 +15,13 @@ const yaml = require("js-yaml"); const visit = require("unist-util-visit"); const visitAncestors = require("unist-util-visit-parents"); const mdast2string = require("mdast-util-to-string"); +const { exec } = require("child_process"); const isAbsoluteUrl = require("is-absolute-url"); const fileToMetadata = {}; const args = process.argv.slice(2); const basePath = path.resolve(args[0], "docs/"); +const destination = path.resolve(args[1]); (async () => { const processor = unified() @@ -101,7 +103,7 @@ const basePath = path.resolve(args[0], "docs/"); file.path = destFilepath; try { await fs.mkdir(path.dirname(file.path), { recursive: true }); - } catch {} + } catch { } await write(file); }; @@ -111,22 +113,25 @@ const basePath = path.resolve(args[0], "docs/"); const markdownToProcess = mdIndex.nav; const version = mdIndex.site_name.match(/pglogical (\d+\.\d+)/)[1]; - const destPath = path.resolve("product_docs", "docs", "pglogical", version); + const destPath = path.resolve(destination, "product_docs", "docs", "pglogical", version); const indexFilename = "index.md"; fileToMetadata[indexFilename] = {}; for (const dirEntry of markdownToProcess) { if (!dirEntry) continue; - await processEntry(dirEntry, destPath, indexFilename); + await processEntry(dirEntry, basePath, indexFilename); } // write out index w/ navigation tree await process( path.resolve(basePath, indexFilename), indexFilename, - path.resolve(destPath, "index.mdx"), + path.resolve(basePath, "index.mdx"), ); + + // copy select files, removing those that have been deleted + await exec(`rsync --archive --recursive --delete --include="*/" --include="*.mdx" --include="*.png" --include="*.jpg" --include="*.jpeg" --include="*.svg" --exclude="*" ${basePath}/ ${destPath}/`); })(); // GPP leaves the files littered with these; they alter parsing by flipping sections to HTML context @@ -191,7 +196,7 @@ function pglogicalTransformer() { if (node.value.trim()) console.warn( `${file.path}:${node.position.start.line}:${node.position.start.column} Stripping HTML content:\n\t ` + - node.value, + node.value, ); parent.children.splice(index, 1); @@ -220,8 +225,8 @@ function pglogicalTransformer() { const blockTypes = ["root", "paragraph", "listItem", "blockquote"]; for ( let i = ancestors.length - 1, - parent = ancestors[ancestors.length - 1], - child = node; + parent = ancestors[ancestors.length - 1], + child = node; i >= 0; --i, child = parent, parent = ancestors[i] ) { diff --git a/scripts/source/test/sync-bdr.json b/scripts/source/test/sync-bdr.json index 99042edfb64..2828abe85bc 100644 --- a/scripts/source/test/sync-bdr.json +++ b/scripts/source/test/sync-bdr.json @@ -1,8 +1,7 @@ { "client_payload": { - "repo": "EnterpriseDB/bdr", - "ref": "2461cd8dbad961935e7cf91e81903d42a1997e5f", - "sha": "2461cd8dbad961935e7cf91e81903d42a1997e5f" - + "repo": "EnterpriseDB/bdr", + "ref": "45b2304e8e67e88a2020b6c8a087a3d62d36f47d", + "sha": "45b2304e8e67e88a2020b6c8a087a3d62d36f47d" } -} +} \ No newline at end of file