db-tabulator: rename js preprocess to postprocess for clarity

db-tabulator/app.ts

@@ -6,7 +6,7 @@ import {NS_CATEGORY, NS_FILE, NS_MAIN} from "../namespaces";
 import {formatSummary} from "../reports/commons";
 import {MetadataStore} from "./MetadataStore";
 import {HybridMetadataStore} from "./HybridMetadataStore";
-import {applyJsPreprocessing, processQueriesExternally} from "./preprocess";
+import {applyJsPostProcessing, processQueriesExternally} from "./postprocess";
 import {EventEmitter} from "events";
 
 export const BOT_NAME = 'SDZeroBot';
@@ -43,7 +43,7 @@ export function getQueriesFromText(text: string, title: string): Query[] {
 		return [];
 	}
 	return templates.map((template, idx) =>
-		new Query(template, title, idx + 1, !!template.getValue('preprocess_js')?.trim()));
+		new Query(template, title, idx + 1, !!template.getValue('postprocess_js')?.trim()));
 }
 
 export async function processQueries(allQueries: Record<string, Query[]>, notifier?: EventEmitter) {
@@ -134,7 +134,7 @@ export class Query extends EventEmitter {
 	/** Internal tracking: for edit summary */
 	endNotFound = false;
 
-	/** Internal tracking: for queries with JS preprocessing enabled */
+	/** Internal tracking: for queries with JS postprocessing enabled */
 	needsExternalRun = false;
 	needsForceKill = false;
 
@@ -397,12 +397,12 @@ export class Query extends EventEmitter {
 				return String(value);
 			});
 		}
-		if (this.getTemplateValue('preprocess_js')) {
-			const jsCode = stripOuterNowikis(this.getTemplateValue('preprocess_js'));
+		if (this.getTemplateValue('postprocess_js')) {
+			const jsCode = stripOuterNowikis(this.getTemplateValue('postprocess_js'));
 			try {
-				result = await applyJsPreprocessing(result, jsCode, this);
+				result = await applyJsPostProcessing(result, jsCode, this);
 			} catch (e) {
-				log(`[E] Error in applyJsPreprocessing`);
+				log(`[E] Error in applyJsPostProcessing`);
 				log(e);
 			}
 		}

db-tabulator/database-report.hbs

@@ -52,14 +52,14 @@
 		// From Query class:
 		'query-executing': data => `Query (<code>${shorten(data.args[0], 80)}</code>) submitted to database.`,
 		'query-executed': data => `Query finished running in ${data.args[0]} seconds.`,
-		'preprocessing': _ => `Started JS preprocessing on query result.`,
+		'postprocessing': _ => `Started JS postprocessing on query result.`,
     'js-logging': data => `Logging output: <pre>${safeStringify(data.args[0])}</pre>`,
-		'js-no-array': _ => error(`JS preprocess() must return an array. `) + 'Saving result without preprocessing.',
-		'js-invalid-return': _ => error(`JS preprocess() returned a value which is not transferable. `) +
-				'Saving result without preprocessing.',
-		'js-failed': data => error(`JS preprocessing failed. `) + `Error: ${data.args[0]}. Saving result without preprocessing.`,
+		'js-no-array': _ => error(`JS postprocess() must return an array. `) + 'Saving result without postprocessing.',
+		'js-invalid-return': _ => error(`JS postprocess() returned a value which is not transferable. `) +
+				'Saving result without postprocessing.',
+		'js-failed': data => error(`JS postprocessing failed. `) + `Error: ${data.args[0]}. Saving result without postprocessing.`,
 		'process-timed-out': _ => error(`Child process timed out`),
-		'preprocessing-complete': data => `Finished JS preprocessing on query result in ${data.args[0]} seconds.`,
+		'postprocessing-complete': data => `Finished JS postprocessing on query result in ${data.args[0]} seconds.`,
 		'catastrophic-error': _ => error(`Your custom JS code was force-terminated due to excessive memory or time usage.`),
 		'saving': data => `Saving ${link(data.args[0])}.`,
 		'end-not-found': _ => `[WARNING]: No {` + `{database report end}} template was found. Overwriting rest of the page.`,

db-tabulator/external-update.ts

@@ -3,7 +3,7 @@ import {metadataStore, fetchQueriesForPage, processQueriesForPage} from "./app";
 
 /**
  * Entry point invoked in a child Node.js process for queries
- * with custom JS preprocessing enabled.
+ * with custom JS postprocessing enabled.
  */
 (async function () {
 

db-tabulator/isolate.vm.js

@@ -1,5 +1,5 @@
 /* eslint-disable no-unused-vars */
-/* global __mwApiGet, __rawReq, __dbQueryResult, preprocess */
+/* global __mwApiGet, __rawReq, __dbQueryResult, postprocess */
 (async function() {
 	const bot = {
 		async request(url) {
@@ -16,5 +16,5 @@
 
 	"${JS_CODE}";
 
-	return JSON.stringify(await preprocess(JSON.parse(__dbQueryResult)));
+	return JSON.stringify(await postprocess(JSON.parse(__dbQueryResult)));
 })

db-tabulator/preprocess.ts → db-tabulator/postprocess.ts

@@ -75,7 +75,7 @@ const apiClient = new Mwn({
 });
 apiClient.setRequestOptions({ timeout: 10000 });
 
-const preprocessCodeTemplate = fs.readFileSync(__dirname + '/isolate.vm.js')
+const postprocessCodeTemplate = fs.readFileSync(__dirname + '/isolate.vm.js')
     .toString()
     .replace(/^\/\*.*?\*\/$/m, ''); // remove linter comments /* ... */
 
@@ -130,9 +130,9 @@ async function makeSandboxedHttpRequest(url: string) {
     }
 }
 
-export async function applyJsPreprocessing(rows: Record<string, string>[], jsCode: string, query: Query): Promise<Record<string, any>[]> {
-    log(`[+] Applying JS preprocessing for ${query}`);
-    query.emit('preprocessing');
+export async function applyJsPostProcessing(rows: Record<string, string>[], jsCode: string, query: Query): Promise<Record<string, any>[]> {
+    log(`[+] Applying JS postprocessing for ${query}`);
+    query.emit('postprocessing');
     let startTime = process.hrtime.bigint();
 
     // Import dynamically as this has native dependencies
@@ -172,10 +172,10 @@ export async function applyJsPreprocessing(rows: Record<string, string>[], jsCod
 
     let result = rows;
 
-    let doPreprocessing = async () => {
+    let doPostProcessing = async () => {
         try {
-            // jsCode is expected to declare function preprocess(rows) {...}
-            let fullCode = preprocessCodeTemplate.replace('"${JS_CODE}"', jsCode);
+            // jsCode is expected to declare function postprocess(rows) {...}
+            let fullCode = postprocessCodeTemplate.replace('"${JS_CODE}"', jsCode);
             let wrapped = await context.eval(fullCode, {
                 reference: true,
                 timeout: softTimeout
@@ -190,29 +190,29 @@ export async function applyJsPreprocessing(rows: Record<string, string>[], jsCod
                     if (Array.isArray(userCodeResultParsed)) {
                         result = userCodeResultParsed;
                     } else {
-                        log(`[E] JS preprocessing for ${query} returned a non-array: ${userCodeResult.slice(0, 100)} ... Ignoring.`);
-                        query.warnings.push(`JS preprocessing didn't return an array of rows, will be ignored`);
+                        log(`[E] JS postprocessing for ${query} returned a non-array: ${userCodeResult.slice(0, 100)} ... Ignoring.`);
+                        query.warnings.push(`JS postprocessing didn't return an array of rows, will be ignored`);
                         query.emit('js-no-array');
                     }
                 } else {
-                    log(`[E] JS preprocessing for ${query} has an invalid return value: ${userCodeResult}. Ignoring.`);
-                    query.warnings.push(`JS preprocessing must have a transferable return value`);
+                    log(`[E] JS postprocessing for ${query} has an invalid return value: ${userCodeResult}. Ignoring.`);
+                    query.warnings.push(`JS postprocessing must have a transferable return value`);
                     query.emit('js-invalid-return');
                 }
             } catch (e) { // Shouldn't occur as we are the ones doing the JSON.stringify
-                log(`[E] JS preprocessing for ${query} returned a non-JSON: ${userCodeResult.slice(0, 100)}. Ignoring.`);
+                log(`[E] JS postprocessing for ${query} returned a non-JSON: ${userCodeResult.slice(0, 100)}. Ignoring.`);
             }
         } catch (e) {
-            log(`[E] JS preprocessing for ${query} failed: ${e.toString()}`);
+            log(`[E] JS postprocessing for ${query} failed: ${e.toString()}`);
             log(e);
-            query.warnings.push(`JS preprocessing failed: ${e.toString()}`);
+            query.warnings.push(`JS postprocessing failed: ${e.toString()}`);
             query.emit('js-failed', e.toString());
         }
     }
 
     await timedPromise(
         hardTimeout,
-        doPreprocessing(),
+        doPostProcessing(),
         () => {
             // In case isolated-vm timeout doesn't work
             log(`[E] Past ${hardTimeout/1000} second timeout, force-disposing isolate`);
@@ -222,8 +222,8 @@ export async function applyJsPreprocessing(rows: Record<string, string>[], jsCod
 
     let endTime = process.hrtime.bigint();
     let timeTaken = (Number(endTime - startTime) / 1e9).toFixed(3);
-    log(`[+] JS preprocessing for ${query} took ${timeTaken} seconds, cpuTime: ${isolate.cpuTime}, wallTime: ${isolate.wallTime}.`);
-    query.emit('preprocessing-complete', timeTaken);
+    log(`[+] JS postprocessing for ${query} took ${timeTaken} seconds, cpuTime: ${isolate.cpuTime}, wallTime: ${isolate.wallTime}.`);
+    query.emit('postprocessing-complete', timeTaken);
 
     return result;
 }

db-tabulator/test.ts

@@ -4,7 +4,7 @@ import assert = require("assert");
 import {NoMetadataStore} from "./NoMetadataStore";
 import {Template} from "../../mwn/build/wikitext";
 import {MwnDate} from "../../mwn";
-import {applyJsPreprocessing} from "./preprocess";
+import {applyJsPostProcessing} from "./postprocess";
 
 describe('db-tabulator', () => {
 
@@ -28,10 +28,10 @@ describe('db-tabulator', () => {
 		assert.strictEqual(isUpdateDue(new bot.date().subtract(40, 'hour'), 2), true);
 	});
 
-	it('applyJsPreprocessing', async () => {
-		console.log(await applyJsPreprocessing(
+	it('applyJsPostProcessing', async () => {
+		console.log(await applyJsPostProcessing(
 			[{id: '1', name: 'Main Page'}, {id: '2', name: "Talk:Main Page"}],
-			`function preprocess(rows) {
+			`function postprocess(rows) {
 				rows.forEach(row => {
 					row.id = parseInt(row.id) + 100;
 				})