diff --git a/db-tabulator/app.ts b/db-tabulator/app.ts index 04b515c..e13ef9b 100644 --- a/db-tabulator/app.ts +++ b/db-tabulator/app.ts @@ -6,7 +6,7 @@ import {NS_CATEGORY, NS_FILE, NS_MAIN} from "../namespaces"; import {formatSummary} from "../reports/commons"; import {MetadataStore} from "./MetadataStore"; import {HybridMetadataStore} from "./HybridMetadataStore"; -import {applyJsPreprocessing, processQueriesExternally} from "./preprocess"; +import {applyJsPostProcessing, processQueriesExternally} from "./postprocess"; import {EventEmitter} from "events"; export const BOT_NAME = 'SDZeroBot'; @@ -43,7 +43,7 @@ export function getQueriesFromText(text: string, title: string): Query[] { return []; } return templates.map((template, idx) => - new Query(template, title, idx + 1, !!template.getValue('preprocess_js')?.trim())); + new Query(template, title, idx + 1, !!template.getValue('postprocess_js')?.trim())); } export async function processQueries(allQueries: Record, notifier?: EventEmitter) { @@ -134,7 +134,7 @@ export class Query extends EventEmitter { /** Internal tracking: for edit summary */ endNotFound = false; - /** Internal tracking: for queries with JS preprocessing enabled */ + /** Internal tracking: for queries with JS postprocessing enabled */ needsExternalRun = false; needsForceKill = false; @@ -397,12 +397,12 @@ export class Query extends EventEmitter { return String(value); }); } - if (this.getTemplateValue('preprocess_js')) { - const jsCode = stripOuterNowikis(this.getTemplateValue('preprocess_js')); + if (this.getTemplateValue('postprocess_js')) { + const jsCode = stripOuterNowikis(this.getTemplateValue('postprocess_js')); try { - result = await applyJsPreprocessing(result, jsCode, this); + result = await applyJsPostProcessing(result, jsCode, this); } catch (e) { - log(`[E] Error in applyJsPreprocessing`); + log(`[E] Error in applyJsPostProcessing`); log(e); } } diff --git a/db-tabulator/database-report.hbs b/db-tabulator/database-report.hbs index 2e5c6a3..350b6e9 100644 --- a/db-tabulator/database-report.hbs +++ b/db-tabulator/database-report.hbs @@ -52,14 +52,14 @@ // From Query class: 'query-executing': data => `Query (${shorten(data.args[0], 80)}) submitted to database.`, 'query-executed': data => `Query finished running in ${data.args[0]} seconds.`, - 'preprocessing': _ => `Started JS preprocessing on query result.`, + 'postprocessing': _ => `Started JS postprocessing on query result.`, 'js-logging': data => `Logging output:
${safeStringify(data.args[0])}
`, - 'js-no-array': _ => error(`JS preprocess() must return an array. `) + 'Saving result without preprocessing.', - 'js-invalid-return': _ => error(`JS preprocess() returned a value which is not transferable. `) + - 'Saving result without preprocessing.', - 'js-failed': data => error(`JS preprocessing failed. `) + `Error: ${data.args[0]}. Saving result without preprocessing.`, + 'js-no-array': _ => error(`JS postprocess() must return an array. `) + 'Saving result without postprocessing.', + 'js-invalid-return': _ => error(`JS postprocess() returned a value which is not transferable. `) + + 'Saving result without postprocessing.', + 'js-failed': data => error(`JS postprocessing failed. `) + `Error: ${data.args[0]}. Saving result without postprocessing.`, 'process-timed-out': _ => error(`Child process timed out`), - 'preprocessing-complete': data => `Finished JS preprocessing on query result in ${data.args[0]} seconds.`, + 'postprocessing-complete': data => `Finished JS postprocessing on query result in ${data.args[0]} seconds.`, 'catastrophic-error': _ => error(`Your custom JS code was force-terminated due to excessive memory or time usage.`), 'saving': data => `Saving ${link(data.args[0])}.`, 'end-not-found': _ => `[WARNING]: No {` + `{database report end}} template was found. Overwriting rest of the page.`, diff --git a/db-tabulator/external-update.ts b/db-tabulator/external-update.ts index 3306fa7..9adbbcf 100644 --- a/db-tabulator/external-update.ts +++ b/db-tabulator/external-update.ts @@ -3,7 +3,7 @@ import {metadataStore, fetchQueriesForPage, processQueriesForPage} from "./app"; /** * Entry point invoked in a child Node.js process for queries - * with custom JS preprocessing enabled. + * with custom JS postprocessing enabled. */ (async function () { diff --git a/db-tabulator/isolate.vm.js b/db-tabulator/isolate.vm.js index 9e61268..3ba8953 100644 --- a/db-tabulator/isolate.vm.js +++ b/db-tabulator/isolate.vm.js @@ -1,5 +1,5 @@ /* eslint-disable no-unused-vars */ -/* global __mwApiGet, __rawReq, __dbQueryResult, preprocess */ +/* global __mwApiGet, __rawReq, __dbQueryResult, postprocess */ (async function() { const bot = { async request(url) { @@ -16,5 +16,5 @@ "${JS_CODE}"; - return JSON.stringify(await preprocess(JSON.parse(__dbQueryResult))); + return JSON.stringify(await postprocess(JSON.parse(__dbQueryResult))); }) diff --git a/db-tabulator/preprocess.ts b/db-tabulator/postprocess.ts similarity index 83% rename from db-tabulator/preprocess.ts rename to db-tabulator/postprocess.ts index 179caee..3c93c5e 100644 --- a/db-tabulator/preprocess.ts +++ b/db-tabulator/postprocess.ts @@ -75,7 +75,7 @@ const apiClient = new Mwn({ }); apiClient.setRequestOptions({ timeout: 10000 }); -const preprocessCodeTemplate = fs.readFileSync(__dirname + '/isolate.vm.js') +const postprocessCodeTemplate = fs.readFileSync(__dirname + '/isolate.vm.js') .toString() .replace(/^\/\*.*?\*\/$/m, ''); // remove linter comments /* ... */ @@ -130,9 +130,9 @@ async function makeSandboxedHttpRequest(url: string) { } } -export async function applyJsPreprocessing(rows: Record[], jsCode: string, query: Query): Promise[]> { - log(`[+] Applying JS preprocessing for ${query}`); - query.emit('preprocessing'); +export async function applyJsPostProcessing(rows: Record[], jsCode: string, query: Query): Promise[]> { + log(`[+] Applying JS postprocessing for ${query}`); + query.emit('postprocessing'); let startTime = process.hrtime.bigint(); // Import dynamically as this has native dependencies @@ -172,10 +172,10 @@ export async function applyJsPreprocessing(rows: Record[], jsCod let result = rows; - let doPreprocessing = async () => { + let doPostProcessing = async () => { try { - // jsCode is expected to declare function preprocess(rows) {...} - let fullCode = preprocessCodeTemplate.replace('"${JS_CODE}"', jsCode); + // jsCode is expected to declare function postprocess(rows) {...} + let fullCode = postprocessCodeTemplate.replace('"${JS_CODE}"', jsCode); let wrapped = await context.eval(fullCode, { reference: true, timeout: softTimeout @@ -190,29 +190,29 @@ export async function applyJsPreprocessing(rows: Record[], jsCod if (Array.isArray(userCodeResultParsed)) { result = userCodeResultParsed; } else { - log(`[E] JS preprocessing for ${query} returned a non-array: ${userCodeResult.slice(0, 100)} ... Ignoring.`); - query.warnings.push(`JS preprocessing didn't return an array of rows, will be ignored`); + log(`[E] JS postprocessing for ${query} returned a non-array: ${userCodeResult.slice(0, 100)} ... Ignoring.`); + query.warnings.push(`JS postprocessing didn't return an array of rows, will be ignored`); query.emit('js-no-array'); } } else { - log(`[E] JS preprocessing for ${query} has an invalid return value: ${userCodeResult}. Ignoring.`); - query.warnings.push(`JS preprocessing must have a transferable return value`); + log(`[E] JS postprocessing for ${query} has an invalid return value: ${userCodeResult}. Ignoring.`); + query.warnings.push(`JS postprocessing must have a transferable return value`); query.emit('js-invalid-return'); } } catch (e) { // Shouldn't occur as we are the ones doing the JSON.stringify - log(`[E] JS preprocessing for ${query} returned a non-JSON: ${userCodeResult.slice(0, 100)}. Ignoring.`); + log(`[E] JS postprocessing for ${query} returned a non-JSON: ${userCodeResult.slice(0, 100)}. Ignoring.`); } } catch (e) { - log(`[E] JS preprocessing for ${query} failed: ${e.toString()}`); + log(`[E] JS postprocessing for ${query} failed: ${e.toString()}`); log(e); - query.warnings.push(`JS preprocessing failed: ${e.toString()}`); + query.warnings.push(`JS postprocessing failed: ${e.toString()}`); query.emit('js-failed', e.toString()); } } await timedPromise( hardTimeout, - doPreprocessing(), + doPostProcessing(), () => { // In case isolated-vm timeout doesn't work log(`[E] Past ${hardTimeout/1000} second timeout, force-disposing isolate`); @@ -222,8 +222,8 @@ export async function applyJsPreprocessing(rows: Record[], jsCod let endTime = process.hrtime.bigint(); let timeTaken = (Number(endTime - startTime) / 1e9).toFixed(3); - log(`[+] JS preprocessing for ${query} took ${timeTaken} seconds, cpuTime: ${isolate.cpuTime}, wallTime: ${isolate.wallTime}.`); - query.emit('preprocessing-complete', timeTaken); + log(`[+] JS postprocessing for ${query} took ${timeTaken} seconds, cpuTime: ${isolate.cpuTime}, wallTime: ${isolate.wallTime}.`); + query.emit('postprocessing-complete', timeTaken); return result; } diff --git a/db-tabulator/test.ts b/db-tabulator/test.ts index f0faa01..1a9cf7f 100644 --- a/db-tabulator/test.ts +++ b/db-tabulator/test.ts @@ -4,7 +4,7 @@ import assert = require("assert"); import {NoMetadataStore} from "./NoMetadataStore"; import {Template} from "../../mwn/build/wikitext"; import {MwnDate} from "../../mwn"; -import {applyJsPreprocessing} from "./preprocess"; +import {applyJsPostProcessing} from "./postprocess"; describe('db-tabulator', () => { @@ -28,10 +28,10 @@ describe('db-tabulator', () => { assert.strictEqual(isUpdateDue(new bot.date().subtract(40, 'hour'), 2), true); }); - it('applyJsPreprocessing', async () => { - console.log(await applyJsPreprocessing( + it('applyJsPostProcessing', async () => { + console.log(await applyJsPostProcessing( [{id: '1', name: 'Main Page'}, {id: '2', name: "Talk:Main Page"}], - `function preprocess(rows) { + `function postprocess(rows) { rows.forEach(row => { row.id = parseInt(row.id) + 100; })