From 63c007d520eecdd7db3094bac86d18273aaace1a Mon Sep 17 00:00:00 2001
From: Siddharth VP <siddharthvp@gmail.com>
Date: Thu, 17 Jun 2021 21:58:09 +0530
Subject: [PATCH] db-tb: update, escape comment fields

---
 TextExtractor.test.js | 16 +++-------------
 db-tabulator/app.ts   | 28 +++++++++++++++++-----------
 db-tabulator/main.ts  |  2 ++
 reports/commons.js    |  2 +-
 4 files changed, 23 insertions(+), 25 deletions(-)

diff --git a/TextExtractor.test.js b/TextExtractor.test.js
index 9f513da..c4aa40f 100644
--- a/TextExtractor.test.js
+++ b/TextExtractor.test.js
@@ -1,5 +1,6 @@
 const assert = require('assert');
 const {bot, TextExtractor} = require('./botbase');
+const {preprocessDraftForExtract} = require('./reports/commons')
 
 describe('TextExtractor', () => {
 	before(function() {
@@ -33,22 +34,11 @@ Arthur was an fine tailor.
 	});
 
 	it('runs preprocessHook', function () {
-		let text = `[[User:Example]] 21:09, 30 May 2020 (UTC){{AFC submission|t||ts=20200530210953|u=Harshit567|ns=118|demo=}}
+		let text = `[[User:Example]] 21:09, 30 May 2020 (UTC){{AfC submission|t||ts=20200530210953|u=Harshit567|ns=118|demo=}}
 	
 ==References==`;
 
-		let extract = TextExtractor.getExtract(text, 250, 500, function(text) {
-			let wkt = new bot.wikitext(text);
-			wkt.parseTemplates({
-				namePredicate: name => {
-					return /infobox/i.test(name) || name === 'AFC submission';
-				}
-			});
-			for (let template of wkt.templates) {
-				wkt.removeEntity(template);
-			}
-			return wkt.getText();
-		});
+		let extract = TextExtractor.getExtract(text, 250, 500, preprocessDraftForExtract);
 
 		assert.strictEqual(extract, `[[User:Example]] 21:09, 30 May 2020 (UTC)`);
 	});
diff --git a/db-tabulator/app.ts b/db-tabulator/app.ts
index 5d40380..b850e1c 100644
--- a/db-tabulator/app.ts
+++ b/db-tabulator/app.ts
@@ -3,10 +3,11 @@ import { enwikidb, SQLError } from "../db";
 import { Template } from "../../mwn/build/wikitext";
 import { arrayChunk, lowerFirst, readFile, writeFile } from "../utils";
 import { NS_CATEGORY, NS_FILE } from "../namespaces";
+const {formatSummary} = require('../reports/commons');
 
 export const BOT_NAME = 'SDZeroBot';
-export const TEMPLATE = 'User:SDZeroBot/Database report';
-export const TEMPLATE_END = 'User:SDZeroBot/Database report end';
+export const TEMPLATE = 'Database report';
+export const TEMPLATE_END = 'Database report end';
 export const SUBSCRIPTIONS_CATEGORY = 'SDZeroBot database report subscriptions';
 export const QUERY_TIMEOUT = 600;
 export const FAKE_INPUT_FILE = 'fake-configs.wikitext';
@@ -27,10 +28,6 @@ export async function fetchQueries(): Promise<Record<string, Query[]>> {
 		if (pg.ns === 0) { // sanity check: don't work in mainspace
 			continue;
 		}
-		// Only work in bot/op userspaces until BRFA approval
-		if (!pg.title.startsWith('User:SD0001/') && !pg.title.startsWith('User:SDZeroBot/')) {
-			continue;
-		}
 		let text = pg.revisions[0].content;
 		allQueries[pg.title] = getQueriesFromText(text, pg.title);
 	}
@@ -56,10 +53,6 @@ export async function processQueries(allQueries: Record<string, Query[]>) {
 }
 
 export async function fetchQueriesForPage(page: string): Promise<Query[]> {
-	// Only work in bot/op userspaces until BRFA approval
-	if (!page.startsWith('User:SD0001/') && !page.startsWith('User:SDZeroBot/')) {
-		return null;
-	}
 	let text = (await bot.read(page))?.revisions?.[0]?.content;
 	if (!text) {
 		return null;
@@ -89,6 +82,7 @@ class Query {
 	sql: string;
 	wikilinkConfig: Array<{columnIndex: number, namespace: string, showNamespace: boolean}>;
 	excerptConfig: Array<{srcIndex: number, destIndex: number, namespace: string, charLimit: number, charHardLimit: number}>;
+	commentConfig: number[];
 	warnings: string[] = [];
 
 	constructor(template: Template, page: string) {
@@ -143,6 +137,11 @@ class Query {
 				return true;
 			}) || [];
 
+		// TODO: show warning on wrong syntax
+		this.commentConfig = this.getTemplateValue('comments')
+			?.split(',')
+			.map(e => parseInt(e.trim()) + 1) || [];
+
 		this.excerptConfig = this.getTemplateValue('excerpts')
 			?.split(',')
 			.map(e => {
@@ -255,7 +254,7 @@ class Query {
 			// Stringify everything
 			result = this.transformColumn(result, i, (value: string | number | null | Date) => {
 				if (value === null) return '';
-				if (value instanceof Date) return value.toISOString(); // is this ever possible?
+				if (value instanceof Date) return value.toISOString();
 				return String(value);
 			});
 		}
@@ -307,6 +306,13 @@ class Query {
 			});
 		});
 
+		// Format edit summaries / log action summaries
+		this.commentConfig.forEach(columnIndex => {
+			result = this.transformColumn(result, columnIndex, (value) => {
+				return formatSummary(value);
+			});
+		});
+
 		this.getTemplateValue('remove_underscores')?.split(',').forEach(num => {
 			let columnIndex = parseInt(num.trim());
 			if (isNaN(columnIndex)) {
diff --git a/db-tabulator/main.ts b/db-tabulator/main.ts
index 8084a48..6b9df66 100644
--- a/db-tabulator/main.ts
+++ b/db-tabulator/main.ts
@@ -17,6 +17,8 @@ import { ENWIKI_DB_HOST } from "../db";
  * Support linkification with ns numbers from another column
  *
  * Pending:
+ * Support pagination
+ * Create Module:Database report for syntax checking
  *
  */
 
diff --git a/reports/commons.js b/reports/commons.js
index c86e5e0..43b10f8 100644
--- a/reports/commons.js
+++ b/reports/commons.js
@@ -148,7 +148,7 @@ function preprocessDraftForExtract(text) {
 	let wkt = new bot.wikitext(text);
 	wkt.parseTemplates({
 		namePredicate: name => {
-			return /infobox/i.test(name) || name === 'AFC submission';
+			return /infobox/i.test(name) || name === 'AfC submission';
 		}
 	});
 	for (let template of wkt.templates) {