db-tabulator: handle double submits to webservice

db-tabulator/web-endpoint.ts

@@ -1,37 +1,73 @@
 import * as express from "express";
 import { checkShutoff, fetchQueriesForPage, processQueriesForPage, SHUTOFF_PAGE, TEMPLATE } from "./app";
 import { createLogStream, mapPath } from "../utils";
+import {bot} from "../botbase";
+import {getRedisInstance} from "../redis";
 
 const router = express.Router();
 
 const log = createLogStream(mapPath('~/web-dbtb.out'));
+const redis = getRedisInstance();
+
+/** Store the list of pages currently undergoing update as a redis set */
+const redisKey = 'web-db-tabulator-pages';
 
 router.get('/', async function (req, res, next) {
 	let {page} = req.query as {page: string};
 
-	const [shutoffText, queries] = await Promise.all([
+	const [shutoffText, queries, revId] = await Promise.all([
 		checkShutoff(),
-		fetchQueriesForPage(page)
+		fetchQueriesForPage(page),
+		getLatestRevId(page),
 	]);
 
+	if (revId === -1) {
+		return res.status(404).render('oneline', { text: `The page ${page} does not exist.` });
+	}
+
 	if (shutoffText) {
 		log(`[E] Refused run on ${page} as task is shut off. Shutoff page content: ${shutoffText}`);
-		res.render('oneline', {
+		return res.status(422).render('oneline', {
 			text: `Bot is current shut off via ${SHUTOFF_PAGE}. The page should be blank for it to work.`
 		});
-		return;
 	}
 
-	res.render('database-report', {
+	const pgKey = page + ':' + revId;
+	if (await redis.sismember(redisKey, pgKey).catch(handleRedisError)) {
+		return res.status(409).render('oneline', {
+			text: `An update is already in progress for report(s) on page ${page} (revid ${revId})`
+		});
+	}
+	redis.sadd(redisKey, pgKey).catch(handleRedisError);
+
+	res.status(202).render('database-report', {
 		page,
 		template: TEMPLATE,
 		noQueries: queries.length === 0
 	});
 	if (queries) {
 		log(`Started processing ${page}`);
-		await processQueriesForPage(queries);
+		try { // should never throw but still ...
+			await processQueriesForPage(queries);
+		} finally {
+			redis.srem(redisKey, pgKey).catch(handleRedisError);
+		}
 		log(`Finished processing ${page}`);
 	}
 });
 
+async function getLatestRevId(page: string) {
+	let response = await bot.query({ prop: 'revisions', titles: page, rvprop: 'ids', rvlimit: 1 });
+	let pg = response.query.pages[0];
+	if (pg.missing) {
+		return -1;
+	}
+	return pg.revisions[0].revid;
+}
+
+function handleRedisError(e: Error) {
+	log(`[E] Error in redis operation: `, e);
+	return false; // in sismember check, act as if value is not present
+}
+
 export default router;

redis.ts

@@ -36,6 +36,8 @@ export function getRedisConfig(config: redis.ClientOpts = {}): redis.ClientOpts
 	return {
 		host: onToolforge() ? REDIS_HOST : '127.0.0.1',
 		port: onToolforge() ? 6379 : 4713,
+		enable_offline_queue: false,
+		connect_timeout: 500,
 		// Prefixing per https://wikitech.wikimedia.org/wiki/Help:Toolforge/Redis_for_Toolforge#Security
 		// A secret prefix string is stored in redis-key-prefix.txt
 		prefix: readFile(__dirname + '/redis-key-prefix.txt'),