diff --git a/db-tabulator/web-endpoint.ts b/db-tabulator/web-endpoint.ts index 4eca862..001c430 100644 --- a/db-tabulator/web-endpoint.ts +++ b/db-tabulator/web-endpoint.ts @@ -1,37 +1,73 @@ import * as express from "express"; import { checkShutoff, fetchQueriesForPage, processQueriesForPage, SHUTOFF_PAGE, TEMPLATE } from "./app"; import { createLogStream, mapPath } from "../utils"; +import {bot} from "../botbase"; +import {getRedisInstance} from "../redis"; const router = express.Router(); const log = createLogStream(mapPath('~/web-dbtb.out')); +const redis = getRedisInstance(); + +/** Store the list of pages currently undergoing update as a redis set */ +const redisKey = 'web-db-tabulator-pages'; router.get('/', async function (req, res, next) { let {page} = req.query as {page: string}; - const [shutoffText, queries] = await Promise.all([ + const [shutoffText, queries, revId] = await Promise.all([ checkShutoff(), - fetchQueriesForPage(page) + fetchQueriesForPage(page), + getLatestRevId(page), ]); + if (revId === -1) { + return res.status(404).render('oneline', { text: `The page ${page} does not exist.` }); + } + if (shutoffText) { log(`[E] Refused run on ${page} as task is shut off. Shutoff page content: ${shutoffText}`); - res.render('oneline', { + return res.status(422).render('oneline', { text: `Bot is current shut off via ${SHUTOFF_PAGE}. The page should be blank for it to work.` }); - return; } - res.render('database-report', { + const pgKey = page + ':' + revId; + if (await redis.sismember(redisKey, pgKey).catch(handleRedisError)) { + return res.status(409).render('oneline', { + text: `An update is already in progress for report(s) on page ${page} (revid ${revId})` + }); + } + redis.sadd(redisKey, pgKey).catch(handleRedisError); + + res.status(202).render('database-report', { page, template: TEMPLATE, noQueries: queries.length === 0 }); if (queries) { log(`Started processing ${page}`); - await processQueriesForPage(queries); + try { // should never throw but still ... + await processQueriesForPage(queries); + } finally { + redis.srem(redisKey, pgKey).catch(handleRedisError); + } log(`Finished processing ${page}`); } }); +async function getLatestRevId(page: string) { + let response = await bot.query({ prop: 'revisions', titles: page, rvprop: 'ids', rvlimit: 1 }); + let pg = response.query.pages[0]; + if (pg.missing) { + return -1; + } + return pg.revisions[0].revid; +} + +function handleRedisError(e: Error) { + log(`[E] Error in redis operation: `, e); + return false; // in sismember check, act as if value is not present +} + export default router; diff --git a/redis.ts b/redis.ts index a47a6bf..eb6f742 100644 --- a/redis.ts +++ b/redis.ts @@ -36,6 +36,8 @@ export function getRedisConfig(config: redis.ClientOpts = {}): redis.ClientOpts return { host: onToolforge() ? REDIS_HOST : '127.0.0.1', port: onToolforge() ? 6379 : 4713, + enable_offline_queue: false, + connect_timeout: 500, // Prefixing per https://wikitech.wikimedia.org/wiki/Help:Toolforge/Redis_for_Toolforge#Security // A secret prefix string is stored in redis-key-prefix.txt prefix: readFile(__dirname + '/redis-key-prefix.txt'),