dyk-counts: namespace; in-mem caching for dupe event detection

eventstream-router/routes/dyk-counts.ts → dyk-counts/eventstream-trigger.ts

@@ -1,9 +1,11 @@
-import { bot } from "../../botbase";
-import { Route } from "../app";
-import { createLocalSSHTunnel } from "../../utils";
-import { ENWIKI_DB_HOST, enwikidb } from "../../db";
-import {Redis, getRedisInstance} from "../../redis";
-import {RecentChangeStreamEvent} from "../RecentChangeStreamEvent";
+import { bot } from "../botbase";
+import { Route } from "../eventstream-router/app";
+import { createLocalSSHTunnel } from "../utils";
+import { ENWIKI_DB_HOST, enwikidb } from "../db";
+import {Redis, getRedisInstance} from "../redis";
+import {RecentChangeStreamEvent} from "../eventstream-router/RecentChangeStreamEvent";
+import {Cache, CacheClass} from "memory-cache";
+import {ReplyError} from 'redis';
 
 export default class DykCountsTask extends Route {
     name = 'dyk-counts';
@@ -13,6 +15,7 @@ export default class DykCountsTask extends Route {
 
 	counts: Record<string, number> = {};
 	unflushedChanges: Record<string, string[]> = {};
+	dupeCache: CacheClass<string, boolean> = new Cache();
 	lastFlushTime: number = 0;
 	isFlushScheduled = false;
 
@@ -97,10 +100,18 @@ export default class DykCountsTask extends Route {
 
 	async worker(data: RecentChangeStreamEvent) {
 		let {user, title} = data;
+
+		// Check that we are not getting the same event a second time, which happens occasionally
+		if (this.dupeCache.get(title)) {
+			this.log(`[E] Ignoring [[${title}]] present in dupe cache`);
+			return;
+		}
+		this.dupeCache.put(title, true, 300); // 5 min timeout
+
 		this.counts[user] = (this.counts[user] || 0) + 1;
 		this.redis.hincrby('dyk-counts', user, 1).catch(e => this.redisError(e));
 		this.unflushedChanges[user] = (this.unflushedChanges[user] || []).concat(title);
-		this.log(`[i] Crediting "${user}" for [[${data.title}]]`);
+		this.log(`[i] Crediting "${user}" for [[${title}]]`);
 
 		if (Date.now() - this.lastFlushTime > this.minFlushInterval) {
 			this.flushCounts();
@@ -113,7 +124,10 @@ export default class DykCountsTask extends Route {
 		}
 	}
 
-	redisError(err: Error) {
+	redisError(err: ReplyError) {
+		if (err.command === 'HMSET') {
+			err.args = [err.args[0], '<snipped>']; // too big to log!
+		}
 		this.log(`[E] Redis error: `, err)
 	}
 }

webservice/routes/dyk.ts → dyk-counts/web-endpoint.ts

@@ -1,6 +1,6 @@
 import * as express from "express";
-import { enwikidb } from "../../../SDZeroBot/db";
-import { getRedisInstance } from "../../../SDZeroBot/redis";
+import { enwikidb } from "../db";
+import { getRedisInstance } from "../redis";
 
 const router = express.Router();
 const db = new enwikidb();
@@ -25,21 +25,7 @@ router.get('/noms/:user', async (req, res, next) => {
 	const user = req.params.user.replace(/_/g, ' ');
 
 	let count = await redis.hget('dyk-counts', user) as unknown as string;
-	if (!count) {
-		const result = await db.query(`
-			SELECT COUNT(*) AS count FROM revision_userindex
-			JOIN page ON rev_page = page_id
-			JOIN actor_revision ON rev_actor = actor_id
-			WHERE page_namespace = 10
-			AND page_is_redirect = 0
-			AND rev_parent_id = 0
-			AND page_title LIKE 'Did_you_know_nominations/%'
-			AND actor_name = ?
-		`, [user]);
-		count = String(result[0].count);
-		redis.hset('dyk-counts', user, count);
-	}
-	res.end(count);
+	res.end(count || '0');
 });
 
 export default router;

eventstream-router/main.ts

@@ -12,7 +12,7 @@ import botActivityMonitor from "../bot-monitor/eventstream-trigger";
 import dbTabulator from "../db-tabulator/eventstream-trigger";
 import dbTabulatorMetadata from "../db-tabulator/eventstream-metadata-maintainer";
 import shutoffsMonitor from "./routes/shutoffs-monitor";
-import dykCountsTask from "./routes/dyk-counts";
+import dykCountsTask from "../dyk-counts/eventstream-trigger";
 import purger from "./routes/purger"
 
 const routeClasses = [

package.json

@@ -17,6 +17,7 @@
     "@kubernetes/client-node": "^0.18.1",
     "@types/async-redis": "^1.1.1",
     "@types/express": "^4.17.11",
+    "@types/memory-cache": "^0.2.5",
     "@types/node": "^14.11.10",
     "@types/nodemailer": "^6.4.7",
     "@types/object-hash": "^1.3.4",
@@ -25,6 +26,7 @@
     "eventsource": "^1.0.7",
     "express": "^4.17.1",
     "jsdom": "^16.2.1",
+    "memory-cache": "^0.2.0",
     "minimist": "^1.2.5",
     "moment": "^2.29.1",
     "mwn": "0.9.0",

webservice/route-registry.ts

@@ -6,7 +6,7 @@ import summaryRouter from "./routes/summary";
 import dbReportRouter from '../../SDZeroBot/db-tabulator/web-endpoint';
 import gansRouter from '../../SDZeroBot/most-gans/web-endpoint';
 import articleSearchRouter from './routes/articlesearch';
-import dykRouter from './routes/dyk';
+import dykRouter from '../../SDZeroBot/dyk-counts/web-endpoint';
 import gitsync from "./routes/gitsync";
 
 export function registerRoutes(app: express.Router) {