Skip to content

Commit

Permalink
eventstream-router: add more detail to stream error emails
Browse files Browse the repository at this point in the history
  • Loading branch information
siddharthvp committed Apr 25, 2024
1 parent 1f2977a commit e507831
Showing 1 changed file with 10 additions and 4 deletions.
14 changes: 10 additions & 4 deletions eventstream-router/check.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* stream process is still working, and restart
* it if it isn't.
*/
import {emailOnError} from '../botbase';
import {bot, emailOnError, fs} from '../botbase';
import {mapPath} from "../utils";
import {execSync} from 'child_process';
import {restartDeployment} from "../k8s";
Expand All @@ -21,16 +21,22 @@ for (const [job, dir] of Object.entries(streamJobs)) {
// eslint-disable-next-line no-empty
let match; for (match of tail.matchAll(testRgx)) {} // now match is the last matched

let date = new Date(match?.[1]);
let currentDate = new Date();
let date = new bot.Date(match?.[1]);
let currentDate = new bot.Date();

let diff = currentDate.getTime() - date.getTime();

let minutesDiff = diff / 1000 / 60;

if (!match || minutesDiff > 30) {
let err = new Error();
err.stack = `no recent entries. Restarting ${job}\n\nLast entry found: ${match?.[0]} `;
let lastSeenTime = new bot.Date(parseInt(fs.readFileSync('./last-seen.txt').toString()) * 1000);
err.stack = [
`no recent log entries. Restarting ${job}`,
`Last log entry: ${match?.[0]}`,
`Last event timestamp: ${lastSeenTime.format('YYYY-MM-DD HH:mm:ss')}`,
`Current time: ${currentDate.format('YYYY-MM-DD HH:mm:ss')}`
].join('\n\n');
emailOnError(err, job);

process.chdir(mapPath(dir));
Expand Down

0 comments on commit e507831

Please sign in to comment.