feat: complete stack with logging

.dockerignore

@@ -19,8 +19,5 @@ ley.config.mjs
 .gitattributes
 .gitignore
 
-a.ci.sh
-a.sh
-
 LICENSE
 **/README.md

.env.private.example

@@ -0,0 +1,5 @@
+P_PASSWORD="only used by parseable. dictates its password - and that'll dictate the token"
+
+DISCORD_TOKEN=
+DISCORD_CLIENT_ID=
+PARSEABLE_AUTH=

.env.public

@@ -0,0 +1,9 @@
+REDIS_URL=redis://redis:6379
+DISCORD_PROXY_URL=http://discord-proxy:9000
+POSTGRES_HOST=postgres
+POSTGRES_PORT=5432
+POSTGRES_USER=chatsift
+POSTGRES_PASSWORD=admin
+POSTGRES_DATABASE=chatsift
+PARSEABLE_DOMAIN=http://parseable:8000
+PARSEABLE_STREAM=gateway

.gitignore

@@ -2,7 +2,7 @@
 !.vscode/settings.json
 
 node_modules
-.env
+.env.private
 
 coverage/*
 **/dist/*

Dockerfile

@@ -1,13 +1,11 @@
-# TODO: Split into diff images once https://github.com/vercel/turbo/issues/2791 is fixed
+FROM node:20-alpine
+LABEL name "chatsift-bots"
 
-FROM node:18-alpine
-LABEL name "automoderator"
-
-WORKDIR /usr/automoderator
+WORKDIR /usr/
 
 RUN apk add --update \
 && apk add --no-cache ca-certificates \
-&& apk add --no-cache --virtual .build-deps curl git python3 alpine-sdk openssl1.1-compat
+&& apk add --no-cache --virtual .build-deps curl git python3 alpine-sdk
 
 COPY turbo.json package.json tsconfig.json yarn.lock .yarnrc.yml ./
 COPY .yarn ./.yarn
@@ -16,9 +14,6 @@ COPY packages/core/package.json ./packages/core/package.json
 
 COPY services/discord-proxy/package.json ./services/discord-proxy/package.json
 COPY services/gateway/package.json ./services/gateway/package.json
-COPY services/interactions/package.json ./services/interactions/package.json
-COPY services/logging/package.json ./services/logging/package.json
-COPY services/task-runner/package.json ./services/task-runner/package.json
 
 RUN yarn --immutable
 
@@ -29,9 +24,6 @@ COPY packages/core ./packages/core
 
 COPY services/discord-proxy ./services/discord-proxy
 COPY services/gateway ./services/gateway
-COPY services/interactions ./services/interactions
-COPY services/logging ./services/logging
-COPY services/task-runner ./services/task-runner
 
 ARG TURBO_TEAM
 ENV TURBO_TEAM=$TURBO_TEAM

docker-compose.yml

@@ -4,19 +4,76 @@ services:
   postgres:
     image: postgres:12-alpine
     environment:
-      POSTGRES_USER: 'automoderator'
+      POSTGRES_USER: 'chatsift'
       POSTGRES_PASSWORD: 'admin'
-      POSTGRES_DB: 'automoderator'
+      POSTGRES_DB: 'chatsift'
     volumes:
       - postgres-data:/var/lib/postgresql/data
     restart: unless-stopped
     ports:
-      - 127.0.0.1:${DATABASE_PORT}:5432
+      - 127.0.0.1:5432:5432
     healthcheck:
-      test: ['CMD-SHELL', 'pg_isready -U automoderator']
+      test: ['CMD-SHELL', 'pg_isready -U chatsift']
       interval: 10s
       timeout: 5s
 
+  parseable:
+    image: parseable/parseable:v1.2.0
+    restart: unless-stopped
+    expose:
+      - '8000'
+    ports:
+      - 127.0.0.1:8000:8000
+    environment:
+      P_FS_DIR: '/parseable/data'
+      P_STAGING_DIR: '/parseable/staging'
+      P_USERNAME: 'admin'
+    env_file:
+      - ./.env.private
+    volumes:
+      - parseable-data:/parseable/data
+      - parseable-staging:/parseable/staging
+    command: ['parseable', 'local-store']
+
+  redis:
+    image: redis:6-alpine
+    # TODO: In the future, consider using a volume for persistence once we know we have no leaks
+    # volumes:
+      # - redis-data:/data
+    restart: unless-stopped
+    healthcheck:
+      test: ['CMD-SHELL', 'redis-cli ping']
+      interval: 10s
+      timeout: 5s
+
+  discord-proxy:
+    image: chatsift-bots:latest
+    build:
+      context: ./
+      dockerfile: ./Dockerfile
+    restart: unless-stopped
+    env_file:
+      - ./.env.public
+      - ./.env.private
+    command: ['node', '--enable-source-maps', './services/discord-proxy/dist/index.js']
+
+  gateway:
+    image: chatsift-bots:latest
+    build:
+      context: ./
+      dockerfile: ./Dockerfile
+    restart: unless-stopped
+    env_file:
+      - ./.env.public
+      - ./.env.private
+    command: ['node', '--enable-source-maps', './services/gateway/dist/index.js']
+
 volumes:
   postgres-data:
-    name: 'automoderator-v3-postgres-data'
+    name: 'chatsift-bots-postgres-data'
+  parseable-data:
+    name: 'chatsift-bots-parseable-data'
+  parseable-staging:
+    name: 'chatsift-bots-parseable-staging'
+  redis-data:
+    name: 'chatsift-bots-redis-data'

packages/core/src/cache/CacheFactory.ts

@@ -1,4 +1,4 @@
-import { inject } from 'inversify';
+import { inject, injectable } from 'inversify';
 import { Redis } from 'ioredis';
 import { INJECTION_TOKENS } from '../container.js';
 import type { ICache } from './ICache.js';
@@ -9,6 +9,7 @@ import type { ICacheEntity } from './entities/ICacheEntity';
  * @remarks
  * Since this is a singleton factroy, we "cache our caches" in a WeakMap to avoid additional computation on subsequent calls.
  */
+@injectable()
 export class CacheFactory {
 	private readonly caches = new WeakMap<ICacheEntity<unknown>, ICache<unknown>>();
 

packages/core/src/index.ts

@@ -21,6 +21,7 @@ export * from './util/encode.js';
 export * from './util/parseRelativeTime.js';
 export * from './util/PermissionsBitField.js';
 export * from './util/promiseAllObject.js';
+export * from './util/setupCrashLogs.js';
 
 export * from './container.js';
 export * from './db.js';

packages/core/src/singletons/DependencyManager.ts

@@ -9,6 +9,7 @@ import { GuildCacheEntity, type CachedGuild } from '../cache/entities/GuildCache
 import type { ICacheEntity } from '../cache/entities/ICacheEntity.js';
 import { INJECTION_TOKENS, globalContainer } from '../container.js';
 import type { DB } from '../db.js';
+import type { TransportOptions } from '../util/loggingTransport.js';
 import { Env } from './Env.js';
 
 // no proper ESM support
@@ -29,7 +30,6 @@ const {
  */
 export class DependencyManager {
 	public constructor(private readonly env: Env) {
-		this.registerLogger();
 		this.registerStructures();
 	}
 
@@ -66,13 +66,23 @@ export class DependencyManager {
 		return database;
 	}
 
-	private registerLogger(): void {
-		const transport = createPinoLogger.transport({
-			target: '../util/loggingTransport.js',
-		});
+	public registerLogger(stream: string): Logger {
+		const options: TransportOptions = {
+			domain: this.env.parseableDomain,
+			auth: this.env.parseableAuth,
+			stream,
+		};
 
-		const logger = createPinoLogger({ level: 'trace', transport });
+		const logger = createPinoLogger({
+			level: 'trace',
+			transport: {
+				target: '../util/loggingTransport.js',
+				options,
+			},
+		});
 		globalContainer.bind<Logger>(INJECTION_TOKENS.logger).toConstantValue(logger);
+
+		return logger;
 	}
 
 	private registerStructures(): void {

packages/core/src/singletons/Env.ts

@@ -6,42 +6,52 @@ import { injectable } from 'inversify';
  * The environment variables for the application, provided as a singleton.
  */
 export class Env {
-	public readonly discordToken = process.env.DISCORD_TOKEN!;
+	public readonly discordToken: string = process.env.DISCORD_TOKEN!;
 
-	public readonly discordClientId = process.env.DISCORD_CLIENT_ID!;
+	public readonly discordClientId: string = process.env.DISCORD_CLIENT_ID!;
 
-	public readonly redisUrl = process.env.REDIS_URL!;
+	public readonly redisUrl: string = process.env.REDIS_URL!;
 
 	public readonly nodeEnv: 'dev' | 'prod' = (process.env.NODE_ENV ?? 'prod') as 'dev' | 'prod';
 
 	public readonly discordProxyURL = process.env.DISCORD_PROXY_URL!;
 
 	public readonly postgresHost: string = process.env.POSTGRES_HOST!;
 
-	public readonly postgresPort: number = Number(process.env.POSTGRES_PORT ?? '5432');
+	public readonly postgresPort: number = Number(process.env.POSTGRES_PORT!);
 
 	public readonly postgresUser: string = process.env.POSTGRES_USER!;
 
 	public readonly postgresPassword: string = process.env.POSTGRES_PASSWORD!;
 
 	public readonly postgresDatabase: string = process.env.POSTGRES_DATABASE!;
 
+	public readonly parseableDomain: string = process.env.PARSEABLE_DOMAIN!;
+
+	public readonly parseableAuth: string = process.env.PARSEABLE_AUTH!;
+
 	private readonly REQUIRED_KEYS = [
 		'DISCORD_TOKEN',
 		'DISCORD_CLIENT_ID',
+
 		'REDIS_URL',
+
 		'DISCORD_PROXY_URL',
+
 		'POSTGRES_HOST',
+		'POSTGRES_PORT',
 		'POSTGRES_USER',
 		'POSTGRES_PASSWORD',
 		'POSTGRES_DATABASE',
+
+		'PARSEABLE_DOMAIN',
+		'PARSEABLE_AUTH',
 	] as const;
 
 	public constructor() {
-		for (const key of this.REQUIRED_KEYS) {
-			if (!(key in process.env)) {
-				throw new Error(`Missing environment variable ${key}`);
-			}
+		const missingKeys = this.REQUIRED_KEYS.filter((key) => !(key in process.env));
+		if (missingKeys.length) {
+			throw new Error(`Missing environment variables: ${missingKeys.join(', ')}`);
 		}
 	}
 }

packages/core/src/util/loggingTransport.ts

@@ -4,6 +4,7 @@
 // Note that this file should never be imported/exported. Pino spawns it as a worker thread.
 
 import { URL } from 'node:url';
+import pino from 'pino';
 import buildPinoTransport from 'pino-abstract-transport';
 
 /**
@@ -46,8 +47,20 @@ export default async function transport(options: TransportOptions) {
 	});
 }
 
+function transformLogData(data: any) {
+	if ('level' in data && typeof data.level === 'number') {
+		data.level = pino.levels.labels[data.level];
+	}
+
+	if ('datetime' in data) {
+		delete data.datetime;
+	}
+
+	return data;
+}
+
 async function handleLog(options: LogData) {
-	const body = JSON.stringify(options.data);
+	const body = JSON.stringify(transformLogData(options.data));
 
 	const res = await fetch(new URL('/api/v1/ingest', options.domain), {
 		method: 'POST',
@@ -117,7 +130,7 @@ async function ensureStream(options: TransportOptions): Promise<void> {
 		// Streams must not be empty before setting up retention
 		await handleLog({
 			...options,
-			data: { message: 'Log stream created', level: 'info', datetime: new Date().toISOString() },
+			data: { msg: 'Log stream created', level: pino.levels.values.info, datetime: new Date().toISOString() },
 		});
 
 		const retentionResponse = await fetch(new URL(`/api/v1/logstream/${stream}/retention`, domain), {

packages/core/src/util/setupCrashLogs.ts

@@ -0,0 +1,11 @@
+import process from 'node:process';
+import type { Logger } from 'pino';
+import { INJECTION_TOKENS, globalContainer } from '../container.js';
+
+export function setupCrashLogs() {
+	const logger = globalContainer.get<Logger>(INJECTION_TOKENS.logger);
+
+	process.on('uncaughtExceptionMonitor', (error, origin) => {
+		logger.fatal({ error, origin }, 'Uncaught exception. Likely a hard crash');
+	});
+}

services/discord-proxy/src/index.ts

@@ -1,8 +1,13 @@
 import 'reflect-metadata';
-import { DependencyManager, globalContainer } from '@automoderator/core';
+import { DependencyManager, globalContainer, setupCrashLogs } from '@automoderator/core';
 import { ProxyServer } from './server.js';
 
-const _dependencyManager = globalContainer.get(DependencyManager);
+const dependencyManager = globalContainer.get(DependencyManager);
+const logger = dependencyManager.registerLogger('discordproxy');
+dependencyManager.registerRedis();
+
+setupCrashLogs();
 
 const server = globalContainer.get(ProxyServer);
-server.listen(8_000);
+server.listen(9_000);
+logger.info('Listening on port 9000');

services/discord-proxy/src/server.ts

@@ -35,8 +35,8 @@ export class ProxyServer {
 
 		if (method === RequestMethod.Get) {
 			const cached = await this.cache.fetch(fullRoute);
-			this.logger.debug(cached, 'cache hit');
 			if (cached !== null) {
+				this.logger.debug(cached, 'cache hit');
 				res.statusCode = 200;
 				res.setHeader('Content-Type', 'application/json');
 				return res.end(JSON.stringify(cached));
@@ -74,7 +74,7 @@ export class ProxyServer {
 			}
 
 			const data = await parseResponse(discordResponse);
-			this.logger.debug(data, 'response');
+			this.logger.trace(data, 'response');
 			res.write(JSON.stringify(data));
 
 			await this.cache.update(fullRoute, data);

services/gateway/src/index.ts

@@ -1,10 +1,13 @@
 import 'reflect-metadata';
-import { globalContainer, DependencyManager } from '@automoderator/core';
+import { globalContainer, DependencyManager, setupCrashLogs } from '@automoderator/core';
 import { Gateway } from './gateway.js';
 
 const dependencyManager = globalContainer.get(DependencyManager);
+dependencyManager.registerLogger('gateway');
 dependencyManager.registerRedis();
 dependencyManager.registerApi();
 
+setupCrashLogs();
+
 const gateway = globalContainer.get(Gateway);
 await gateway.connect();