diff --git a/.dockerignore b/.dockerignore index 785e667e..a395aedf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,8 +19,5 @@ ley.config.mjs .gitattributes .gitignore -a.ci.sh -a.sh - LICENSE **/README.md diff --git a/.env.example b/.env.example deleted file mode 100644 index b3a2b341..00000000 --- a/.env.example +++ /dev/null @@ -1,6 +0,0 @@ -DISCORD_TOKEN= -REDIS_URL= -NODE_ENV=dev -DISCORD_PROXY_URL= -DATABASE_PORT=5432 -DATABASE_URL="postgresql://automoderator:admin@localhost:${DATABASE_PORT}/automoderator" diff --git a/.env.private.example b/.env.private.example new file mode 100644 index 00000000..41e85997 --- /dev/null +++ b/.env.private.example @@ -0,0 +1,5 @@ +P_PASSWORD="only used by parseable. dictates its password - and that'll dictate the token" + +DISCORD_TOKEN= +DISCORD_CLIENT_ID= +PARSEABLE_AUTH= diff --git a/.env.public b/.env.public new file mode 100644 index 00000000..65a74c58 --- /dev/null +++ b/.env.public @@ -0,0 +1,9 @@ +REDIS_URL=redis://redis:6379 +DISCORD_PROXY_URL=http://discord-proxy:9000 +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 +POSTGRES_USER=chatsift +POSTGRES_PASSWORD=admin +POSTGRES_DATABASE=chatsift +PARSEABLE_DOMAIN=http://parseable:8000 +PARSEABLE_STREAM=gateway diff --git a/.gitignore b/.gitignore index 65108146..1ccbf3da 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ !.vscode/settings.json node_modules -.env +.env.private coverage/* **/dist/* diff --git a/Dockerfile b/Dockerfile index b36fda3e..869b6f58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,11 @@ -# TODO: Split into diff images once https://github.com/vercel/turbo/issues/2791 is fixed +FROM node:20-alpine +LABEL name "chatsift-bots" -FROM node:18-alpine -LABEL name "automoderator" - -WORKDIR /usr/automoderator +WORKDIR /usr/ RUN apk add --update \ && apk add --no-cache ca-certificates \ -&& apk add --no-cache --virtual .build-deps curl git python3 alpine-sdk openssl1.1-compat +&& apk add --no-cache --virtual .build-deps curl git python3 alpine-sdk COPY turbo.json package.json tsconfig.json yarn.lock .yarnrc.yml ./ COPY .yarn ./.yarn @@ -16,9 +14,6 @@ COPY packages/core/package.json ./packages/core/package.json COPY services/discord-proxy/package.json ./services/discord-proxy/package.json COPY services/gateway/package.json ./services/gateway/package.json -COPY services/interactions/package.json ./services/interactions/package.json -COPY services/logging/package.json ./services/logging/package.json -COPY services/task-runner/package.json ./services/task-runner/package.json RUN yarn --immutable @@ -29,9 +24,6 @@ COPY packages/core ./packages/core COPY services/discord-proxy ./services/discord-proxy COPY services/gateway ./services/gateway -COPY services/interactions ./services/interactions -COPY services/logging ./services/logging -COPY services/task-runner ./services/task-runner ARG TURBO_TEAM ENV TURBO_TEAM=$TURBO_TEAM diff --git a/docker-compose.yml b/docker-compose.yml index 29a6aabc..c79bef56 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,19 +4,76 @@ services: postgres: image: postgres:12-alpine environment: - POSTGRES_USER: 'automoderator' + POSTGRES_USER: 'chatsift' POSTGRES_PASSWORD: 'admin' - POSTGRES_DB: 'automoderator' + POSTGRES_DB: 'chatsift' volumes: - postgres-data:/var/lib/postgresql/data restart: unless-stopped ports: - - 127.0.0.1:${DATABASE_PORT}:5432 + - 127.0.0.1:5432:5432 healthcheck: - test: ['CMD-SHELL', 'pg_isready -U automoderator'] + test: ['CMD-SHELL', 'pg_isready -U chatsift'] interval: 10s timeout: 5s + parseable: + image: parseable/parseable:v1.2.0 + restart: unless-stopped + expose: + - '8000' + ports: + - 127.0.0.1:8000:8000 + environment: + P_FS_DIR: '/parseable/data' + P_STAGING_DIR: '/parseable/staging' + P_USERNAME: 'admin' + env_file: + - ./.env.private + volumes: + - parseable-data:/parseable/data + - parseable-staging:/parseable/staging + command: ['parseable', 'local-store'] + + redis: + image: redis:6-alpine + # TODO: In the future, consider using a volume for persistence once we know we have no leaks + # volumes: + # - redis-data:/data + restart: unless-stopped + healthcheck: + test: ['CMD-SHELL', 'redis-cli ping'] + interval: 10s + timeout: 5s + + discord-proxy: + image: chatsift-bots:latest + build: + context: ./ + dockerfile: ./Dockerfile + restart: unless-stopped + env_file: + - ./.env.public + - ./.env.private + command: ['node', '--enable-source-maps', './services/discord-proxy/dist/index.js'] + + gateway: + image: chatsift-bots:latest + build: + context: ./ + dockerfile: ./Dockerfile + restart: unless-stopped + env_file: + - ./.env.public + - ./.env.private + command: ['node', '--enable-source-maps', './services/gateway/dist/index.js'] + volumes: postgres-data: - name: 'automoderator-v3-postgres-data' + name: 'chatsift-bots-postgres-data' + parseable-data: + name: 'chatsift-bots-parseable-data' + parseable-staging: + name: 'chatsift-bots-parseable-staging' + redis-data: + name: 'chatsift-bots-redis-data' diff --git a/packages/core/src/cache/CacheFactory.ts b/packages/core/src/cache/CacheFactory.ts index 2c0df019..768c771e 100644 --- a/packages/core/src/cache/CacheFactory.ts +++ b/packages/core/src/cache/CacheFactory.ts @@ -1,4 +1,4 @@ -import { inject } from 'inversify'; +import { inject, injectable } from 'inversify'; import { Redis } from 'ioredis'; import { INJECTION_TOKENS } from '../container.js'; import type { ICache } from './ICache.js'; @@ -9,6 +9,7 @@ import type { ICacheEntity } from './entities/ICacheEntity'; * @remarks * Since this is a singleton factroy, we "cache our caches" in a WeakMap to avoid additional computation on subsequent calls. */ +@injectable() export class CacheFactory { private readonly caches = new WeakMap, ICache>(); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 7a7c58af..9d3d6500 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -21,6 +21,7 @@ export * from './util/encode.js'; export * from './util/parseRelativeTime.js'; export * from './util/PermissionsBitField.js'; export * from './util/promiseAllObject.js'; +export * from './util/setupCrashLogs.js'; export * from './container.js'; export * from './db.js'; diff --git a/packages/core/src/singletons/DependencyManager.ts b/packages/core/src/singletons/DependencyManager.ts index 0e961c36..14ceacc5 100644 --- a/packages/core/src/singletons/DependencyManager.ts +++ b/packages/core/src/singletons/DependencyManager.ts @@ -9,6 +9,7 @@ import { GuildCacheEntity, type CachedGuild } from '../cache/entities/GuildCache import type { ICacheEntity } from '../cache/entities/ICacheEntity.js'; import { INJECTION_TOKENS, globalContainer } from '../container.js'; import type { DB } from '../db.js'; +import type { TransportOptions } from '../util/loggingTransport.js'; import { Env } from './Env.js'; // no proper ESM support @@ -29,7 +30,6 @@ const { */ export class DependencyManager { public constructor(private readonly env: Env) { - this.registerLogger(); this.registerStructures(); } @@ -66,13 +66,23 @@ export class DependencyManager { return database; } - private registerLogger(): void { - const transport = createPinoLogger.transport({ - target: '../util/loggingTransport.js', - }); + public registerLogger(stream: string): Logger { + const options: TransportOptions = { + domain: this.env.parseableDomain, + auth: this.env.parseableAuth, + stream, + }; - const logger = createPinoLogger({ level: 'trace', transport }); + const logger = createPinoLogger({ + level: 'trace', + transport: { + target: '../util/loggingTransport.js', + options, + }, + }); globalContainer.bind(INJECTION_TOKENS.logger).toConstantValue(logger); + + return logger; } private registerStructures(): void { diff --git a/packages/core/src/singletons/Env.ts b/packages/core/src/singletons/Env.ts index 11b80107..87391646 100644 --- a/packages/core/src/singletons/Env.ts +++ b/packages/core/src/singletons/Env.ts @@ -6,11 +6,11 @@ import { injectable } from 'inversify'; * The environment variables for the application, provided as a singleton. */ export class Env { - public readonly discordToken = process.env.DISCORD_TOKEN!; + public readonly discordToken: string = process.env.DISCORD_TOKEN!; - public readonly discordClientId = process.env.DISCORD_CLIENT_ID!; + public readonly discordClientId: string = process.env.DISCORD_CLIENT_ID!; - public readonly redisUrl = process.env.REDIS_URL!; + public readonly redisUrl: string = process.env.REDIS_URL!; public readonly nodeEnv: 'dev' | 'prod' = (process.env.NODE_ENV ?? 'prod') as 'dev' | 'prod'; @@ -18,7 +18,7 @@ export class Env { public readonly postgresHost: string = process.env.POSTGRES_HOST!; - public readonly postgresPort: number = Number(process.env.POSTGRES_PORT ?? '5432'); + public readonly postgresPort: number = Number(process.env.POSTGRES_PORT!); public readonly postgresUser: string = process.env.POSTGRES_USER!; @@ -26,22 +26,32 @@ export class Env { public readonly postgresDatabase: string = process.env.POSTGRES_DATABASE!; + public readonly parseableDomain: string = process.env.PARSEABLE_DOMAIN!; + + public readonly parseableAuth: string = process.env.PARSEABLE_AUTH!; + private readonly REQUIRED_KEYS = [ 'DISCORD_TOKEN', 'DISCORD_CLIENT_ID', + 'REDIS_URL', + 'DISCORD_PROXY_URL', + 'POSTGRES_HOST', + 'POSTGRES_PORT', 'POSTGRES_USER', 'POSTGRES_PASSWORD', 'POSTGRES_DATABASE', + + 'PARSEABLE_DOMAIN', + 'PARSEABLE_AUTH', ] as const; public constructor() { - for (const key of this.REQUIRED_KEYS) { - if (!(key in process.env)) { - throw new Error(`Missing environment variable ${key}`); - } + const missingKeys = this.REQUIRED_KEYS.filter((key) => !(key in process.env)); + if (missingKeys.length) { + throw new Error(`Missing environment variables: ${missingKeys.join(', ')}`); } } } diff --git a/packages/core/src/util/loggingTransport.ts b/packages/core/src/util/loggingTransport.ts index feb64b83..b17df33d 100644 --- a/packages/core/src/util/loggingTransport.ts +++ b/packages/core/src/util/loggingTransport.ts @@ -4,6 +4,7 @@ // Note that this file should never be imported/exported. Pino spawns it as a worker thread. import { URL } from 'node:url'; +import pino from 'pino'; import buildPinoTransport from 'pino-abstract-transport'; /** @@ -46,8 +47,20 @@ export default async function transport(options: TransportOptions) { }); } +function transformLogData(data: any) { + if ('level' in data && typeof data.level === 'number') { + data.level = pino.levels.labels[data.level]; + } + + if ('datetime' in data) { + delete data.datetime; + } + + return data; +} + async function handleLog(options: LogData) { - const body = JSON.stringify(options.data); + const body = JSON.stringify(transformLogData(options.data)); const res = await fetch(new URL('/api/v1/ingest', options.domain), { method: 'POST', @@ -117,7 +130,7 @@ async function ensureStream(options: TransportOptions): Promise { // Streams must not be empty before setting up retention await handleLog({ ...options, - data: { message: 'Log stream created', level: 'info', datetime: new Date().toISOString() }, + data: { msg: 'Log stream created', level: pino.levels.values.info, datetime: new Date().toISOString() }, }); const retentionResponse = await fetch(new URL(`/api/v1/logstream/${stream}/retention`, domain), { diff --git a/packages/core/src/util/setupCrashLogs.ts b/packages/core/src/util/setupCrashLogs.ts new file mode 100644 index 00000000..829fdc21 --- /dev/null +++ b/packages/core/src/util/setupCrashLogs.ts @@ -0,0 +1,11 @@ +import process from 'node:process'; +import type { Logger } from 'pino'; +import { INJECTION_TOKENS, globalContainer } from '../container.js'; + +export function setupCrashLogs() { + const logger = globalContainer.get(INJECTION_TOKENS.logger); + + process.on('uncaughtExceptionMonitor', (error, origin) => { + logger.fatal({ error, origin }, 'Uncaught exception. Likely a hard crash'); + }); +} diff --git a/services/discord-proxy/src/index.ts b/services/discord-proxy/src/index.ts index b0c2cc06..e76d691b 100644 --- a/services/discord-proxy/src/index.ts +++ b/services/discord-proxy/src/index.ts @@ -1,8 +1,13 @@ import 'reflect-metadata'; -import { DependencyManager, globalContainer } from '@automoderator/core'; +import { DependencyManager, globalContainer, setupCrashLogs } from '@automoderator/core'; import { ProxyServer } from './server.js'; -const _dependencyManager = globalContainer.get(DependencyManager); +const dependencyManager = globalContainer.get(DependencyManager); +const logger = dependencyManager.registerLogger('discordproxy'); +dependencyManager.registerRedis(); + +setupCrashLogs(); const server = globalContainer.get(ProxyServer); -server.listen(8_000); +server.listen(9_000); +logger.info('Listening on port 9000'); diff --git a/services/discord-proxy/src/server.ts b/services/discord-proxy/src/server.ts index d4a10eab..a45abba0 100644 --- a/services/discord-proxy/src/server.ts +++ b/services/discord-proxy/src/server.ts @@ -35,8 +35,8 @@ export class ProxyServer { if (method === RequestMethod.Get) { const cached = await this.cache.fetch(fullRoute); - this.logger.debug(cached, 'cache hit'); if (cached !== null) { + this.logger.debug(cached, 'cache hit'); res.statusCode = 200; res.setHeader('Content-Type', 'application/json'); return res.end(JSON.stringify(cached)); @@ -74,7 +74,7 @@ export class ProxyServer { } const data = await parseResponse(discordResponse); - this.logger.debug(data, 'response'); + this.logger.trace(data, 'response'); res.write(JSON.stringify(data)); await this.cache.update(fullRoute, data); diff --git a/services/gateway/src/index.ts b/services/gateway/src/index.ts index 5c9756f5..8db59093 100644 --- a/services/gateway/src/index.ts +++ b/services/gateway/src/index.ts @@ -1,10 +1,13 @@ import 'reflect-metadata'; -import { globalContainer, DependencyManager } from '@automoderator/core'; +import { globalContainer, DependencyManager, setupCrashLogs } from '@automoderator/core'; import { Gateway } from './gateway.js'; const dependencyManager = globalContainer.get(DependencyManager); +dependencyManager.registerLogger('gateway'); dependencyManager.registerRedis(); dependencyManager.registerApi(); +setupCrashLogs(); + const gateway = globalContainer.get(Gateway); await gateway.connect();