Run event publisher with Nodejs

Bun wasn't handling the disconnect event in the same way, which resulted in downtime.
Also set automatic restarts in docker compose.
Renamed event-subscriber to more accurate "api".
Added graceful shutdown to event-publisher for faster restarts.

Signed-off-by: Brian Evans <[email protected]>

Caddyfile

@@ -17,11 +17,11 @@
         }
 
         handle @websockets {
-                reverse_proxy event-subscriber:3000
+                reverse_proxy api:3000
         }
 
         # for health checks
         handle_path /events/* {
-                reverse_proxy event-subscriber:3000
+                reverse_proxy api:3000
         }
 }

docker-compose.yml

@@ -8,11 +8,6 @@ services:
     logging:
       driver: local
 
-  client: # builds the frontend into a volume to be served by caddy
-    build: client-pure
-    volumes:
-      - client_dist:/client/dist # this MUST match the workdir set in the Dockerfile
-
   webserver:
     image: caddy
     restart: unless-stopped
@@ -30,24 +25,28 @@ services:
       SITE_ADDRESS: ${SITE_ADDRESS}
 
   event-publisher:
-    build: server
-    command: bun run src/redis/streamToRedis.ts
+    build:
+      context: server
+      dockerfile: Publisher.Dockerfile
     environment:
       PUBSUB_REDIS_IP: redis
     env_file:
       - .api.env
     logging:
       driver: local
     depends_on: [ redis ]
+    restart: on-failure
 
-  event-subscriber:
-    build: server
-    command: bun run src/redis/streamFromRedis.ts
+  api:
+    build:
+      context: server
+      dockerfile: Api.Dockerfile
     environment:
       PUBSUB_REDIS_IP: redis
     logging:
       driver: local
     depends_on: [ redis ]
+    restart: on-failure
 
 
 volumes:

server/Dockerfile → server/Api.Dockerfile

@@ -9,4 +9,4 @@ COPY . .
 
 EXPOSE 3000
 
-CMD ["bun", "start"]
+CMD ["bun", "--smol", "run", "src/redis/streamFromRedis.ts"]

server/Publisher.Dockerfile

@@ -0,0 +1,22 @@
+FROM oven/bun as builder
+
+WORKDIR /companies-stream/server
+
+COPY package.json bun.lockb ./
+RUN bun install --frozen-lockfile
+
+COPY . .
+
+RUN bun build src/redis/streamToRedis.ts --outdir dist --target=node
+# hack to fix Bun bundler https://github.com/oven-sh/bun/issues/6168
+RUN echo 'import { createRequire as createImportMetaRequire } from "module"; import.meta.require ||= (id) => createImportMetaRequire(import.meta.url)(id);' | cat - dist/streamToRedis.js > temp && mv temp dist/streamToRedis.js
+
+
+FROM node:lts
+
+WORKDIR /companies-stream/server
+
+COPY --from=builder /companies-stream/server/dist /companies-stream/server/dist
+COPY --from=builder /companies-stream/server/package.json /companies-stream/server/package.json
+
+CMD ["node", "dist/streamToRedis.js"]

server/src/redis/streamToRedis.ts

@@ -4,6 +4,7 @@ import { restKeyHolder, streamKeyHolder } from "../utils/KeyHolder.js"
 import { setTimeout } from "node:timers/promises"
 import pino from "pino"
 import { streamPaths } from "../streams/streamPaths.js"
+import { Transform } from "stream"
 /*
 
   This file listens to the Companies House long polling streaming API, and when events are received, they are posted
@@ -43,7 +44,29 @@ const startStream = streamPath => getMostRecentTimepoint(streamPath)
       .then(() => logger.info({ streamPath }, "Restarting stream, after waiting 60 seconds since disconnected."))
       .then(() => startStream(streamPath))))// restart on end
 
+const streams = new Set<Transform>()
 for (const streamPath of streamPaths) {
-  await startStream(streamPath)
+  streams.add(await startStream(streamPath))
   await setTimeout(5000) // space them out 5 seconds
 }
+
+
+async function shutdown() {
+  const requestTime = performance.now()
+  try {
+    logger.flush()
+    console.log("Graceful shutdown commenced", new Date())
+    for (const stream of streams) {
+      stream.destroy()
+    }
+    await client.quit()
+    logger.flush()
+  } finally {
+    const waitingNs = performance.now() - requestTime
+    console.log("Graceful shutdown finished", new Date(), "in", waitingNs / 1000 / 1000, "ms")
+    process.exit()
+  }
+}
+
+process.on("SIGINT", shutdown) // quit on ctrl-c when running docker in terminal
+process.on("SIGTERM", shutdown)// quit properly on docker stop

server/tsconfig.json

@@ -9,8 +9,9 @@
     "module": "esnext",
     "target": "esnext",
     "moduleResolution": "bundler",
-    "noEmit": true,
-    "allowImportingTsExtensions": true,
+    "outDir": "dist",
+    "noEmit": false,
+    "allowImportingTsExtensions": false,
     "moduleDetection": "force",
     "allowJs": true,
     "esModuleInterop": true,
@@ -22,5 +23,8 @@
     "composite": true,
     "downlevelIteration": true,
     "allowSyntheticDefaultImports": true
-  }
+  },
+  "include": [
+    "src"
+  ]
 }