Monitor external dependency health (#1891)

* all user journeys working

* ...

* change scenarios from objects to functions

* ...

* add prefix to logged metric

* start on ext deps

* both monitors running with pm2

* elasticache check

* extra line

* move directories all tests, monitors, etc. working

* look at localhost when host isnt set

* docs

* correct v3 url

* correct docs

* use underscore for metrics

* update readme

README.md

@@ -138,6 +138,30 @@ For details on environment configuration, including optional variables, see
 
 ## Running the Server
 
+You will need to have Redis running in cluster mode.
+The easiest way to get it running is to download and compile it.
+But, we plan to Dockerize all of our dependencies in the near future.
+Until then:
+
+```
+cd $HOME
+wget https://github.com/redis/redis/archive/7.2.4.tar.gz
+tar xvf 7.2.4.tar.gz
+cd redis-7.2.4
+make
+./utils/create-cluster/create-cluster start
+./utils/create-cluster/create-cluster create -f
+```
+
+When you're done with it:
+
+```
+./utils/create-cluster/create-cluster stop
+cd $HOME
+rm 7.2.tar.gz
+rm -rf redis-7.2.4
+```
+
 Start the server with `mix phx.server`
 
 Then, visit the site at http://localhost:4001/
@@ -146,6 +170,27 @@ Then, visit the site at http://localhost:4001/
 
 [Algolia](https://www.algolia.com) powers our search features. Sometimes after content updates or GTFS releases we will find that the search results do not contain up-to-date results. When this happens you can re-index the Algolia data by running: `mix algolia.update`.
 
+## Integration Tests
+
+```
+npm install --ignore-scripts
+npx playwright test all-scenarios
+```
+
+## Load Tests
+
+```
+npm install --ignore-scripts
+npx artillery run ./integration/load_tests/all-scenarios.yml --target http://localhost:4001
+```
+
+## Monitoring
+
+```
+npm install --ignore-scripts
+npx pm2-runtime ./integration/monitor/ecosystem.config.js
+```
+
 ## Commiting Code
 
 When commiting code a bunch of checks are run using [git pre-commit hook](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks)

deploy/monitor/Dockerfile

@@ -3,11 +3,13 @@ FROM node:21-bookworm
 WORKDIR /home/runner
 
 COPY package*.json .
-COPY monitor /home/runner/monitor
-COPY scenarios /home/runner/scenarios
 
 RUN npm ci --ignore-scripts
 RUN npx playwright install chromium
 RUN npx playwright install-deps
 
-CMD ["node", "monitor/all-scenarios.js"]
+RUN ./node_modules/pm2/bin/pm2 install pm2-logrotate
+
+COPY integration /home/runner/integration
+
+CMD ["./node_modules/pm2/bin/pm2-runtime", "./integration/monitor/ecosystem.config.js"]

integration/e2e_tests/all-scenarios.spec.js

@@ -0,0 +1,36 @@
+const fs = require("fs");
+const path = require("path");
+const { performance } = require("node:perf_hooks");
+const { test } = require("@playwright/test");
+
+const { fileToMetricName } = require("../utils");
+
+const filesPath = path.join(__dirname, "..", "scenarios");
+const files = fs.readdirSync(filesPath);
+
+const baseURL = process.env.HOST
+  ? `https://${process.env.HOST}`
+  : "http://localhost:4001";
+
+files.forEach((file) => {
+  test.describe("All scenarios", (_) => {
+    const filePath = path.join(filesPath, file);
+    const { scenario } = require(filePath);
+
+    const name = fileToMetricName(file);
+
+    test(name, async ({ page }) => {
+      const start = performance.now();
+
+      await scenario({ page, baseURL });
+
+      const end = performance.now();
+      const duration = Math.floor(end - start);
+
+      test.info().annotations.push({
+        type: "performance",
+        description: `duration: ${duration}ms`,
+      });
+    });
+  });
+});

integration/health_checks/cms.js

@@ -0,0 +1,10 @@
+const { status200 } = require("../utils");
+
+const options = {
+  baseURL: process.env.DRUPAL_ROOT,
+  url: "/pantheon_healthcheck",
+};
+
+exports.check = async (_) => {
+  return status200(options);
+};

integration/health_checks/dotcom.js

@@ -0,0 +1,10 @@
+const { status200 } = require("../utils");
+
+const options = {
+  baseURL: process.env.HOST ? `https://${process.env.HOST}` : 'http://localhost:4001',
+  url: "/_health",
+};
+
+exports.check = async (_) => {
+  return status200(options);
+};

integration/health_checks/elasticache.js

@@ -0,0 +1,19 @@
+const { createCluster } = require("redis");
+
+const url = `redis://${process.env.REDIS_HOST || "127.0.0.1"}:${process.env.REDIS_PORT || "6379"}`;
+
+exports.check = async (_) => {
+  const cluster = createCluster({ rootNodes: [{ url }] });
+
+  let healthy = false;
+
+  try {
+    await cluster.connect();
+    await cluster.quit();
+    healthy = true;
+  } catch (e) {
+    healthy = false;
+  }
+
+  return healthy;
+};

integration/health_checks/otp.js

@@ -0,0 +1,10 @@
+const { status200 } = require("../utils");
+
+const options = {
+  baseURL: process.env.OPEN_TRIP_PLANNER_URL,
+  url: "/health",
+};
+
+exports.check = async (_) => {
+  return status200(options);
+};

integration/health_checks/v3-api.js

@@ -0,0 +1,10 @@
+const { status200 } = require("../utils");
+
+const options = {
+  baseURL: process.env.V3_URL,
+  url: "/_health",
+};
+
+exports.check = async (_) => {
+  return status200(options);
+};

integration/load_tests/all-scenarios.js

@@ -0,0 +1,15 @@
+const fs = require("fs");
+const path = require("path");
+
+const { fileToMetricName } = require("../utils");
+
+const filesPath = path.join(__dirname, "..", "scenarios");
+const files = fs.readdirSync(filesPath);
+
+files.forEach((file) => {
+  const { scenario } = require(path.join(filesPath, file));
+
+  exports[fileToMetricName(file)] = async function (page, context) {
+    await scenario({ page, baseURL: context.vars.target });
+  };
+});

integration/monitor/all-health-checks.js

@@ -0,0 +1,33 @@
+const cron = require("node-cron");
+const fs = require("fs");
+const Logger = require("node-json-logger");
+const path = require("path");
+const StatsD = require("hot-shots");
+
+const { fileToMetricName } = require("../utils");
+
+const prefix = "dotcom.monitor.healthcheck.";
+
+const client = new StatsD({ prefix });
+const logger = new Logger();
+
+const filesPath = path.join(__dirname, "..", "health_checks");
+const files = fs.readdirSync(filesPath);
+
+cron.schedule("* * * * *", (_) => {
+  files.forEach(async (file, index) => {
+    setTimeout(
+      async (_) => {
+        const filePath = path.join(filesPath, file);
+        const { check } = require(filePath);
+
+        const name = fileToMetricName(file);
+        const value = (await check()) ? 1 : 0;
+
+        client.gauge(name, value);
+        logger.info({ metric: `${prefix}${name}`, value });
+      },
+      (60000 / files.length) * index,
+    );
+  });
+});

integration/monitor/all-scenarios.js

@@ -0,0 +1,28 @@
+const cron = require("node-cron");
+const fs = require("fs");
+const path = require("path");
+const { Worker } = require("worker_threads");
+
+const { fileToMetricName } = require("../utils");
+
+const filesPath = path.join(__dirname, "..", "scenarios");
+
+const workers = fs.readdirSync(filesPath).map((file) => {
+  const name = fileToMetricName(file);
+  const worker = new Worker(path.join(__dirname, "worker.js"), {
+    workerData: { name, path: path.join(filesPath, file) },
+  });
+
+  return worker;
+});
+
+cron.schedule("* * * * *", (_) => {
+  workers.forEach((worker, index) => {
+    setTimeout(
+      (_) => {
+        worker.postMessage(null);
+      },
+      (60000 / workers.length) * index,
+    );
+  });
+});

integration/monitor/ecosystem.config.js

@@ -0,0 +1,14 @@
+module.exports = {
+  apps: [
+    {
+      name: "all-health-checks",
+      script: "./integration/monitor/all-health-checks.js",
+      instances: 1,
+    },
+    {
+      name: "all-scenarios",
+      script: "./integration/monitor/all-scenarios.js",
+      instances: 1,
+    },
+  ],
+};

integration/monitor/worker.js

@@ -0,0 +1,34 @@
+const { chromium } = require("playwright");
+const Logger = require("node-json-logger");
+const { parentPort, workerData } = require("worker_threads");
+const { performance } = require("node:perf_hooks");
+const StatsD = require("hot-shots");
+
+const prefix = "dotcom.monitor.";
+
+const client = new StatsD({ prefix });
+const logger = new Logger();
+
+const baseURL = process.env.HOST
+  ? `https://${process.env.HOST}`
+  : "http://localhost:4001";
+
+parentPort.on("message", async (_) => {
+  const { scenario } = require(workerData.path);
+
+  const browser = await chromium.launch();
+  const context = await browser.newContext();
+  const page = await context.newPage();
+
+  const start = performance.now();
+
+  await scenario({ page, baseURL });
+
+  const end = performance.now();
+  const duration = Math.floor(end - start);
+
+  client.gauge(workerData.name, duration);
+  logger.info({ metric: `${prefix}${workerData.name}`, duration });
+
+  await browser.close();
+});

integration/scenarios/find-transit-near-me.js

@@ -0,0 +1,17 @@
+const { expect } = require("@playwright/test");
+
+exports.scenario = async ({ page, baseURL }) => {
+  await page.goto(`${baseURL}/transit-near-me`);
+
+  await page
+    .locator("input#search-transit-near-me__input")
+    .pressSequentially("Boston City Hall");
+  await page.waitForSelector("div.c-search-bar__-dataset-locations");
+  await page.keyboard.press("ArrowDown");
+  await page.keyboard.press("Enter");
+
+  await page.waitForSelector("div.m-tnm-sidebar__route");
+  await expect
+    .poll(async () => page.locator("div.m-tnm-sidebar__route").count())
+    .toBeGreaterThan(0);
+};

integration/scenarios/plan-a-trip-from-homepage.js

@@ -0,0 +1,35 @@
+const { expect } = require("@playwright/test");
+
+exports.scenario = async ({ page, baseURL }) => {
+  await page.goto(`${baseURL}/`);
+
+  await page
+    .locator(".m-tabbed-nav__icon-text", { hasText: "Trip Planner" })
+    .click();
+
+  await page.locator("input#from").pressSequentially("South Station");
+  await page.waitForSelector(
+    "div#from-autocomplete-results span.c-search-bar__-dropdown-menu",
+  );
+  await page.keyboard.press("ArrowDown");
+  await page.keyboard.press("Enter");
+
+  await page.locator("input#to").pressSequentially("North Station");
+  await page.waitForSelector(
+    "div#to-autocomplete-results span.c-search-bar__-dropdown-menu",
+  );
+  await page.keyboard.press("ArrowDown");
+  await page.keyboard.press("Enter");
+
+  await page.locator("button#trip-plan__submit").click();
+
+  await expect(
+    page.getByRole("heading", { name: "Trip Planner" }),
+  ).toBeVisible();
+
+  await expect
+    .poll(async () =>
+      page.locator("div.m-trip-plan-results__itinerary").count(),
+    )
+    .toBeGreaterThan(0);
+};