structure changes, add web fetch api support

packages/node/src/lib/ReadMe.ts

@@ -1,14 +1,14 @@
-import type { Options } from './log';
+import type { Options } from './shared/options';
 import type { NextFunction, Request, Response } from 'express';
 
 import pkg from '../../package.json';
 import config from '../config';
 
 import findAPIKey from './find-api-key';
 import { getGroupByApiKey } from './get-group-id';
-import { log } from './log';
-import { logger } from './logger';
+import { log } from './metrics-node/log';
 import { buildSetupView } from './setup-readme-view';
+import { logger } from './shared/logger';
 import { testVerifyWebhook } from './test-verify-webhook';
 import verifyWebhook from './verify-webhook';
 

packages/node/src/lib/metrics-fetch/construct-payload.ts

@@ -0,0 +1,64 @@
+import type { OutgoingLogBody } from '../shared/metrics-log';
+import type { PayloadData, LogOptions } from '../shared/options';
+
+import { randomUUID } from 'node:crypto';
+import os from 'os';
+
+import { version } from '../../../package.json';
+import { mask } from '../shared/mask';
+
+import { processRequest } from './process-request';
+import { processResponse } from './process-response';
+
+export function getProto(req: Request): 'http' | 'https' {
+ return req.url.startsWith('https://') ? 'https' : 'http';
+}
+
+export function constructPayload(
+ req: Request,
+ res: Response,
+ payloadData: PayloadData,
+ logOptions: LogOptions,
+): OutgoingLogBody {
+ const serverTime = payloadData.responseEndDateTime.getTime() - payloadData.startedDateTime.getTime();
+
+ return {
+ _id: payloadData.logId || randomUUID(),
+ _version: 3,
+ group: {
+ id: mask(payloadData.apiKey),
+ label: payloadData.label,
+ email: payloadData.email,
+ },
+ clientIPAddress: req.headers.get('x-forwarded-for') || '',
+ development: !!logOptions?.development,
+ request: {
+ log: {
+ version: '1.2',
+ creator: {
+ name: 'readme-metrics (node)',
+ version,
+ // x64-darwin21.3.0/14.19.3
+ comment: `${os.arch()}-${os.platform()}${os.release()}/${process.versions.node}`,
+ },
+ entries: [
+ {
+ pageref: payloadData.routePath
+ ? payloadData.routePath
+ : new URL(req.url || '', `${getProto(req)}://${req.headers.get('host')}`).toString(),
+ startedDateTime: payloadData.startedDateTime.toISOString(),
+ time: serverTime,
+ request: processRequest(req, payloadData.requestBody, logOptions),
+ response: processResponse(res, payloadData.responseBody, logOptions),
+ cache: {},
+ timings: {
+ // This requires us to know the time the request was sent to the server, so we're skipping it for now
+ wait: 0,
+ receive: serverTime,
+ },
+ },
+ ],
+ },
+ },
+ };
+}

packages/node/src/lib/metrics-fetch/extract-body.ts

@@ -0,0 +1,16 @@
+export default function extractBody(target: Request | Response) {
+ const contentType = target.headers.get('content-type');
+ let body;
+
+ if (contentType?.includes('json')) {
+ target.json().then(data => {
+ body = data;
+ });
+ } else {
+ target.text().then(data => {
+ body = data;
+ });
+ }
+
+ return body;
+}

packages/node/src/lib/metrics-fetch/index.ts

@@ -0,0 +1,5 @@
+import { getProjectBaseUrl } from '../shared/get-project-base-url';
+
+import { log } from './log';
+
+export { getProjectBaseUrl, log };

packages/node/src/lib/metrics-fetch/log.ts

@@ -0,0 +1,111 @@
+import type { GroupingObject, OutgoingLogBody } from '../shared/metrics-log';
+import type { Options } from '../shared/options';
+
+import { randomUUID } from 'node:crypto';
+
+import clamp from 'lodash/clamp';
+
+import config from '../../config';
+import { getProjectBaseUrl } from '../shared/get-project-base-url';
+import { logger } from '../shared/logger';
+import { metricsAPICall } from '../shared/metrics-log';
+
+import { constructPayload } from './construct-payload';
+import extractBody from './extract-body';
+
+let queue: OutgoingLogBody[] = [];
+
+export function doSend(readmeApiKey: string, options: Options) {
+ // Copy the queue so we can send all the requests in one batch
+ const json = [...queue];
+ // Clear out the queue so we don't resend any data in the future
+ queue = [];
+
+ // Make the log call
+ metricsAPICall(readmeApiKey, json, options).catch(err => {
+ // Silently discard errors and timeouts.
+ if (options.development) {
+ logger.error({ message: 'Failed to capture API request.', err });
+ }
+ });
+
+ logger.debug({ message: 'Queue flushed.', args: { queue } });
+}
+// Make sure we flush the queue if the process is exited
+process.on('exit', doSend);
+
+function setDocumentationHeader(res: Response, baseLogUrl: string, logId: string) {
+ // This is to catch the potential race condition where `getProjectBaseUrl()`
+ // takes longer to respond than the original req/res to finish. Without this
+ // we would get an error that would be very difficult to trace. This could
+ // do with a test, but it's a little difficult to test. Maybe with a nock()
+ // delay timeout.
+ const documentationUrl = `${baseLogUrl}/logs/${logId}`;
+ logger.verbose({
+ message: 'Created URL to your API request log.',
+ args: { 'x-documentation-url': documentationUrl },
+ });
+ res.headers.set('x-documentation-url', documentationUrl);
+}
+/**
+ * This method will send supplied API requests to ReadMe Metrics.
+ *
+ * @see {@link https://readme.com/metrics}
+ * @see {@link https://docs.readme.com/docs/sending-logs-to-readme-with-nodejs}
+ * @param readmeApiKey The API key for your ReadMe project. This ensures your requests end up in
+ * your dashboard. You can read more about the API key in
+ * [our docs](https://docs.readme.com/reference/authentication).
+ * @param req This is your incoming request object from your HTTP server and/or framework.
+ * @param res This is your outgoing response object for your HTTP server and/or framework.
+ * @param group A function that helps translate incoming request data to our metrics grouping data.
+ * @param options Additional options. See the documentation for more details.
+ */
+export function log(readmeApiKey: string, req: Request, res: Response, group: GroupingObject, options: Options = {}) {
+ if (req.method === 'OPTIONS') return undefined;
+ if (!readmeApiKey) throw new Error('You must provide your ReadMe API key');
+ if (!group) throw new Error('You must provide a group');
+ if (options.logger) {
+ if (typeof options.logger === 'boolean') logger.configure({ isLoggingEnabled: true });
+ else logger.configure({ isLoggingEnabled: true, strategy: options.logger });
+ }
+
+ // Ensures the buffer length is between 1 and 30
+ const bufferLength = clamp(options.bufferLength || config.bufferLength, 1, 30);
+
+ const startedDateTime = new Date();
+ const logId = randomUUID();
+
+ // baseLogUrl can be provided, but if it isn't then we
+ // attempt to fetch it from the ReadMe API
+ if (typeof options.baseLogUrl === 'string') {
+ setDocumentationHeader(res, options.baseLogUrl, logId);
+ } else {
+ getProjectBaseUrl(readmeApiKey).then(baseLogUrl => {
+ setDocumentationHeader(res, baseLogUrl, logId);
+ });
+ }
+
+ const requestBody = extractBody(req);
+ const responseBody = extractBody(res);
+
+ const payload = constructPayload(
+ req,
+ res,
+ {
+ ...group,
+ logId,
+ startedDateTime,
+ responseEndDateTime: new Date(),
+ routePath: '',
+ responseBody,
+ requestBody,
+ },
+ options,
+ );
+
+ queue.push(payload);
+ logger.debug({ message: 'Request enqueued.', args: { queue } });
+ if (queue.length >= bufferLength) doSend(readmeApiKey, options);
+
+ return logId;
+}

packages/node/src/lib/metrics-fetch/process-request.ts

@@ -0,0 +1,118 @@
+import type { LogOptions } from '../shared/options';
+import type { Cookie, Param, PostData, Request as HarRequest } from 'har-format';
+
+import * as qs from 'querystring';
+import url, { URL } from 'url';
+
+import * as contentType from 'content-type';
+
+import { mask } from '../shared/mask';
+import { objectToArray, searchToArray } from '../shared/object-to-array';
+import {
+ fixHeader,
+ redactOtherProperties,
+ redactProperties,
+ isApplicationJson,
+ parseRequestBody,
+} from '../shared/processing-helpers';
+
+import { getProto } from './construct-payload';
+import { headersToObject } from './process-response';
+
+export function processRequest(
+ req: Request,
+ requestBody?: Record<string, unknown> | string,
+ options?: LogOptions,
+): HarRequest {
+ const protocol = fixHeader(req.headers.get('x-forwarded-proto') || '')?.toLowerCase() || getProto(req);
+ const host = fixHeader(req.headers.get('x-forwarded-host') || '') || req.headers.get('host');
+
+ const denylist = options?.denylist || options?.blacklist;
+ const allowlist = options?.allowlist || options?.whitelist;
+
+ let mimeType = '';
+ try {
+ mimeType = contentType.parse(req.headers.get('content-type') || '').type;
+ } catch (e) {} // eslint-disable-line no-empty
+
+ let reqBody = typeof requestBody === 'string' ? parseRequestBody(requestBody, mimeType) : requestBody;
+ let postData: PostData | undefined;
+
+ let headers = headersToObject(req.headers);
+
+ if (denylist) {
+ reqBody = typeof reqBody === 'object' ? redactProperties(reqBody, denylist) : reqBody;
+ headers = redactProperties(headers, denylist);
+ }
+
+ if (allowlist && !denylist) {
+ reqBody = typeof reqBody === 'object' ? redactOtherProperties(reqBody, allowlist) : reqBody;
+ headers = redactOtherProperties(headers, allowlist);
+ }
+
+ if (mimeType === 'application/x-www-form-urlencoded') {
+ postData = {
+ mimeType,
+ // `reqBody` is likely to be an object, but can be empty if no HTTP body sent
+ params: objectToArray((reqBody || {}) as Record<string, unknown>) as Param[],
+ };
+ } else if (isApplicationJson(mimeType)) {
+ postData = {
+ mimeType,
+ text: typeof reqBody === 'object' || Array.isArray(reqBody) ? JSON.stringify(reqBody) : reqBody || '',
+ };
+ } else if (mimeType) {
+ let stringBody = '';
+
+ try {
+ stringBody = typeof reqBody === 'string' ? reqBody : JSON.stringify(reqBody);
+ } catch (e) {
+ stringBody = '[ReadMe is unable to handle circular JSON. Please contact support if you have any questions.]';
+ }
+
+ postData = {
+ mimeType,
+ // Do our best to record *some sort of body* even if it's not 100% accurate.
+ text: stringBody,
+ };
+ }
+
+ // We use a fake host here because we rely on the host header which could be redacted.
+ // We only ever use this reqUrl with the fake hostname for the pathname and querystring.
+ // req.originalUrl is express specific, req.url is node.js
+ const reqUrl = new URL(req.url || '', 'https://readme.io');
+
+ if (headers.authorization) {
+ req.headers.set('authorization', mask(headers.authorization as string));
+ }
+
+ const requestData: HarRequest = {
+ method: req.method || '',
+ url: url.format({
+ // Handle cases where some reverse proxies put two protocols into x-forwarded-proto
+ // This line does the following: "https,http" -> "https"
+ // https://github.com/readmeio/metrics-sdks/issues/378
+ protocol: protocol.split(',')[0],
+ host,
+ pathname: reqUrl.pathname,
+ // Search includes the leading questionmark, format assumes there isn't one, so we trim that off.
+ query: qs.parse(reqUrl.search.substring(1)),
+ }),
+ httpVersion: `${getProto(req).toUpperCase()}/5`, // todo: figure out what we can do with this, there is no analogue in fetch api
+ headers: objectToArray(headers, { castToString: true }),
+ queryString: searchToArray(reqUrl.searchParams),
+ postData,
+ // TODO: When readme starts accepting these, send the correct values
+ cookies: [] satisfies Cookie[],
+ headersSize: -1,
+ bodySize: -1,
+ } as const;
+
+ if (typeof requestData.postData === 'undefined') {
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
+ const { postData: postDataToBeOmitted, ...remainingRequestData } = requestData;
+ return remainingRequestData;
+ }
+
+ return requestData;
+}