From 7056d1ff9bd3ed000a30fb974d2956f5c0c942e5 Mon Sep 17 00:00:00 2001 From: bjarneo Date: Mon, 29 Jan 2024 07:24:38 +0100 Subject: [PATCH] feat: add the new rate limit (#263) * feat: add the new rate limit This rate limit will use the method, path and ip as a key, and will be route based rate limiter. Also, this rate limit will be in memory for now using a Map as the store. * chore: remove the commented code * refactor: minor changes * fix: create env vars to config for the rate limit * fix: fix the default options * chore: use arrow function * chore: use spread --- README.md | 48 ++++++++++++++------------- config/default.cjs | 6 ++++ package-lock.json | 57 ++++---------------------------- package.json | 3 +- server.js | 38 ++++++++++----------- src/server/plugins/rate-limit.js | 54 ++++++++++++++++++++++++++++++ 6 files changed, 112 insertions(+), 94 deletions(-) create mode 100644 src/server/plugins/rate-limit.js diff --git a/README.md b/README.md index 319b9b85..bdbfe6be 100644 --- a/README.md +++ b/README.md @@ -119,29 +119,31 @@ npx hemmelig --help ## Environment variables -| ENV vars | Description | Default | -| ------------------------------|:---------------------------------------------------------------------:| --------------------:| -| `SECRET_LOCAL_HOSTNAME` | The local hostname for the fastify instance | 0.0.0.0 | -| `SECRET_PORT` | The port number for the fastify instance | 3000 | -| `SECRET_HOST` | Used for i.e. set cors/cookies to your domain name | "" | -| `SECRET_MAX_TEXT_SIZE` | The max text size for the secret. Is set in kb. i.e. 256 for 256kb. | 256 | -| `SECRET_JWT_SECRET` | Override this for the secret signin JWT tokens for log in | good_luck_have_fun | -| `SECRET_ROOT_USER` | Override this for the root account username | groot | -| `SECRET_ROOT_PASSWORD` | This is the root password, override it with your own password | iamgroot | -| `SECRET_ROOT_EMAIL` | This is the root email, override it with your own email | groot@hemmelig.app | -| `SECRET_FILE_SIZE` | Set the total allowed upload file size in mb. | 4 | -| `SECRET_FORCED_LANGUAGE` | Set the default language for the application. | en | -| `SECRET_UPLOAD_RESTRICTION` | Set the restriction for uploads to signed in users | "true" | -| `SECRET_DO_SPACES_ENDPOINT` | The Digital Ocean Spaces/AWS s3 endpoint | "" | -| `SECRET_DO_SPACES_KEY` | The Digital Ocean Spaces/AWS s3 key | "" | -| `SECRET_DO_SPACES_SECRET` | The Digital Ocean Spaces/AWS s3 secret | "" | -| `SECRET_DO_SPACES_BUCKET` | The Digital Ocean Spaces/AWS s3 bucket name | "" | -| `SECRET_DO_SPACES_FOLDER` | The Digital Ocean Spaces/AWS s3 folder for the uploaded files | "" | -| `SECRET_AWS_S3_REGION` | The Digital AWS s3 region | "" | -| `SECRET_AWS_S3_KEY` | The Digital AWS s3 key | "" | -| `SECRET_AWS_S3_SECRET` | The Digital AWS s3 secret | "" | -| `SECRET_AWS_S3_BUCKET` | The Digital AWS s3 bucket name | "" | -| `SECRET_AWS_S3_FOLDER` | The Digital AWS s3 folder for the uploaded files | "" | +| ENV vars | Description | Default | +| --------------------------------|:---------------------------------------------------------------------:| --------------------:| +| `SECRET_LOCAL_HOSTNAME` | The local hostname for the fastify instance | 0.0.0.0 | +| `SECRET_PORT` | The port number for the fastify instance | 3000 | +| `SECRET_HOST` | Used for i.e. set cors/cookies to your domain name | "" | +| `SECRET_MAX_TEXT_SIZE` | The max text size for the secret. Is set in kb. i.e. 256 for 256kb. | 256 | +| `SECRET_JWT_SECRET` | Override this for the secret signin JWT tokens for log in | good_luck_have_fun | +| `SECRET_ROOT_USER` | Override this for the root account username | groot | +| `SECRET_ROOT_PASSWORD` | This is the root password, override it with your own password | iamgroot | +| `SECRET_ROOT_EMAIL` | This is the root email, override it with your own email | groot@hemmelig.app | +| `SECRET_FILE_SIZE` | Set the total allowed upload file size in mb. | 4 | +| `SECRET_FORCED_LANGUAGE` | Set the default language for the application. | en | +| `SECRET_UPLOAD_RESTRICTION` | Set the restriction for uploads to signed in users | "true" | +| `SECRET_RATE_LIMIT_MAX` | The maximum allowed requests each time frame | 1000 | +| `SECRET_RATE_LIMIT_TIME_WINDOW` | The time window for the requests before being rate limited in seconds | 60 | +| `SECRET_DO_SPACES_ENDPOINT` | The Digital Ocean Spaces/AWS s3 endpoint | "" | +| `SECRET_DO_SPACES_KEY` | The Digital Ocean Spaces/AWS s3 key | "" | +| `SECRET_DO_SPACES_SECRET` | The Digital Ocean Spaces/AWS s3 secret | "" | +| `SECRET_DO_SPACES_BUCKET` | The Digital Ocean Spaces/AWS s3 bucket name | "" | +| `SECRET_DO_SPACES_FOLDER` | The Digital Ocean Spaces/AWS s3 folder for the uploaded files | "" | +| `SECRET_AWS_S3_REGION` | The Digital AWS s3 region | "" | +| `SECRET_AWS_S3_KEY` | The Digital AWS s3 key | "" | +| `SECRET_AWS_S3_SECRET` | The Digital AWS s3 secret | "" | +| `SECRET_AWS_S3_BUCKET` | The Digital AWS s3 bucket name | "" | +| `SECRET_AWS_S3_FOLDER` | The Digital AWS s3 folder for the uploaded files | "" | ## Supported languages diff --git a/config/default.cjs b/config/default.cjs index 9f8cc639..10c7c7ae 100644 --- a/config/default.cjs +++ b/config/default.cjs @@ -21,6 +21,8 @@ const { SECRET_AWS_S3_FOLDER = '', SECRET_MAX_TEXT_SIZE = 256, // 256 kb SECRET_UPLOAD_RESTRICTION = 'true', // true = only allow uploads from signed in users + SECRET_RATE_LIMIT_MAX = 1000, + SECRET_RATE_LIMIT_TIME_WINDOW = 60, NODE_ENV = 'development', } = process.env; @@ -31,6 +33,10 @@ const config = { port: SECRET_PORT, secret_key: SECRET_MASTER_KEY, upload_restriction: JSON.parse(SECRET_UPLOAD_RESTRICTION), + rateLimit: { + max: Number(SECRET_RATE_LIMIT_MAX), + timeWindow: Number(SECRET_RATE_LIMIT_TIME_WINDOW) * 1000, + }, // root account management account: { root: { diff --git a/package-lock.json b/package-lock.json index 1d9de9d2..b5ab1355 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,6 @@ "@fastify/helmet": "^9.1.0", "@fastify/jwt": "^7.2.3", "@fastify/multipart": "^7.1.1", - "@fastify/rate-limit": "^8.0.0", "@fastify/static": "^6.5.0", "@mantine/core": "^6.0.6", "@mantine/form": "^6.0.6", @@ -29,7 +28,7 @@ "email-validator": "^2.0.4", "extract-domain": "^2.4.8", "fastify": "^4.9.1", - "fastify-plugin": "^3.0.0", + "fastify-plugin": "^3.0.1", "file-type": "^18.0.0", "generate-password-browser": "^1.1.0", "ip-range-check": "^0.2.0", @@ -1993,26 +1992,6 @@ "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-4.2.1.tgz", "integrity": "sha512-dlGKiwLzRBKkEf5J5ho0uAD/Jdv8GQVUbriB3tAX3ehRUXE4gTV3lRd5inEg9li1aLzb0EGj8y2K4/8g1TN06g==" }, - "node_modules/@fastify/rate-limit": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@fastify/rate-limit/-/rate-limit-8.0.0.tgz", - "integrity": "sha512-73pQFgpx6RMmY5nriYQW0AIATZpa/OAvWa5PQ9JHEqgjKMLNv9zLAphWIRh5914aN6sqtv8sLICr3Tp1NbXQIQ==", - "dependencies": { - "fastify-plugin": "^4.0.0", - "ms": "^2.1.3", - "tiny-lru": "^10.0.0" - } - }, - "node_modules/@fastify/rate-limit/node_modules/fastify-plugin": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-4.5.0.tgz", - "integrity": "sha512-79ak0JxddO0utAXAQ5ccKhvs6vX2MGyHHMMsmZkBANrq3hXc1CHzvNPHOcvTsVMEPl5I+NT+RO4YKMGehOfSIg==" - }, - "node_modules/@fastify/rate-limit/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, "node_modules/@fastify/static": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/@fastify/static/-/static-6.5.0.tgz", @@ -4593,9 +4572,9 @@ } }, "node_modules/fastify-plugin": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-3.0.0.tgz", - "integrity": "sha512-ZdCvKEEd92DNLps5n0v231Bha8bkz1DjnPP/aEz37rz/q42Z5JVLmgnqR4DYuNn3NXAO3IDCPyRvgvxtJ4Ym4w==" + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-3.0.1.tgz", + "integrity": "sha512-qKcDXmuZadJqdTm6vlCqioEbyewF60b/0LOFCcYN1B6BIZGlYJumWWOYs70SFYLDAH4YqdE1cxH/RKMG7rFxgA==" }, "node_modules/fastparallel": { "version": "2.4.1", @@ -10122,28 +10101,6 @@ } } }, - "@fastify/rate-limit": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@fastify/rate-limit/-/rate-limit-8.0.0.tgz", - "integrity": "sha512-73pQFgpx6RMmY5nriYQW0AIATZpa/OAvWa5PQ9JHEqgjKMLNv9zLAphWIRh5914aN6sqtv8sLICr3Tp1NbXQIQ==", - "requires": { - "fastify-plugin": "^4.0.0", - "ms": "^2.1.3", - "tiny-lru": "^10.0.0" - }, - "dependencies": { - "fastify-plugin": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-4.5.0.tgz", - "integrity": "sha512-79ak0JxddO0utAXAQ5ccKhvs6vX2MGyHHMMsmZkBANrq3hXc1CHzvNPHOcvTsVMEPl5I+NT+RO4YKMGehOfSIg==" - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - } - } - }, "@fastify/static": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/@fastify/static/-/static-6.5.0.tgz", @@ -12111,9 +12068,9 @@ } }, "fastify-plugin": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-3.0.0.tgz", - "integrity": "sha512-ZdCvKEEd92DNLps5n0v231Bha8bkz1DjnPP/aEz37rz/q42Z5JVLmgnqR4DYuNn3NXAO3IDCPyRvgvxtJ4Ym4w==" + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-3.0.1.tgz", + "integrity": "sha512-qKcDXmuZadJqdTm6vlCqioEbyewF60b/0LOFCcYN1B6BIZGlYJumWWOYs70SFYLDAH4YqdE1cxH/RKMG7rFxgA==" }, "fastparallel": { "version": "2.4.1", diff --git a/package.json b/package.json index 8689004d..ff521193 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,6 @@ "@fastify/helmet": "^9.1.0", "@fastify/jwt": "^7.2.3", "@fastify/multipart": "^7.1.1", - "@fastify/rate-limit": "^8.0.0", "@fastify/static": "^6.5.0", "@mantine/core": "^6.0.6", "@mantine/form": "^6.0.6", @@ -54,7 +53,7 @@ "email-validator": "^2.0.4", "extract-domain": "^2.4.8", "fastify": "^4.9.1", - "fastify-plugin": "^3.0.0", + "fastify-plugin": "^3.0.1", "file-type": "^18.0.0", "generate-password-browser": "^1.1.0", "ip-range-check": "^0.2.0", diff --git a/server.js b/server.js index a27d131b..c7bda079 100644 --- a/server.js +++ b/server.js @@ -1,40 +1,41 @@ // Boot scripts import('./src/server/bootstrap.js'); +import cookie from '@fastify/cookie'; +import cors from '@fastify/cors'; +import helmet from '@fastify/helmet'; +import jwt from '@fastify/jwt'; +import fstatic from '@fastify/static'; import config from 'config'; -import path from 'path'; +import importFastify from 'fastify'; import fs from 'fs'; -import { fileURLToPath } from 'url'; import { JSDOM } from 'jsdom'; -import importFastify from 'fastify'; +import path from 'path'; +import { fileURLToPath } from 'url'; import template from 'y8'; -import helmet from '@fastify/helmet'; -import cors from '@fastify/cors'; -import fstatic from '@fastify/static'; -import cookie from '@fastify/cookie'; -import jwt from '@fastify/jwt'; -import rateLimit from '@fastify/rate-limit'; + +import rateLimit from './src/server/plugins/rate-limit.js'; import adminDecorator from './src/server/decorators/admin.js'; -import jwtDecorator from './src/server/decorators/jwt.js'; -import userFeatures from './src/server/decorators/user-features.js'; import allowedIp from './src/server/decorators/allowed-ip.js'; import attachment from './src/server/decorators/attachment-upload.js'; +import jwtDecorator from './src/server/decorators/jwt.js'; +import userFeatures from './src/server/decorators/user-features.js'; import readCookieAllRoutesHandler from './src/server/prehandlers/cookie-all-routes.js'; -import readOnlyHandler from './src/server/prehandlers/read-only.js'; -import disableUserHandler from './src/server/prehandlers/disable-users.js'; import disableUserAccountCreationHandler from './src/server/prehandlers/disable-user-account-creation.js'; +import disableUserHandler from './src/server/prehandlers/disable-users.js'; +import readOnlyHandler from './src/server/prehandlers/read-only.js'; import restrictOrganizationEmailHandler from './src/server/prehandlers/restrict-organization-email.js'; -import usersRoute from './src/server/controllers/admin/users.js'; +import accountRoute from './src/server/controllers/account.js'; import adminSettingsRoute from './src/server/controllers/admin/settings.js'; +import usersRoute from './src/server/controllers/admin/users.js'; import authenticationRoute from './src/server/controllers/authentication.js'; -import accountRoute from './src/server/controllers/account.js'; import downloadRoute from './src/server/controllers/download.js'; +import healthzRoute from './src/server/controllers/healthz.js'; import secretRoute from './src/server/controllers/secret.js'; import statsRoute from './src/server/controllers/stats.js'; -import healthzRoute from './src/server/controllers/healthz.js'; const isDev = process.env.NODE_ENV === 'development'; @@ -45,11 +46,10 @@ const fastify = importFastify({ bodyLimit: MAX_FILE_BYTES, }); -// https://github.com/fastify/fastify-rate-limit fastify.register(rateLimit, { prefix: '/api/', - max: 10000, - timeWindow: '1 minute', + max: config.get('rateLimit.max'), + timeWindow: config.get('rateLimit.timeWindow'), }); // https://github.com/fastify/fastify-helmet diff --git a/src/server/plugins/rate-limit.js b/src/server/plugins/rate-limit.js new file mode 100644 index 00000000..05d51be9 --- /dev/null +++ b/src/server/plugins/rate-limit.js @@ -0,0 +1,54 @@ +import fp from 'fastify-plugin'; +import getClientIp from '../helpers/client-ip.js'; + +const store = new Map(); + +const defaultOptions = { + prefix: '/', + max: 1000, // x requests + timeWindow: 60 * 1000, // 1 minute +}; + +// Consider to extract this functionality to a separate plugin, +// and publish it on NPM +// Refactor the code into a class +// and make a redis adapter for it +export default fp((fastify, options = {}, done) => { + const settings = { ...defaultOptions, ...options }; + + fastify.decorate('rateLimit', async (req, res) => { + if (!req.url.startsWith(settings.prefix)) { + done(); + } + + const ip = getClientIp(req.headers); + const key = `${req.method}${req.url}${ip}`; + + const currentTime = Date.now(); + + if (!store.has(key) || currentTime > store.get(key)?.reset) { + store.set(key, { + count: 0, + reset: currentTime + settings.timeWindow, + }); + } + + const current = store.get(key); + current.count += 1; + + const resetTime = Math.floor((current.reset - currentTime) / 1000); + + if (current.count > settings.max && currentTime <= current.reset) { + res.header('X-RateLimit-Reset', resetTime); + + return res.code(429).send({ message: 'Too many requests, please try again later.' }); + } + + res.header('X-RateLimit-Limit', settings.max); + res.header('X-RateLimit-Remaining', settings.max - current.count); + res.header('X-RateLimit-Reset', resetTime); + }); + + fastify.addHook('onRequest', fastify.rateLimit); + done(); +});