From 23da05e8643a42b300a5dc82913cb7d8167c9b3d Mon Sep 17 00:00:00 2001 From: Jacob Bare Date: Tue, 30 Nov 2021 08:29:18 -0600 Subject: [PATCH 1/3] Set correct exposed port --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index efdae7f5c..7bc4082ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -381,6 +381,7 @@ services: <<: *env <<: *env-newrelic EMBEDLY_API_KEY: ${EMBEDLY_API_KEY-} + EXPOSED_PORT: 10402 ports: - "10402:80" From 4bbcc730f1d3db524a103937a1cb6b54ab6e5e79 Mon Sep 17 00:00:00 2001 From: Jacob Bare Date: Tue, 30 Nov 2021 08:29:27 -0600 Subject: [PATCH 2/3] Add redis caching --- docker-compose.yml | 3 +++ services/oembed/package.json | 1 + services/oembed/src/env.js | 1 + services/oembed/src/redis.js | 4 ++++ 4 files changed, 9 insertions(+) create mode 100644 services/oembed/src/redis.js diff --git a/docker-compose.yml b/docker-compose.yml index 7bc4082ad..ddef96760 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -380,8 +380,11 @@ services: environment: <<: *env <<: *env-newrelic + REDIS_CACHE_DSN: ${REDIS_CACHE_DSN-redis://redis} EMBEDLY_API_KEY: ${EMBEDLY_API_KEY-} EXPOSED_PORT: 10402 + depends_on: + - redis ports: - "10402:80" diff --git a/services/oembed/package.json b/services/oembed/package.json index ae4e505e2..7b564514c 100644 --- a/services/oembed/package.json +++ b/services/oembed/package.json @@ -19,6 +19,7 @@ "body-parser": "^1.19.0", "express": "^4.17.1", "helmet": "^3.23.3", + "ioredis": "^4.27.10", "newrelic": "^6.14.0" } } diff --git a/services/oembed/src/env.js b/services/oembed/src/env.js index 440a6cbc1..4d77b5fb4 100644 --- a/services/oembed/src/env.js +++ b/services/oembed/src/env.js @@ -16,6 +16,7 @@ module.exports = cleanEnv(process.env, { NEW_RELIC_LICENSE_KEY: nonemptystr({ desc: 'The license key for New Relic.', devDefault: '(unset)' }), EXPOSED_HOST: str({ desc: 'The external host to run on.', default: 'localhost' }), EXPOSED_PORT: port({ desc: 'The external port to run on.', default: 10013 }), + REDIS_CACHE_DSN: nonemptystr({ desc: 'The Redis DSN where cache values should be saved.' }), TERMINUS_TIMEOUT: num({ desc: 'Number of milliseconds before forceful exiting', default: 1000 }), TERMINUS_SHUTDOWN_DELAY: num({ desc: 'Number of milliseconds before the HTTP server starts its shutdown', default: 10000 }), }); diff --git a/services/oembed/src/redis.js b/services/oembed/src/redis.js new file mode 100644 index 000000000..2eb1b5f1b --- /dev/null +++ b/services/oembed/src/redis.js @@ -0,0 +1,4 @@ +const Redis = require('ioredis'); +const { REDIS_CACHE_DSN } = require('./env'); + +module.exports = new Redis(REDIS_CACHE_DSN); From d319c31ef562362482256ae5bf8cc9021eb8854e Mon Sep 17 00:00:00 2001 From: Jacob Bare Date: Tue, 30 Nov 2021 08:48:16 -0600 Subject: [PATCH 3/3] Add caching to routes POST requests will set to cache, but never read from it GET requests will set to, and read from, cache. If `no-cache` is present in the `cache-control` header, a fresh/non-cached version will be retrieved, set to cache, and returned --- services/oembed/src/app.js | 24 +++++++++++++++++++----- services/oembed/src/cache.js | 30 ++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 services/oembed/src/cache.js diff --git a/services/oembed/src/app.js b/services/oembed/src/app.js index 264dcce33..7fb1c8914 100644 --- a/services/oembed/src/app.js +++ b/services/oembed/src/app.js @@ -3,6 +3,7 @@ const helmet = require('helmet'); const { json } = require('body-parser'); const { asyncRoute } = require('@parameter1/base-cms-utils'); const embedly = require('./embedly'); +const cache = require('./cache'); const app = express(); const dev = process.env.NODE_ENV === 'development'; @@ -12,15 +13,28 @@ app.use(json()); app.set('trust proxy', ['loopback', 'linklocal', 'uniquelocal']); app.post('/', asyncRoute(async (req, res) => { - const { body } = req; - const data = await embedly.oembed(body.url, body.params); + const { url, ...params } = req.body; + const data = await embedly.oembed(url, params); + // post will set to cache, but not read from it + await cache.setFor({ url, params, data }); res.json(data); })); app.get('/', asyncRoute(async (req, res) => { - const { query } = req; - const data = await embedly.oembed(query.url, query); - res.json(data); + const { url, ...params } = req.query; + const cacheControl = req.headers['cache-control']; + const noCache = cacheControl && /no-cache/i.test(cacheControl); + + // allow for fresh data retrieval + const cached = noCache ? null : await cache.getFor({ url, params }); + res.set('X-Cache', cached ? 'hit' : 'miss'); + if (cached) { + res.set('Age', cached.age); + return res.json(cached.data); + } + const data = await embedly.oembed(url, params); + await cache.setFor({ url, params, data }); + return res.json(data); })); // eslint-disable-next-line no-unused-vars diff --git a/services/oembed/src/cache.js b/services/oembed/src/cache.js new file mode 100644 index 000000000..6bc2568dd --- /dev/null +++ b/services/oembed/src/cache.js @@ -0,0 +1,30 @@ +const { createHash } = require('crypto'); +const redis = require('./redis'); + +const createCacheKey = ({ url, params } = {}) => { + const hash = createHash('sha256').update(JSON.stringify({ url, params })).digest('hex'); + return `base_oembed:${hash}`; +}; + +const getFor = async ({ url, params } = {}) => { + const key = createCacheKey({ url, params }); + const serialized = await redis.get(key); + if (!serialized) return null; + const cacheValue = JSON.parse(serialized); + const age = Math.round((Date.now() - cacheValue.cacheTime) / 1000); + return { ...cacheValue, age }; +}; + +const setFor = async ({ url, params, data } = {}) => { + const key = createCacheKey({ url, params }); + const cacheValue = { data, cacheTime: Date.now() }; + const ttl = 60 * 60 * 24 * 3; // cache for 72 hours + await redis.set(key, JSON.stringify(cacheValue), 'EX', ttl); +}; + + +module.exports = { + createCacheKey, + getFor, + setFor, +};