diff --git a/client/package.json b/client/package.json index 978ba05042..12a6580115 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "@coralproject/talk", - "version": "9.2.0", + "version": "9.3.0", "author": "The Coral Project", "homepage": "https://coralproject.net/", "sideEffects": [ diff --git a/common/lib/constants.ts b/common/lib/constants.ts index db19d6d600..4c9b6e9815 100644 --- a/common/lib/constants.ts +++ b/common/lib/constants.ts @@ -86,6 +86,12 @@ export const DEFAULT_SESSION_DURATION = 90 * TIME.DAY; */ export const COMMENT_REPEAT_POST_DURATION = 6 * TIME.MINUTE; +/** + * COUNTS_V2_CACHE_DURATION is the length of time in seconds that the comment + * counts for a story are cached for the counts v2 endoint. + */ +export const COUNTS_V2_CACHE_DURATION = 1 * TIME.DAY; + /** * SPOILER_CLASSNAME is the classname that is attached to spoilers. */ diff --git a/common/package.json b/common/package.json index 5d590ced5b..f397dd4ba6 100644 --- a/common/package.json +++ b/common/package.json @@ -1,6 +1,6 @@ { "name": "common", - "version": "9.2.0", + "version": "9.3.0", "description": "", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/config/package.json b/config/package.json index 2ffb9254b5..7c57caa62d 100644 --- a/config/package.json +++ b/config/package.json @@ -1,6 +1,6 @@ { "name": "common", - "version": "9.2.0", + "version": "9.3.0", "description": "", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/server/package.json b/server/package.json index 46c902df9f..e79fc8bb16 100644 --- a/server/package.json +++ b/server/package.json @@ -1,6 +1,6 @@ { "name": "@coralproject/talk", - "version": "9.2.0", + "version": "9.3.0", "author": "The Coral Project", "homepage": "https://coralproject.net/", "sideEffects": [ diff --git a/server/src/core/server/app/handlers/api/story/count.ts b/server/src/core/server/app/handlers/api/story/count.ts index 9356df22b6..406e73abf3 100644 --- a/server/src/core/server/app/handlers/api/story/count.ts +++ b/server/src/core/server/app/handlers/api/story/count.ts @@ -1,22 +1,35 @@ import Joi from "joi"; +import { COUNTS_V2_CACHE_DURATION } from "coral-common/common/lib/constants"; import { CountJSONPData } from "coral-common/common/lib/types/count"; import { AppOptions } from "coral-server/app"; import { validate } from "coral-server/app/request/body"; import { MongoContext } from "coral-server/data/context"; +import logger from "coral-server/logger"; import { retrieveManyStoryRatings } from "coral-server/models/comment"; import { PUBLISHED_STATUSES } from "coral-server/models/comment/constants"; -import { Story } from "coral-server/models/story"; -import { Tenant } from "coral-server/models/tenant"; +import { retrieveStoryCommentCounts, Story } from "coral-server/models/story"; +import { hasFeatureFlag, Tenant } from "coral-server/models/tenant"; import { I18n, translate } from "coral-server/services/i18n"; import { find } from "coral-server/services/stories"; import { RequestHandler, TenantCoralRequest } from "coral-server/types/express"; -import { GQLSTORY_MODE } from "coral-server/graph/schema/__generated__/types"; +import { + GQLFEATURE_FLAG, + GQLSTORY_MODE, +} from "coral-server/graph/schema/__generated__/types"; const NUMBER_CLASS_NAME = "coral-count-number"; const TEXT_CLASS_NAME = "coral-count-text"; +interface CountsV2Body { + storyIDs: string[]; +} + +const CountsV2BodySchema = Joi.object().keys({ + storyIDs: Joi.array().items(Joi.string().required()).required().max(100), +}); + export type JSONPCountOptions = Pick< AppOptions, "mongo" | "tenantCache" | "i18n" @@ -171,3 +184,103 @@ async function calculateStoryCount( 0 ); } + +export type CountV2Options = Pick; + +interface CountResult { + storyID: string; + redisCount?: number; // set if count came from redis + mongoCount?: number; // set if count came from mongo + count: number; // always set, can come from mongo or redis +} + +export const computeCountKey = (tenantID: string, storyID: string) => { + return `${tenantID}:${storyID}:count`; +}; + +export const countsV2Handler = + ({ mongo, redis }: CountV2Options): RequestHandler => + async (req, res, next) => { + const { tenant } = req.coral; + + try { + if (!hasFeatureFlag(tenant, GQLFEATURE_FLAG.COUNTS_V2)) { + return res.status(400).send("Counts V2 api not enabled"); + } + + const { storyIDs }: CountsV2Body = validate(CountsV2BodySchema, req.body); + + // grab what keys we can that are already in Redis with one bulk call + const redisCounts = await redis.mget( + ...storyIDs.map((id) => computeCountKey(tenant.id, id)) + ); + + // quickly iterate over our results and see if we're missing any of the + // values for our requested story id's + const countResults = new Map(); + const missingIDs: string[] = []; + for (let i = 0; i < storyIDs.length; i++) { + const storyID = storyIDs[i]; + const redisCount = redisCounts[i]; + + if (redisCount !== null && redisCount !== undefined) { + try { + const count = parseInt(redisCount, 10); + countResults.set(storyID, { storyID, redisCount: count, count }); + } catch { + missingIDs.push(storyID); + } + } else { + missingIDs.push(storyID); + } + } + + // compute out the counts for any story id's we couldn't + // get a count from Redis + for (const missingID of missingIDs) { + const count = await retrieveStoryCommentCounts( + mongo, + tenant.id, + missingID + ); + + const key = computeCountKey(tenant.id, missingID); + await redis.set(key, count, "EX", COUNTS_V2_CACHE_DURATION); + logger.debug("set story count for counts v2 in redis cache", { + storyID: missingID, + count, + }); + + countResults.set(missingID, { + storyID: missingID, + mongoCount: count, + count, + }); + } + + // strictly follow the result set based on the story id's + // we were given from the caller. Then return the values + // from our combined redis/mongo results. + // + // this means if a user asked for id's ["1", "2", "3", "does-not-exist", "1"], they would + // receive counts like so: + // [ + // { storyID: 1, redisCount: 2, count: 2 }, + // { storyID: 2, redisCount: 5, count: 5 }, + // { storyID: 3, mongoCount: 2, count: 2 }, + // null, + // { storyID: 1, redisCount: 2, count: 2 } + // ] + // + // many of our counts endpoints appreciate this adherence. + const results: Array = []; + for (const storyID of storyIDs) { + const value = countResults.get(storyID) ?? null; + results.push(value); + } + + res.send(JSON.stringify(results)); + } catch (err) { + return next(err); + } + }; diff --git a/server/src/core/server/app/router/api/story.ts b/server/src/core/server/app/router/api/story.ts index e89ca7c073..8ea3a3f02e 100644 --- a/server/src/core/server/app/router/api/story.ts +++ b/server/src/core/server/app/router/api/story.ts @@ -1,14 +1,20 @@ +import bytes from "bytes"; + import { AppOptions } from "coral-server/app"; import { activeJSONPHandler, countHandler, countJSONPHandler, + countsV2Handler, ratingsJSONPHandler, } from "coral-server/app/handlers"; +import { jsonMiddleware } from "coral-server/app/middleware"; import cacheMiddleware from "coral-server/app/middleware/cache"; import { createAPIRouter } from "./helpers"; +const REQUEST_MAX = bytes("100kb"); + export function createStoryRouter(app: AppOptions) { const redisCacheDuration = app.config.get("jsonp_cache_max_age"); const immutable = app.config.get("jsonp_cache_immutable"); @@ -24,5 +30,8 @@ export function createStoryRouter(app: AppOptions) { router.get("/active.js", activeJSONPHandler(app)); router.get("/ratings.js", ratingsJSONPHandler(app)); + // v2 of count api + router.post("/counts/v2", jsonMiddleware(REQUEST_MAX), countsV2Handler(app)); + return router; } diff --git a/server/src/core/server/graph/schema/schema.graphql b/server/src/core/server/graph/schema/schema.graphql index 4aaa08ecd8..acb3431be2 100644 --- a/server/src/core/server/graph/schema/schema.graphql +++ b/server/src/core/server/graph/schema/schema.graphql @@ -590,6 +590,11 @@ enum FEATURE_FLAG { users, and commentActions more quickly. It is disabled by default. """ DATA_CACHE + + """ + COUNTS_V2 will allow the use of the new /counts/v2 api endpoint to retrieve story comment counts. + """ + COUNTS_V2 } # The moderation mode of the site. diff --git a/server/src/core/server/models/story/story.ts b/server/src/core/server/models/story/story.ts index 3dd47a4e50..8a530a7e0e 100644 --- a/server/src/core/server/models/story/story.ts +++ b/server/src/core/server/models/story/story.ts @@ -22,6 +22,7 @@ import { TenantResource } from "coral-server/models/tenant"; import { dotize } from "coral-server/utils/dotize"; import { + GQLCOMMENT_STATUS, GQLSTORY_MODE, GQLStoryMetadata, GQLStorySettings, @@ -1027,3 +1028,37 @@ export async function markStoryAsUnarchived( return result.value; } + +interface CountResult { + count?: number; +} + +export async function retrieveStoryCommentCounts( + mongo: MongoContext, + tenantID: string, + storyID: string +) { + const cursor = mongo.comments().aggregate([ + { + $match: { + tenantID, + storyID, + status: { $in: [GQLCOMMENT_STATUS.APPROVED, GQLCOMMENT_STATUS.NONE] }, + }, + }, + { $count: "count" }, + ]); + + const hasNext = await cursor.hasNext(); + if (!hasNext) { + return 0; + } + + const next = await cursor.next(); + const result = next as CountResult; + if (!result) { + return 0; + } + + return result.count ?? 0; +} diff --git a/server/src/core/server/stacks/helpers/updateAllCommentCounts.ts b/server/src/core/server/stacks/helpers/updateAllCommentCounts.ts index fca81158da..19cf4f32cb 100644 --- a/server/src/core/server/stacks/helpers/updateAllCommentCounts.ts +++ b/server/src/core/server/stacks/helpers/updateAllCommentCounts.ts @@ -11,10 +11,11 @@ import { CommentTagCounts, updateSharedCommentCounts, } from "coral-server/models/comment"; +import { PUBLISHED_STATUSES } from "coral-server/models/comment/constants"; import { CommentTag } from "coral-server/models/comment/tag"; import { updateSiteCounts } from "coral-server/models/site"; import { updateStoryCounts } from "coral-server/models/story"; -import { Tenant } from "coral-server/models/tenant"; +import { hasFeatureFlag, Tenant } from "coral-server/models/tenant"; import { updateUserCommentCounts } from "coral-server/models/user"; import { calculateCounts, @@ -23,8 +24,10 @@ import { import { I18n } from "coral-server/services/i18n"; import { AugmentedRedis } from "coral-server/services/redis"; +import { COUNTS_V2_CACHE_DURATION } from "coral-common/common/lib/constants"; import { GQLCommentTagCounts, + GQLFEATURE_FLAG, GQLTAG, } from "coral-server/graph/schema/__generated__/types"; @@ -217,6 +220,20 @@ export default async function updateAllCommentCounts( } } } + + if (hasFeatureFlag(tenant, GQLFEATURE_FLAG.COUNTS_V2)) { + if (updatedStory) { + const totalCount = calculateTotalPublishedCommentCount( + updatedStory.commentCounts.status + ); + if (PUBLISHED_STATUSES.includes(input.after.status)) { + const key = `${tenant.id}:${storyID}:count`; + + // set/update the count + await redis.set(key, totalCount, "EX", COUNTS_V2_CACHE_DURATION); + } + } + } } if (options.updateSite) {