Merge pull request #4663 from coralproject/develop

v9.3.0

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@coralproject/talk",
-  "version": "9.2.0",
+  "version": "9.3.0",
   "author": "The Coral Project",
   "homepage": "https://coralproject.net/",
   "sideEffects": [

common/lib/constants.ts

@@ -86,6 +86,12 @@ export const DEFAULT_SESSION_DURATION = 90 * TIME.DAY;
  */
 export const COMMENT_REPEAT_POST_DURATION = 6 * TIME.MINUTE;
 
+/**
+ * COUNTS_V2_CACHE_DURATION is the length of time in seconds that the comment
+ * counts for a story are cached for the counts v2 endoint.
+ */
+export const COUNTS_V2_CACHE_DURATION = 1 * TIME.DAY;
+
 /**
  * SPOILER_CLASSNAME is the classname that is attached to spoilers.
  */

common/package.json

@@ -1,6 +1,6 @@
 {
   "name": "common",
-  "version": "9.2.0",
+  "version": "9.3.0",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

config/package.json

@@ -1,6 +1,6 @@
 {
   "name": "common",
-  "version": "9.2.0",
+  "version": "9.3.0",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@coralproject/talk",
-  "version": "9.2.0",
+  "version": "9.3.0",
   "author": "The Coral Project",
   "homepage": "https://coralproject.net/",
   "sideEffects": [

server/src/core/server/app/handlers/api/story/count.ts

@@ -1,22 +1,35 @@
 import Joi from "joi";
 
+import { COUNTS_V2_CACHE_DURATION } from "coral-common/common/lib/constants";
 import { CountJSONPData } from "coral-common/common/lib/types/count";
 import { AppOptions } from "coral-server/app";
 import { validate } from "coral-server/app/request/body";
 import { MongoContext } from "coral-server/data/context";
+import logger from "coral-server/logger";
 import { retrieveManyStoryRatings } from "coral-server/models/comment";
 import { PUBLISHED_STATUSES } from "coral-server/models/comment/constants";
-import { Story } from "coral-server/models/story";
-import { Tenant } from "coral-server/models/tenant";
+import { retrieveStoryCommentCounts, Story } from "coral-server/models/story";
+import { hasFeatureFlag, Tenant } from "coral-server/models/tenant";
 import { I18n, translate } from "coral-server/services/i18n";
 import { find } from "coral-server/services/stories";
 import { RequestHandler, TenantCoralRequest } from "coral-server/types/express";
 
-import { GQLSTORY_MODE } from "coral-server/graph/schema/__generated__/types";
+import {
+  GQLFEATURE_FLAG,
+  GQLSTORY_MODE,
+} from "coral-server/graph/schema/__generated__/types";
 
 const NUMBER_CLASS_NAME = "coral-count-number";
 const TEXT_CLASS_NAME = "coral-count-text";
 
+interface CountsV2Body {
+  storyIDs: string[];
+}
+
+const CountsV2BodySchema = Joi.object().keys({
+  storyIDs: Joi.array().items(Joi.string().required()).required().max(100),
+});
+
 export type JSONPCountOptions = Pick<
   AppOptions,
   "mongo" | "tenantCache" | "i18n"
@@ -171,3 +184,103 @@ async function calculateStoryCount(
     0
   );
 }
+
+export type CountV2Options = Pick<AppOptions, "mongo" | "redis">;
+
+interface CountResult {
+  storyID: string;
+  redisCount?: number; // set if count came from redis
+  mongoCount?: number; // set if count came from mongo
+  count: number; // always set, can come from mongo or redis
+}
+
+export const computeCountKey = (tenantID: string, storyID: string) => {
+  return `${tenantID}:${storyID}:count`;
+};
+
+export const countsV2Handler =
+  ({ mongo, redis }: CountV2Options): RequestHandler<TenantCoralRequest> =>
+  async (req, res, next) => {
+    const { tenant } = req.coral;
+
+    try {
+      if (!hasFeatureFlag(tenant, GQLFEATURE_FLAG.COUNTS_V2)) {
+        return res.status(400).send("Counts V2 api not enabled");
+      }
+
+      const { storyIDs }: CountsV2Body = validate(CountsV2BodySchema, req.body);
+
+      // grab what keys we can that are already in Redis with one bulk call
+      const redisCounts = await redis.mget(
+        ...storyIDs.map((id) => computeCountKey(tenant.id, id))
+      );
+
+      // quickly iterate over our results and see if we're missing any of the
+      // values for our requested story id's
+      const countResults = new Map<string, CountResult>();
+      const missingIDs: string[] = [];
+      for (let i = 0; i < storyIDs.length; i++) {
+        const storyID = storyIDs[i];
+        const redisCount = redisCounts[i];
+
+        if (redisCount !== null && redisCount !== undefined) {
+          try {
+            const count = parseInt(redisCount, 10);
+            countResults.set(storyID, { storyID, redisCount: count, count });
+          } catch {
+            missingIDs.push(storyID);
+          }
+        } else {
+          missingIDs.push(storyID);
+        }
+      }
+
+      // compute out the counts for any story id's we couldn't
+      // get a count from Redis
+      for (const missingID of missingIDs) {
+        const count = await retrieveStoryCommentCounts(
+          mongo,
+          tenant.id,
+          missingID
+        );
+
+        const key = computeCountKey(tenant.id, missingID);
+        await redis.set(key, count, "EX", COUNTS_V2_CACHE_DURATION);
+        logger.debug("set story count for counts v2 in redis cache", {
+          storyID: missingID,
+          count,
+        });
+
+        countResults.set(missingID, {
+          storyID: missingID,
+          mongoCount: count,
+          count,
+        });
+      }
+
+      // strictly follow the result set based on the story id's
+      // we were given from the caller. Then return the values
+      // from our combined redis/mongo results.
+      //
+      // this means if a user asked for id's ["1", "2", "3", "does-not-exist", "1"], they would
+      // receive counts like so:
+      // [
+      //   { storyID: 1, redisCount: 2, count: 2 },
+      //   { storyID: 2, redisCount: 5, count: 5 },
+      //   { storyID: 3, mongoCount: 2, count: 2 },
+      //   null,
+      //   { storyID: 1, redisCount: 2, count: 2 }
+      // ]
+      //
+      // many of our counts endpoints appreciate this adherence.
+      const results: Array<CountResult | null> = [];
+      for (const storyID of storyIDs) {
+        const value = countResults.get(storyID) ?? null;
+        results.push(value);
+      }
+
+      res.send(JSON.stringify(results));
+    } catch (err) {
+      return next(err);
+    }
+  };

server/src/core/server/app/router/api/story.ts

@@ -1,14 +1,20 @@
+import bytes from "bytes";
+
 import { AppOptions } from "coral-server/app";
 import {
   activeJSONPHandler,
   countHandler,
   countJSONPHandler,
+  countsV2Handler,
   ratingsJSONPHandler,
 } from "coral-server/app/handlers";
+import { jsonMiddleware } from "coral-server/app/middleware";
 import cacheMiddleware from "coral-server/app/middleware/cache";
 
 import { createAPIRouter } from "./helpers";
 
+const REQUEST_MAX = bytes("100kb");
+
 export function createStoryRouter(app: AppOptions) {
   const redisCacheDuration = app.config.get("jsonp_cache_max_age");
   const immutable = app.config.get("jsonp_cache_immutable");
@@ -24,5 +30,8 @@ export function createStoryRouter(app: AppOptions) {
   router.get("/active.js", activeJSONPHandler(app));
   router.get("/ratings.js", ratingsJSONPHandler(app));
 
+  // v2 of count api
+  router.post("/counts/v2", jsonMiddleware(REQUEST_MAX), countsV2Handler(app));
+
   return router;
 }

server/src/core/server/graph/schema/schema.graphql

@@ -590,6 +590,11 @@ enum FEATURE_FLAG {
   users, and commentActions more quickly. It is disabled by default.
   """
   DATA_CACHE
+
+  """
+  COUNTS_V2 will allow the use of the new /counts/v2 api endpoint to retrieve story comment counts.
+  """
+  COUNTS_V2
 }
 
 # The moderation mode of the site.

server/src/core/server/models/story/story.ts

@@ -22,6 +22,7 @@ import { TenantResource } from "coral-server/models/tenant";
 import { dotize } from "coral-server/utils/dotize";
 
 import {
+  GQLCOMMENT_STATUS,
   GQLSTORY_MODE,
   GQLStoryMetadata,
   GQLStorySettings,
@@ -1027,3 +1028,37 @@ export async function markStoryAsUnarchived(
 
   return result.value;
 }
+
+interface CountResult {
+  count?: number;
+}
+
+export async function retrieveStoryCommentCounts(
+  mongo: MongoContext,
+  tenantID: string,
+  storyID: string
+) {
+  const cursor = mongo.comments().aggregate([
+    {
+      $match: {
+        tenantID,
+        storyID,
+        status: { $in: [GQLCOMMENT_STATUS.APPROVED, GQLCOMMENT_STATUS.NONE] },
+      },
+    },
+    { $count: "count" },
+  ]);
+
+  const hasNext = await cursor.hasNext();
+  if (!hasNext) {
+    return 0;
+  }
+
+  const next = await cursor.next();
+  const result = next as CountResult;
+  if (!result) {
+    return 0;
+  }
+
+  return result.count ?? 0;
+}

server/src/core/server/stacks/helpers/updateAllCommentCounts.ts

@@ -11,10 +11,11 @@ import {
   CommentTagCounts,
   updateSharedCommentCounts,
 } from "coral-server/models/comment";
+import { PUBLISHED_STATUSES } from "coral-server/models/comment/constants";
 import { CommentTag } from "coral-server/models/comment/tag";
 import { updateSiteCounts } from "coral-server/models/site";
 import { updateStoryCounts } from "coral-server/models/story";
-import { Tenant } from "coral-server/models/tenant";
+import { hasFeatureFlag, Tenant } from "coral-server/models/tenant";
 import { updateUserCommentCounts } from "coral-server/models/user";
 import {
   calculateCounts,
@@ -23,8 +24,10 @@ import {
 import { I18n } from "coral-server/services/i18n";
 import { AugmentedRedis } from "coral-server/services/redis";
 
+import { COUNTS_V2_CACHE_DURATION } from "coral-common/common/lib/constants";
 import {
   GQLCommentTagCounts,
+  GQLFEATURE_FLAG,
   GQLTAG,
 } from "coral-server/graph/schema/__generated__/types";
 
@@ -217,6 +220,20 @@ export default async function updateAllCommentCounts(
         }
       }
     }
+
+    if (hasFeatureFlag(tenant, GQLFEATURE_FLAG.COUNTS_V2)) {
+      if (updatedStory) {
+        const totalCount = calculateTotalPublishedCommentCount(
+          updatedStory.commentCounts.status
+        );
+        if (PUBLISHED_STATUSES.includes(input.after.status)) {
+          const key = `${tenant.id}:${storyID}:count`;
+
+          // set/update the count
+          await redis.set(key, totalCount, "EX", COUNTS_V2_CACHE_DURATION);
+        }
+      }
+    }
   }
 
   if (options.updateSite) {