feat: highlighted views and queries

.infra/crons.ts

@@ -85,4 +85,8 @@ export const crons: Cron[] = [
       memory: '1Gi',
     },
   },
+  {
+    name: 'update-highlighted-views',
+    schedule: '15 4 * * 0',
+  },
 ];

__tests__/cron/updateHighlightedViews.ts

@@ -0,0 +1,10 @@
+import { crons } from '../../src/cron/index';
+import cron from '../../src/cron/updateHighlightedViews';
+
+describe('updateHighlightedViews cron', () => {
+  it('should be registered', () => {
+    const registeredWorker = crons.find((item) => item.name === cron.name);
+
+    expect(registeredWorker).toBeDefined();
+  });
+});

src/cron/index.ts

@@ -11,6 +11,7 @@ import personalizedDigest from './personalizedDigest';
 import generateSearchInvites from './generateSearchInvites';
 import checkReferralReminder from './checkReferralReminder';
 import dailyDigest from './dailyDigest';
+import updateHighlightedViews from './updateHighlightedViews';
 
 export const crons: Cron[] = [
   updateViews,
@@ -25,4 +26,5 @@ export const crons: Cron[] = [
   generateSearchInvites,
   checkReferralReminder,
   dailyDigest,
+  updateHighlightedViews,
 ];

src/cron/updateHighlightedViews.ts

@@ -0,0 +1,41 @@
+import { Cron } from './cron';
+import { PopularPost } from '../entity/PopularPost';
+import { PopularSource } from '../entity/PopularSource';
+import { PopularTag } from '../entity/PopularTag';
+import { PopularVideoSource } from '../entity/PopularVideoSource';
+import { TrendingPost } from '../entity/TrendingPost';
+import { TrendingSource } from '../entity/TrendingSource';
+import { TrendingTag } from '../entity/TrendingTag';
+import { PopularVideoPost } from '../entity/PopularVideoPost';
+
+const cron: Cron = {
+  name: 'update-highlighted-views',
+  handler: async (con, logger) => {
+    const viewsToRefresh = [
+      TrendingPost,
+      TrendingSource,
+      TrendingTag,
+      PopularPost,
+      PopularSource,
+      PopularTag,
+      PopularVideoPost,
+      PopularVideoSource,
+    ];
+
+    try {
+      await con.transaction(async (manager) => {
+        for (const viewToRefresh of viewsToRefresh) {
+          await manager.query(
+            `REFRESH MATERIALIZED VIEW ${con.getRepository(viewToRefresh).metadata.tableName}`,
+          );
+        }
+      });
+
+      logger.info({}, 'highlighted views updated');
+    } catch (err) {
+      logger.error({ err }, 'failed to update highlighted views');
+    }
+  },
+};
+
+export default cron;

src/entity/PopularPost.ts

@@ -0,0 +1,36 @@
+import { DataSource, ViewColumn, ViewEntity } from 'typeorm';
+import { Post } from './posts';
+
+@ViewEntity({
+  materialized: true,
+  expression: (dataSource: DataSource) =>
+    dataSource
+      .createQueryBuilder()
+      .select('"sourceId"')
+      .addSelect('"tagsStr"')
+      .addSelect('"contentCuration"')
+      .addSelect('"createdAt"')
+      .addSelect('upvotes - downvotes r')
+      .from(Post, 'p')
+      .where(
+        'not p.private and p."createdAt" > now() - interval \'60 day\' and upvotes > downvotes',
+      )
+      .orderBy('r', 'DESC')
+      .limit(1000),
+})
+export class PopularPost {
+  @ViewColumn()
+  sourceId: string;
+
+  @ViewColumn()
+  tagsStr: string;
+
+  @ViewColumn()
+  contentCuration: string;
+
+  @ViewColumn()
+  createdAt: Date;
+
+  @ViewColumn()
+  r: number;
+}

src/entity/PopularSource.ts

@@ -0,0 +1,22 @@
+import { DataSource, ViewColumn, ViewEntity } from 'typeorm';
+import { PopularPost } from './PopularPost';
+
+@ViewEntity({
+  materialized: true,
+  expression: (dataSource: DataSource) =>
+    dataSource
+      .createQueryBuilder()
+      .select('"sourceId"')
+      .addSelect('avg(r) r')
+      .from(PopularPost, 'base')
+      .groupBy('"sourceId"')
+      .having('count(*) > 5')
+      .orderBy('r', 'DESC'),
+})
+export class PopularSource {
+  @ViewColumn()
+  sourceId: string;
+
+  @ViewColumn()
+  r: number;
+}

src/entity/PopularTag.ts

@@ -0,0 +1,22 @@
+import { DataSource, ViewColumn, ViewEntity } from 'typeorm';
+import { PopularPost } from './PopularPost';
+
+@ViewEntity({
+  materialized: true,
+  expression: (dataSource: DataSource) =>
+    dataSource
+      .createQueryBuilder()
+      .select('unnest(string_to_array("tagsStr", \',\')) tag')
+      .addSelect('avg(r) r')
+      .from(PopularPost, 'base')
+      .groupBy('tag')
+      .having('count(*) > 10')
+      .orderBy('r', 'DESC'),
+})
+export class PopularTag {
+  @ViewColumn()
+  tag: string;
+
+  @ViewColumn()
+  r: number;
+}

src/entity/PopularVideoPost.ts

@@ -0,0 +1,36 @@
+import { DataSource, ViewColumn, ViewEntity } from 'typeorm';
+import { Post } from './posts';
+
+@ViewEntity({
+  materialized: true,
+  expression: (dataSource: DataSource) =>
+    dataSource
+      .createQueryBuilder()
+      .select('"sourceId"')
+      .addSelect('"tagsStr"')
+      .addSelect('"contentCuration"')
+      .addSelect('"createdAt"')
+      .addSelect('upvotes - downvotes r')
+      .from(Post, 'p')
+      .where(
+        'not p.private and p."createdAt" > now() - interval \'60 day\' and upvotes > downvotes and "type" = \'video:youtube\'',
+      )
+      .orderBy('r', 'DESC')
+      .limit(1000),
+})
+export class PopularVideoPost {
+  @ViewColumn()
+  sourceId: string;
+
+  @ViewColumn()
+  tagsStr: string;
+
+  @ViewColumn()
+  contentCuration: string;
+
+  @ViewColumn()
+  createdAt: Date;
+
+  @ViewColumn()
+  r: number;
+}

src/entity/PopularVideoSource.ts

@@ -0,0 +1,26 @@
+import { DataSource, ViewColumn, ViewEntity } from 'typeorm';
+import { PopularVideoPost } from './PopularVideoPost';
+
+@ViewEntity({
+  materialized: true,
+  expression: (dataSource: DataSource) =>
+    dataSource
+      .createQueryBuilder()
+      .select('"sourceId"')
+      .addSelect('avg(r) r')
+      .addSelect('count(*) posts')
+      .from(PopularVideoPost, 'base')
+      .groupBy('"sourceId"')
+      .having('count(*) > 5')
+      .orderBy('r', 'DESC'),
+})
+export class PopularVideoSource {
+  @ViewColumn()
+  sourceId: string;
+
+  @ViewColumn()
+  r: number;
+
+  @ViewColumn()
+  posts: number;
+}

src/entity/TrendingPost.ts

@@ -0,0 +1,34 @@
+import { DataSource, ViewColumn, ViewEntity } from 'typeorm';
+import { Post } from './posts';
+
+@ViewEntity({
+  materialized: true,
+  expression: (dataSource: DataSource) =>
+    dataSource
+      .createQueryBuilder()
+      .select('"sourceId"')
+      .addSelect('"tagsStr"')
+      .addSelect('"createdAt"')
+      .addSelect(
+        `log(10, upvotes - downvotes) + extract(epoch from ("createdAt" - now() + interval '7 days')) / 200000 r`,
+      )
+      .from(Post, 'p')
+      .where(
+        `not p.private and p."createdAt" > now() - interval '7 day' and upvotes > downvotes`,
+      )
+      .orderBy('r', 'DESC')
+      .limit(100),
+})
+export class TrendingPost {
+  @ViewColumn()
+  sourceId: string;
+
+  @ViewColumn()
+  tagsStr: string;
+
+  @ViewColumn()
+  createdAt: Date;
+
+  @ViewColumn()
+  r: number;
+}

src/entity/TrendingSource.ts

@@ -0,0 +1,22 @@
+import { DataSource, ViewColumn, ViewEntity } from 'typeorm';
+import { TrendingPost } from './TrendingPost';
+
+@ViewEntity({
+  materialized: true,
+  expression: (dataSource: DataSource) =>
+    dataSource
+      .createQueryBuilder()
+      .select('"sourceId"')
+      .addSelect('avg(r) r')
+      .from(TrendingPost, 'base')
+      .groupBy('"sourceId"')
+      .having('count(*) > 1')
+      .orderBy('r', 'DESC'),
+})
+export class TrendingSource {
+  @ViewColumn()
+  sourceId: string;
+
+  @ViewColumn()
+  r: number;
+}

src/entity/TrendingTag.ts

@@ -0,0 +1,22 @@
+import { DataSource, ViewColumn, ViewEntity } from 'typeorm';
+import { TrendingPost } from './TrendingPost';
+
+@ViewEntity({
+  materialized: true,
+  expression: (dataSource: DataSource) =>
+    dataSource
+      .createQueryBuilder()
+      .select('unnest(string_to_array("tagsStr", \',\')) tag')
+      .addSelect('avg(r) r')
+      .from(TrendingPost, 'base')
+      .groupBy('tag')
+      .having('count(*) > 1')
+      .orderBy('r', 'DESC'),
+})
+export class TrendingTag {
+  @ViewColumn()
+  tag: string;
+
+  @ViewColumn()
+  r: number;
+}

src/migration/1717019094737-HighlightedViews.ts

@@ -0,0 +1,44 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class HighlightedViews1717019094737 implements MigrationInterface {
+    name = 'HighlightedViews1717019094737'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`CREATE MATERIALIZED VIEW "trending_post" AS SELECT "sourceId", "tagsStr", "createdAt", log(10, upvotes - downvotes) + extract(epoch from ("createdAt" - now() + interval '7 days')) / 200000 r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval '7 day' and upvotes > downvotes ORDER BY r DESC LIMIT 100`);
+        await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","trending_post","SELECT \"sourceId\", \"tagsStr\", \"createdAt\", log(10, upvotes - downvotes) + extract(epoch from (\"createdAt\" - now() + interval '7 days')) / 200000 r FROM \"public\".\"post\" \"p\" WHERE not \"p\".\"private\" and p.\"createdAt\" > now() - interval '7 day' and upvotes > downvotes ORDER BY r DESC LIMIT 100"]);
+        await queryRunner.query(`CREATE MATERIALIZED VIEW "trending_tag" AS SELECT unnest(string_to_array("tagsStr", ',')) tag, avg(r) r FROM "public"."trending_post" "base" GROUP BY tag HAVING count(*) > 1 ORDER BY r DESC`);
+        await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","trending_tag","SELECT unnest(string_to_array(\"tagsStr\", ',')) tag, avg(r) r FROM \"public\".\"trending_post\" \"base\" GROUP BY tag HAVING count(*) > 1 ORDER BY r DESC"]);
+        await queryRunner.query(`CREATE MATERIALIZED VIEW "trending_source" AS SELECT "sourceId", avg(r) r FROM "public"."trending_post" "base" GROUP BY "sourceId" HAVING count(*) > 1 ORDER BY r DESC`);
+        await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","trending_source","SELECT \"sourceId\", avg(r) r FROM \"public\".\"trending_post\" \"base\" GROUP BY \"sourceId\" HAVING count(*) > 1 ORDER BY r DESC"]);
+        await queryRunner.query(`CREATE MATERIALIZED VIEW "popular_video_post" AS SELECT "sourceId", "tagsStr", "contentCuration", "createdAt", upvotes - downvotes r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval '60 day' and upvotes > downvotes and "type" = 'video:youtube' ORDER BY r DESC LIMIT 1000`);
+        await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","popular_video_post","SELECT \"sourceId\", \"tagsStr\", \"contentCuration\", \"createdAt\", upvotes - downvotes r FROM \"public\".\"post\" \"p\" WHERE not \"p\".\"private\" and p.\"createdAt\" > now() - interval '60 day' and upvotes > downvotes and \"type\" = 'video:youtube' ORDER BY r DESC LIMIT 1000"]);
+        await queryRunner.query(`CREATE MATERIALIZED VIEW "popular_post" AS SELECT "sourceId", "tagsStr", "contentCuration", "createdAt", upvotes - downvotes r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval '60 day' and upvotes > downvotes ORDER BY r DESC LIMIT 1000`);
+        await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","popular_post","SELECT \"sourceId\", \"tagsStr\", \"contentCuration\", \"createdAt\", upvotes - downvotes r FROM \"public\".\"post\" \"p\" WHERE not \"p\".\"private\" and p.\"createdAt\" > now() - interval '60 day' and upvotes > downvotes ORDER BY r DESC LIMIT 1000"]);
+        await queryRunner.query(`CREATE MATERIALIZED VIEW "popular_tag" AS SELECT unnest(string_to_array("tagsStr", ',')) tag, avg(r) r FROM "public"."popular_post" "base" GROUP BY tag HAVING count(*) > 10 ORDER BY r DESC`);
+        await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","popular_tag","SELECT unnest(string_to_array(\"tagsStr\", ',')) tag, avg(r) r FROM \"public\".\"popular_post\" \"base\" GROUP BY tag HAVING count(*) > 10 ORDER BY r DESC"]);
+        await queryRunner.query(`CREATE MATERIALIZED VIEW "popular_video_source" AS SELECT "sourceId", avg(r) r, count(*) posts FROM "public"."popular_video_post" "base" GROUP BY "sourceId" HAVING count(*) > 5 ORDER BY r DESC`);
+        await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","popular_video_source","SELECT \"sourceId\", avg(r) r, count(*) posts FROM \"public\".\"popular_video_source\" \"base\" GROUP BY \"sourceId\" HAVING count(*) > 5 ORDER BY r DESC"]);
+        await queryRunner.query(`CREATE MATERIALIZED VIEW "popular_source" AS SELECT "sourceId", avg(r) r FROM "public"."popular_post" "base" GROUP BY "sourceId" HAVING count(*) > 5 ORDER BY r DESC`);
+        await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","popular_source","SELECT \"sourceId\", avg(r) r FROM \"public\".\"popular_post\" \"base\" GROUP BY \"sourceId\" HAVING count(*) > 5 ORDER BY r DESC"]);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","popular_source","public"]);
+        await queryRunner.query(`DROP MATERIALIZED VIEW "popular_source"`);
+        await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","popular_video_source","public"]);
+        await queryRunner.query(`DROP MATERIALIZED VIEW "popular_video_source"`);
+        await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","popular_tag","public"]);
+        await queryRunner.query(`DROP MATERIALIZED VIEW "popular_tag"`);
+        await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","popular_post","public"]);
+        await queryRunner.query(`DROP MATERIALIZED VIEW "popular_post"`);
+        await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","popular_video_post","public"]);
+        await queryRunner.query(`DROP MATERIALIZED VIEW "popular_video_post"`);
+        await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","trending_source","public"]);
+        await queryRunner.query(`DROP MATERIALIZED VIEW "trending_source"`);
+        await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","trending_tag","public"]);
+        await queryRunner.query(`DROP MATERIALIZED VIEW "trending_tag"`);
+        await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","trending_post","public"]);
+        await queryRunner.query(`DROP MATERIALIZED VIEW "trending_post"`);
+    }
+
+}