From 254c147eabedc80c6b9d95191f3074c37adc3892 Mon Sep 17 00:00:00 2001 From: capJavert Date: Wed, 29 May 2024 17:18:31 +0200 Subject: [PATCH 1/7] feat: add trending views --- src/entity/TrendingPost.ts | 34 ++++++++++++++++++++++++++++++++++ src/entity/TrendingSource.ts | 22 ++++++++++++++++++++++ src/entity/TrendingTag.ts | 22 ++++++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 src/entity/TrendingPost.ts create mode 100644 src/entity/TrendingSource.ts create mode 100644 src/entity/TrendingTag.ts diff --git a/src/entity/TrendingPost.ts b/src/entity/TrendingPost.ts new file mode 100644 index 000000000..c97157b9a --- /dev/null +++ b/src/entity/TrendingPost.ts @@ -0,0 +1,34 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; +import { Post } from './posts'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select(`s.id as sourceId`) + .addSelect('tagsStr') + .addSelect('createdAt') + .addSelect( + `log(10, upvotes - downvotes) + extract(epoch from ("createdAt" - now() + interval '7 days')) / 200000 r`, + ) + .from(Post, 'p') + .where( + `not p.private and p."createdAt" > now() - interval '7 day' and upvotes > downvotes`, + ) + .orderBy('r', 'DESC') + .limit(100), +}) +export class TrendingPost { + @ViewColumn() + sourceId: string; + + @ViewColumn() + tagsStr: string; + + @ViewColumn() + createdAt: Date; + + @ViewColumn() + r: number; +} diff --git a/src/entity/TrendingSource.ts b/src/entity/TrendingSource.ts new file mode 100644 index 000000000..3edff94cd --- /dev/null +++ b/src/entity/TrendingSource.ts @@ -0,0 +1,22 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; +import { TrendingPost } from './TrendingPost'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select('sourceId') + .addSelect('avg(r) r') + .from(TrendingPost, 'tp') + .groupBy('sourceId') + .having('count(*) > 1') + .orderBy('r', 'DESC'), +}) +export class TrendingSource { + @ViewColumn() + sourceId: string; + + @ViewColumn() + r: number; +} diff --git a/src/entity/TrendingTag.ts b/src/entity/TrendingTag.ts new file mode 100644 index 000000000..063a2a518 --- /dev/null +++ b/src/entity/TrendingTag.ts @@ -0,0 +1,22 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; +import { TrendingPost } from './TrendingPost'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select('unnest(string_to_array("tagsStr", \',\')) tag') + .addSelect('avg(r) r') + .from(TrendingPost, 'tp') + .groupBy('tag') + .having('count(*) > 1') + .orderBy('r', 'DESC'), +}) +export class TrendingTag { + @ViewColumn() + tag: string; + + @ViewColumn() + r: number; +} From 5a6d500cbbd7f9b306692c09c5722b83aec7f9ce Mon Sep 17 00:00:00 2001 From: capJavert Date: Wed, 29 May 2024 17:18:48 +0200 Subject: [PATCH 2/7] feat: add popular base view --- src/entity/PopularPost.ts | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/entity/PopularPost.ts diff --git a/src/entity/PopularPost.ts b/src/entity/PopularPost.ts new file mode 100644 index 000000000..e31b2f238 --- /dev/null +++ b/src/entity/PopularPost.ts @@ -0,0 +1,36 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; +import { Post } from './posts'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select('sourceId') + .addSelect('tagsStr') + .addSelect('contentCuration') + .addSelect('createdAt') + .addSelect('upvotes - downvotes r') + .from(Post, 'p') + .where( + 'not p.private and p."createdAt" > now() - interval \'60 day\' and upvotes > downvotes', + ) + .orderBy('r', 'DESC') + .limit(1000), +}) +export class PopularPost { + @ViewColumn() + sourceId: string; + + @ViewColumn() + tagsStr: string; + + @ViewColumn() + contentCuration: string; + + @ViewColumn() + createdAt: Date; + + @ViewColumn() + r: number; +} From aeefd61399f463abc941d4fd3c9e99f1356c77c4 Mon Sep 17 00:00:00 2001 From: capJavert Date: Wed, 29 May 2024 23:51:21 +0200 Subject: [PATCH 3/7] feat: add popular views and migration --- src/entity/PopularPost.ts | 8 ++-- src/entity/PopularSource.ts | 22 ++++++++++ src/entity/PopularTag.ts | 22 ++++++++++ src/entity/PopularVideoPost.ts | 36 +++++++++++++++ src/entity/PopularVideoSource.ts | 26 +++++++++++ src/entity/TrendingPost.ts | 6 +-- src/entity/TrendingSource.ts | 6 +-- src/entity/TrendingTag.ts | 2 +- .../1717019094737-HighlightedViews.ts | 44 +++++++++++++++++++ 9 files changed, 161 insertions(+), 11 deletions(-) create mode 100644 src/entity/PopularSource.ts create mode 100644 src/entity/PopularTag.ts create mode 100644 src/entity/PopularVideoPost.ts create mode 100644 src/entity/PopularVideoSource.ts create mode 100644 src/migration/1717019094737-HighlightedViews.ts diff --git a/src/entity/PopularPost.ts b/src/entity/PopularPost.ts index e31b2f238..c1e5a76e9 100644 --- a/src/entity/PopularPost.ts +++ b/src/entity/PopularPost.ts @@ -6,10 +6,10 @@ import { Post } from './posts'; expression: (dataSource: DataSource) => dataSource .createQueryBuilder() - .select('sourceId') - .addSelect('tagsStr') - .addSelect('contentCuration') - .addSelect('createdAt') + .select('"sourceId"') + .addSelect('"tagsStr"') + .addSelect('"contentCuration"') + .addSelect('"createdAt"') .addSelect('upvotes - downvotes r') .from(Post, 'p') .where( diff --git a/src/entity/PopularSource.ts b/src/entity/PopularSource.ts new file mode 100644 index 000000000..7aa8a6438 --- /dev/null +++ b/src/entity/PopularSource.ts @@ -0,0 +1,22 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; +import { PopularPost } from './PopularPost'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select('"sourceId"') + .addSelect('avg(r) r') + .from(PopularPost, 'base') + .groupBy('"sourceId"') + .having('count(*) > 5') + .orderBy('r', 'DESC'), +}) +export class PopularSource { + @ViewColumn() + sourceId: string; + + @ViewColumn() + r: number; +} diff --git a/src/entity/PopularTag.ts b/src/entity/PopularTag.ts new file mode 100644 index 000000000..bc96133cf --- /dev/null +++ b/src/entity/PopularTag.ts @@ -0,0 +1,22 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; +import { PopularPost } from './PopularPost'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select('unnest(string_to_array("tagsStr", \',\')) tag') + .addSelect('avg(r) r') + .from(PopularPost, 'base') + .groupBy('tag') + .having('count(*) > 10') + .orderBy('r', 'DESC'), +}) +export class PopularTag { + @ViewColumn() + tag: string; + + @ViewColumn() + r: number; +} diff --git a/src/entity/PopularVideoPost.ts b/src/entity/PopularVideoPost.ts new file mode 100644 index 000000000..137f6cb96 --- /dev/null +++ b/src/entity/PopularVideoPost.ts @@ -0,0 +1,36 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; +import { Post } from './posts'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select('"sourceId"') + .addSelect('"tagsStr"') + .addSelect('"contentCuration"') + .addSelect('"createdAt"') + .addSelect('upvotes - downvotes r') + .from(Post, 'p') + .where( + 'not p.private and p."createdAt" > now() - interval \'60 day\' and upvotes > downvotes and "type" = \'video:youtube\'', + ) + .orderBy('r', 'DESC') + .limit(1000), +}) +export class PopularVideoPost { + @ViewColumn() + sourceId: string; + + @ViewColumn() + tagsStr: string; + + @ViewColumn() + contentCuration: string; + + @ViewColumn() + createdAt: Date; + + @ViewColumn() + r: number; +} diff --git a/src/entity/PopularVideoSource.ts b/src/entity/PopularVideoSource.ts new file mode 100644 index 000000000..bf3f136af --- /dev/null +++ b/src/entity/PopularVideoSource.ts @@ -0,0 +1,26 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; +import { PopularVideoPost } from './PopularVideoPost'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select('"sourceId"') + .addSelect('avg(r) r') + .addSelect('count(*) posts') + .from(PopularVideoPost, 'base') + .groupBy('"sourceId"') + .having('count(*) > 5') + .orderBy('r', 'DESC'), +}) +export class PopularVideoSource { + @ViewColumn() + sourceId: string; + + @ViewColumn() + r: number; + + @ViewColumn() + posts: number; +} diff --git a/src/entity/TrendingPost.ts b/src/entity/TrendingPost.ts index c97157b9a..349facae7 100644 --- a/src/entity/TrendingPost.ts +++ b/src/entity/TrendingPost.ts @@ -6,9 +6,9 @@ import { Post } from './posts'; expression: (dataSource: DataSource) => dataSource .createQueryBuilder() - .select(`s.id as sourceId`) - .addSelect('tagsStr') - .addSelect('createdAt') + .select('"sourceId"') + .addSelect('"tagsStr"') + .addSelect('"createdAt"') .addSelect( `log(10, upvotes - downvotes) + extract(epoch from ("createdAt" - now() + interval '7 days')) / 200000 r`, ) diff --git a/src/entity/TrendingSource.ts b/src/entity/TrendingSource.ts index 3edff94cd..0ecbcb510 100644 --- a/src/entity/TrendingSource.ts +++ b/src/entity/TrendingSource.ts @@ -6,10 +6,10 @@ import { TrendingPost } from './TrendingPost'; expression: (dataSource: DataSource) => dataSource .createQueryBuilder() - .select('sourceId') + .select('"sourceId"') .addSelect('avg(r) r') - .from(TrendingPost, 'tp') - .groupBy('sourceId') + .from(TrendingPost, 'base') + .groupBy('"sourceId"') .having('count(*) > 1') .orderBy('r', 'DESC'), }) diff --git a/src/entity/TrendingTag.ts b/src/entity/TrendingTag.ts index 063a2a518..0e0d80444 100644 --- a/src/entity/TrendingTag.ts +++ b/src/entity/TrendingTag.ts @@ -8,7 +8,7 @@ import { TrendingPost } from './TrendingPost'; .createQueryBuilder() .select('unnest(string_to_array("tagsStr", \',\')) tag') .addSelect('avg(r) r') - .from(TrendingPost, 'tp') + .from(TrendingPost, 'base') .groupBy('tag') .having('count(*) > 1') .orderBy('r', 'DESC'), diff --git a/src/migration/1717019094737-HighlightedViews.ts b/src/migration/1717019094737-HighlightedViews.ts new file mode 100644 index 000000000..f53b4dca3 --- /dev/null +++ b/src/migration/1717019094737-HighlightedViews.ts @@ -0,0 +1,44 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class HighlightedViews1717019094737 implements MigrationInterface { + name = 'HighlightedViews1717019094737' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE MATERIALIZED VIEW "trending_post" AS SELECT "sourceId", "tagsStr", "createdAt", log(10, upvotes - downvotes) + extract(epoch from ("createdAt" - now() + interval '7 days')) / 200000 r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval '7 day' and upvotes > downvotes ORDER BY r DESC LIMIT 100`); + await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","trending_post","SELECT \"sourceId\", \"tagsStr\", \"createdAt\", log(10, upvotes - downvotes) + extract(epoch from (\"createdAt\" - now() + interval '7 days')) / 200000 r FROM \"public\".\"post\" \"p\" WHERE not \"p\".\"private\" and p.\"createdAt\" > now() - interval '7 day' and upvotes > downvotes ORDER BY r DESC LIMIT 100"]); + await queryRunner.query(`CREATE MATERIALIZED VIEW "trending_tag" AS SELECT unnest(string_to_array("tagsStr", ',')) tag, avg(r) r FROM "public"."trending_post" "base" GROUP BY tag HAVING count(*) > 1 ORDER BY r DESC`); + await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","trending_tag","SELECT unnest(string_to_array(\"tagsStr\", ',')) tag, avg(r) r FROM \"public\".\"trending_post\" \"base\" GROUP BY tag HAVING count(*) > 1 ORDER BY r DESC"]); + await queryRunner.query(`CREATE MATERIALIZED VIEW "trending_source" AS SELECT "sourceId", avg(r) r FROM "public"."trending_post" "base" GROUP BY "sourceId" HAVING count(*) > 1 ORDER BY r DESC`); + await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","trending_source","SELECT \"sourceId\", avg(r) r FROM \"public\".\"trending_post\" \"base\" GROUP BY \"sourceId\" HAVING count(*) > 1 ORDER BY r DESC"]); + await queryRunner.query(`CREATE MATERIALIZED VIEW "popular_video_post" AS SELECT "sourceId", "tagsStr", "contentCuration", "createdAt", upvotes - downvotes r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval '60 day' and upvotes > downvotes and "type" = 'video:youtube' ORDER BY r DESC LIMIT 1000`); + await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","popular_video_post","SELECT \"sourceId\", \"tagsStr\", \"contentCuration\", \"createdAt\", upvotes - downvotes r FROM \"public\".\"post\" \"p\" WHERE not \"p\".\"private\" and p.\"createdAt\" > now() - interval '60 day' and upvotes > downvotes and \"type\" = 'video:youtube' ORDER BY r DESC LIMIT 1000"]); + await queryRunner.query(`CREATE MATERIALIZED VIEW "popular_post" AS SELECT "sourceId", "tagsStr", "contentCuration", "createdAt", upvotes - downvotes r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval '60 day' and upvotes > downvotes ORDER BY r DESC LIMIT 1000`); + await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","popular_post","SELECT \"sourceId\", \"tagsStr\", \"contentCuration\", \"createdAt\", upvotes - downvotes r FROM \"public\".\"post\" \"p\" WHERE not \"p\".\"private\" and p.\"createdAt\" > now() - interval '60 day' and upvotes > downvotes ORDER BY r DESC LIMIT 1000"]); + await queryRunner.query(`CREATE MATERIALIZED VIEW "popular_tag" AS SELECT unnest(string_to_array("tagsStr", ',')) tag, avg(r) r FROM "public"."popular_post" "base" GROUP BY tag HAVING count(*) > 10 ORDER BY r DESC`); + await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","popular_tag","SELECT unnest(string_to_array(\"tagsStr\", ',')) tag, avg(r) r FROM \"public\".\"popular_post\" \"base\" GROUP BY tag HAVING count(*) > 10 ORDER BY r DESC"]); + await queryRunner.query(`CREATE MATERIALIZED VIEW "popular_video_source" AS SELECT "sourceId", avg(r) r, count(*) posts FROM "public"."popular_video_post" "base" GROUP BY "sourceId" HAVING count(*) > 5 ORDER BY r DESC`); + await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","popular_video_source","SELECT \"sourceId\", avg(r) r, count(*) posts FROM \"public\".\"popular_video_source\" \"base\" GROUP BY \"sourceId\" HAVING count(*) > 5 ORDER BY r DESC"]); + await queryRunner.query(`CREATE MATERIALIZED VIEW "popular_source" AS SELECT "sourceId", avg(r) r FROM "public"."popular_post" "base" GROUP BY "sourceId" HAVING count(*) > 5 ORDER BY r DESC`); + await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","popular_source","SELECT \"sourceId\", avg(r) r FROM \"public\".\"popular_post\" \"base\" GROUP BY \"sourceId\" HAVING count(*) > 5 ORDER BY r DESC"]); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","popular_source","public"]); + await queryRunner.query(`DROP MATERIALIZED VIEW "popular_source"`); + await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","popular_video_source","public"]); + await queryRunner.query(`DROP MATERIALIZED VIEW "popular_video_source"`); + await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","popular_tag","public"]); + await queryRunner.query(`DROP MATERIALIZED VIEW "popular_tag"`); + await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","popular_post","public"]); + await queryRunner.query(`DROP MATERIALIZED VIEW "popular_post"`); + await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","popular_video_post","public"]); + await queryRunner.query(`DROP MATERIALIZED VIEW "popular_video_post"`); + await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","trending_source","public"]); + await queryRunner.query(`DROP MATERIALIZED VIEW "trending_source"`); + await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","trending_tag","public"]); + await queryRunner.query(`DROP MATERIALIZED VIEW "trending_tag"`); + await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","trending_post","public"]); + await queryRunner.query(`DROP MATERIALIZED VIEW "trending_post"`); + } + +} From 7012871f01f02b94c1a5d7acb1b614b9595d9fb6 Mon Sep 17 00:00:00 2001 From: capJavert Date: Wed, 29 May 2024 23:54:29 +0200 Subject: [PATCH 4/7] feat: cron to refresh views --- .infra/crons.ts | 4 +++ __tests__/cron/updateHighlightedViews.ts | 10 ++++++ src/cron/index.ts | 2 ++ src/cron/updateHighlightedViews.ts | 41 ++++++++++++++++++++++++ 4 files changed, 57 insertions(+) create mode 100644 __tests__/cron/updateHighlightedViews.ts create mode 100644 src/cron/updateHighlightedViews.ts diff --git a/.infra/crons.ts b/.infra/crons.ts index 33e8fe98f..bed0a5616 100644 --- a/.infra/crons.ts +++ b/.infra/crons.ts @@ -85,4 +85,8 @@ export const crons: Cron[] = [ memory: '1Gi', }, }, + { + name: 'update-highlighted-views', + schedule: '15 4 * * 0', + }, ]; diff --git a/__tests__/cron/updateHighlightedViews.ts b/__tests__/cron/updateHighlightedViews.ts new file mode 100644 index 000000000..d14281ef6 --- /dev/null +++ b/__tests__/cron/updateHighlightedViews.ts @@ -0,0 +1,10 @@ +import { crons } from '../../src/cron/index'; +import cron from '../../src/cron/updateHighlightedViews'; + +describe('updateHighlightedViews cron', () => { + it('should be registered', () => { + const registeredWorker = crons.find((item) => item.name === cron.name); + + expect(registeredWorker).toBeDefined(); + }); +}); diff --git a/src/cron/index.ts b/src/cron/index.ts index 2b90fca3e..a9bebd94c 100644 --- a/src/cron/index.ts +++ b/src/cron/index.ts @@ -11,6 +11,7 @@ import personalizedDigest from './personalizedDigest'; import generateSearchInvites from './generateSearchInvites'; import checkReferralReminder from './checkReferralReminder'; import dailyDigest from './dailyDigest'; +import updateHighlightedViews from './updateHighlightedViews'; export const crons: Cron[] = [ updateViews, @@ -25,4 +26,5 @@ export const crons: Cron[] = [ generateSearchInvites, checkReferralReminder, dailyDigest, + updateHighlightedViews, ]; diff --git a/src/cron/updateHighlightedViews.ts b/src/cron/updateHighlightedViews.ts new file mode 100644 index 000000000..9ceb71bb6 --- /dev/null +++ b/src/cron/updateHighlightedViews.ts @@ -0,0 +1,41 @@ +import { Cron } from './cron'; +import { PopularPost } from '../entity/PopularPost'; +import { PopularSource } from '../entity/PopularSource'; +import { PopularTag } from '../entity/PopularTag'; +import { PopularVideoSource } from '../entity/PopularVideoSource'; +import { TrendingPost } from '../entity/TrendingPost'; +import { TrendingSource } from '../entity/TrendingSource'; +import { TrendingTag } from '../entity/TrendingTag'; +import { PopularVideoPost } from '../entity/PopularVideoPost'; + +const cron: Cron = { + name: 'update-highlighted-views', + handler: async (con, logger) => { + const viewsToRefresh = [ + TrendingPost, + TrendingSource, + TrendingTag, + PopularPost, + PopularSource, + PopularTag, + PopularVideoPost, + PopularVideoSource, + ]; + + try { + await con.transaction(async (manager) => { + for (const viewToRefresh of viewsToRefresh) { + await manager.query( + `REFRESH MATERIALIZED VIEW ${con.getRepository(viewToRefresh).metadata.tableName}`, + ); + } + }); + + logger.info({}, 'highlighted views updated'); + } catch (err) { + logger.error({ err }, 'failed to update highlighted views'); + } + }, +}; + +export default cron; From ecae856ebd3c3e10057801dd28eb3e974c27880f Mon Sep 17 00:00:00 2001 From: Chris Bongers Date: Thu, 30 May 2024 18:49:33 +0700 Subject: [PATCH 5/7] fix: added queries, test and cleanup unused props --- __tests__/__snapshots__/tags.ts.snap | 22 --- __tests__/sources.ts | 178 +++++++++++++++++ __tests__/tags.ts | 88 +++++++-- src/entity/PopularPost.ts | 4 - src/entity/PopularVideoPost.ts | 4 - .../1717019094737-HighlightedViews.ts | 181 ++++++++++++++---- src/schema/sources.ts | 98 ++++++++++ src/schema/tags.ts | 38 ++-- 8 files changed, 501 insertions(+), 112 deletions(-) diff --git a/__tests__/__snapshots__/tags.ts.snap b/__tests__/__snapshots__/tags.ts.snap index 6fd149406..0fe45f1a2 100644 --- a/__tests__/__snapshots__/tags.ts.snap +++ b/__tests__/__snapshots__/tags.ts.snap @@ -1,27 +1,5 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`query popularTags should return most popular tags ordered by value 1`] = ` -Object { - "popularTags": Array [ - Object { - "name": "development", - }, - Object { - "name": "fullstack", - }, - Object { - "name": "golang", - }, - Object { - "name": "rust", - }, - Object { - "name": "webdev", - }, - ], -} -`; - exports[`query searchTags should search for tags and order by value 1`] = ` Object { "searchTags": Object { diff --git a/__tests__/sources.ts b/__tests__/sources.ts index 4753d4acf..35a1cf074 100644 --- a/__tests__/sources.ts +++ b/__tests__/sources.ts @@ -14,6 +14,7 @@ import { NotificationPreferenceSource, Post, PostKeyword, + PostType, SharePost, Source, SourceFeed, @@ -232,6 +233,183 @@ describe('query sources', () => { }); }); +describe('query mostRecentSources', () => { + const QUERY = ` + query MostRecentSources { + mostRecentSources { + id + name + image + public + } + } + `; + + it('should return most recent sources', async () => { + const res = await client.query(QUERY); + expect(res.errors).toBeFalsy(); + expect(res.data).toMatchObject({ + mostRecentSources: [ + { id: 'a', name: 'A', image: 'http://image.com/a', public: true }, + { id: 'b', name: 'B', image: 'http://image.com/b', public: true }, + ], + }); + }); +}); + +describe('query trendingSources', () => { + const QUERY = ` + query TrendingSources { + trendingSources { + id + name + image + public + } + } + `; + + it('should return most trending sources', async () => { + await con.getRepository(Post).save( + new Array(5).fill('a').map((item, index) => { + return { + id: `post_${index}`, + shortId: `post_${index}`, + title: `Post ${index}`, + tagsStr: 'tag1', + upvotes: 10 + index, + createdAt: new Date(), + sourceId: 'a', + }; + }), + ); + await con.getRepository(Post).save({ + id: `post_6`, + shortId: `post_6`, + title: `Post 6`, + tagsStr: 'tag1', + upvotes: 10, + createdAt: new Date(), + sourceId: 'b', + }); + await con.query(`REFRESH MATERIALIZED VIEW trending_post`); + await con.query(`REFRESH MATERIALIZED VIEW trending_source`); + + const res = await client.query(QUERY); + expect(res.errors).toBeFalsy(); + expect(res.data).toMatchObject({ + trendingSources: [ + { id: 'a', name: 'A', image: 'http://image.com/a', public: null }, + ], + }); + }); +}); + +describe('query popularSources', () => { + const QUERY = ` + query PopularSources { + popularSources { + id + name + image + public + } + } + `; + + it('should return most popular sources', async () => { + await con.getRepository(Post).save( + new Array(6).fill('a').map((item, index) => { + return { + id: `post_${index}`, + shortId: `post_${index}`, + title: `Post ${index}`, + tagsStr: 'tag1', + upvotes: 10 + index, + createdAt: new Date(), + sourceId: 'a', + }; + }), + ); + await con.getRepository(Post).save( + new Array(5).fill('b').map((item, index) => { + return { + id: `post_${index}`, + shortId: `post_${index}`, + title: `Post ${index}`, + tagsStr: 'tag1', + upvotes: 10 + index, + createdAt: new Date(), + sourceId: 'a', + }; + }), + ); + await con.query(`REFRESH MATERIALIZED VIEW popular_post`); + await con.query(`REFRESH MATERIALIZED VIEW popular_source`); + + const res = await client.query(QUERY); + expect(res.errors).toBeFalsy(); + expect(res.data).toMatchObject({ + popularSources: [ + { id: 'a', name: 'A', image: 'http://image.com/a', public: null }, + ], + }); + }); +}); + +describe('query topVideoSources', () => { + const QUERY = ` + query TopVideoSources { + topVideoSources { + id + name + image + public + } + } + `; + + it('should return top video sources', async () => { + await con.getRepository(Post).save( + new Array(6).fill('a').map((item, index) => { + return { + id: `post_a_${index}`, + shortId: `post_a_${index}`, + title: `Post ${index}`, + tagsStr: 'tag1', + upvotes: 10 + index, + createdAt: new Date(), + sourceId: 'a', + type: PostType.VideoYouTube, + }; + }), + ); + await con.getRepository(Post).save( + new Array(6).fill('b').map((item, index) => { + return { + id: `post_b_${index}`, + shortId: `post_b_${index}`, + title: `Post ${index}`, + tagsStr: 'tag1', + upvotes: 10 + index, + createdAt: new Date(), + sourceId: 'b', + }; + }), + ); + await con.query(`REFRESH MATERIALIZED VIEW popular_video_post`); + await con.query(`REFRESH MATERIALIZED VIEW popular_video_source`); + + const res = await client.query(QUERY); + expect(res.errors).toBeFalsy(); + expect(res.data).toMatchObject({ + topVideoSources: [ + { id: 'a', name: 'A', image: 'http://image.com/a', public: null }, + ], + }); + }); +}); + describe('query sourceByFeed', () => { const QUERY = ` query SourceByFeed($data: String!) { diff --git a/__tests__/tags.ts b/__tests__/tags.ts index 76016f269..ea390b0a2 100644 --- a/__tests__/tags.ts +++ b/__tests__/tags.ts @@ -7,7 +7,7 @@ import { saveFixtures, testQueryError, } from './helpers'; -import { ArticlePost, Keyword, PostKeyword, Source } from '../src/entity'; +import { ArticlePost, Keyword, Post, PostKeyword, Source } from '../src/entity'; import { keywordsFixture, postRecommendedKeywordsFixture, @@ -68,27 +68,46 @@ describe('query trendingTags', () => { } }`; + beforeEach(async () => { + await saveFixtures(con, Source, sourcesFixture); + let tags = 'tag1'; + await con.getRepository(Post).save( + new Array(20).fill('tag').map((item, index) => { + tags += `,tag${index + 1}`; + return { + id: `post_${index}`, + shortId: `post_${index}`, + title: `Post ${index}`, + tagsStr: tags, + upvotes: 10 + index, + createdAt: new Date(), + sourceId: 'a', + }; + }), + ); + await con.query(`REFRESH MATERIALIZED VIEW trending_post`); + await con.query(`REFRESH MATERIALIZED VIEW trending_tag`); + }); + it('should return most trending tags ordered by value', async () => { const res = await client.query(QUERY); expect(res.data).toMatchObject({ trendingTags: [ - { name: 'development' }, - { name: 'fullstack' }, - { name: 'golang' }, - { name: 'rust' }, - { name: 'webdev' }, + { name: 'tag1' }, + { name: 'tag2' }, + { name: 'tag3' }, + { name: 'tag4' }, + { name: 'tag5' }, + { name: 'tag6' }, + { name: 'tag7' }, + { name: 'tag8' }, + { name: 'tag9' }, + { name: 'tag10' }, ], }); }); it('should return limit of 10 by default', async () => { - await con.getRepository(Keyword).save( - new Array(20).fill('tag').map((item, index) => ({ - value: item + index, - occurances: 0, - status: 'allow', - })), - ); const res = await client.query(QUERY); expect(res.data.trendingTags.length).toBe(10); }); @@ -101,19 +120,46 @@ describe('query popularTags', () => { } }`; + beforeEach(async () => { + await saveFixtures(con, Source, sourcesFixture); + let tags = 'tag1'; + await con.getRepository(Post).save( + new Array(20).fill('tag').map((item, index) => { + tags += `,tag${index + 1}`; + return { + id: `post_${index}`, + shortId: `post_${index}`, + title: `Post ${index}`, + tagsStr: tags, + upvotes: 10 + index, + createdAt: new Date(), + sourceId: 'a', + }; + }), + ); + await con.query(`REFRESH MATERIALIZED VIEW popular_post`); + await con.query(`REFRESH MATERIALIZED VIEW popular_tag`); + }); + it('should return most popular tags ordered by value', async () => { const res = await client.query(QUERY); - expect(res.data).toMatchSnapshot(); + expect(res.data).toMatchObject({ + popularTags: [ + { name: 'tag1' }, + { name: 'tag2' }, + { name: 'tag3' }, + { name: 'tag4' }, + { name: 'tag5' }, + { name: 'tag6' }, + { name: 'tag7' }, + { name: 'tag8' }, + { name: 'tag9' }, + { name: 'tag10' }, + ], + }); }); it('should return limit of 10 by default', async () => { - await con.getRepository(Keyword).save( - new Array(20).fill('tag').map((item, index) => ({ - value: item + index, - occurances: 0, - status: 'allow', - })), - ); const res = await client.query(QUERY); expect(res.data.popularTags.length).toBe(10); }); diff --git a/src/entity/PopularPost.ts b/src/entity/PopularPost.ts index c1e5a76e9..5655cb28a 100644 --- a/src/entity/PopularPost.ts +++ b/src/entity/PopularPost.ts @@ -8,7 +8,6 @@ import { Post } from './posts'; .createQueryBuilder() .select('"sourceId"') .addSelect('"tagsStr"') - .addSelect('"contentCuration"') .addSelect('"createdAt"') .addSelect('upvotes - downvotes r') .from(Post, 'p') @@ -25,9 +24,6 @@ export class PopularPost { @ViewColumn() tagsStr: string; - @ViewColumn() - contentCuration: string; - @ViewColumn() createdAt: Date; diff --git a/src/entity/PopularVideoPost.ts b/src/entity/PopularVideoPost.ts index 137f6cb96..2477c6658 100644 --- a/src/entity/PopularVideoPost.ts +++ b/src/entity/PopularVideoPost.ts @@ -8,7 +8,6 @@ import { Post } from './posts'; .createQueryBuilder() .select('"sourceId"') .addSelect('"tagsStr"') - .addSelect('"contentCuration"') .addSelect('"createdAt"') .addSelect('upvotes - downvotes r') .from(Post, 'p') @@ -25,9 +24,6 @@ export class PopularVideoPost { @ViewColumn() tagsStr: string; - @ViewColumn() - contentCuration: string; - @ViewColumn() createdAt: Date; diff --git a/src/migration/1717019094737-HighlightedViews.ts b/src/migration/1717019094737-HighlightedViews.ts index f53b4dca3..f8f78ecc6 100644 --- a/src/migration/1717019094737-HighlightedViews.ts +++ b/src/migration/1717019094737-HighlightedViews.ts @@ -1,44 +1,147 @@ -import { MigrationInterface, QueryRunner } from "typeorm"; +import { MigrationInterface, QueryRunner } from 'typeorm'; export class HighlightedViews1717019094737 implements MigrationInterface { - name = 'HighlightedViews1717019094737' + name = 'HighlightedViews1717019094737'; - public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query(`CREATE MATERIALIZED VIEW "trending_post" AS SELECT "sourceId", "tagsStr", "createdAt", log(10, upvotes - downvotes) + extract(epoch from ("createdAt" - now() + interval '7 days')) / 200000 r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval '7 day' and upvotes > downvotes ORDER BY r DESC LIMIT 100`); - await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","trending_post","SELECT \"sourceId\", \"tagsStr\", \"createdAt\", log(10, upvotes - downvotes) + extract(epoch from (\"createdAt\" - now() + interval '7 days')) / 200000 r FROM \"public\".\"post\" \"p\" WHERE not \"p\".\"private\" and p.\"createdAt\" > now() - interval '7 day' and upvotes > downvotes ORDER BY r DESC LIMIT 100"]); - await queryRunner.query(`CREATE MATERIALIZED VIEW "trending_tag" AS SELECT unnest(string_to_array("tagsStr", ',')) tag, avg(r) r FROM "public"."trending_post" "base" GROUP BY tag HAVING count(*) > 1 ORDER BY r DESC`); - await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","trending_tag","SELECT unnest(string_to_array(\"tagsStr\", ',')) tag, avg(r) r FROM \"public\".\"trending_post\" \"base\" GROUP BY tag HAVING count(*) > 1 ORDER BY r DESC"]); - await queryRunner.query(`CREATE MATERIALIZED VIEW "trending_source" AS SELECT "sourceId", avg(r) r FROM "public"."trending_post" "base" GROUP BY "sourceId" HAVING count(*) > 1 ORDER BY r DESC`); - await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","trending_source","SELECT \"sourceId\", avg(r) r FROM \"public\".\"trending_post\" \"base\" GROUP BY \"sourceId\" HAVING count(*) > 1 ORDER BY r DESC"]); - await queryRunner.query(`CREATE MATERIALIZED VIEW "popular_video_post" AS SELECT "sourceId", "tagsStr", "contentCuration", "createdAt", upvotes - downvotes r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval '60 day' and upvotes > downvotes and "type" = 'video:youtube' ORDER BY r DESC LIMIT 1000`); - await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","popular_video_post","SELECT \"sourceId\", \"tagsStr\", \"contentCuration\", \"createdAt\", upvotes - downvotes r FROM \"public\".\"post\" \"p\" WHERE not \"p\".\"private\" and p.\"createdAt\" > now() - interval '60 day' and upvotes > downvotes and \"type\" = 'video:youtube' ORDER BY r DESC LIMIT 1000"]); - await queryRunner.query(`CREATE MATERIALIZED VIEW "popular_post" AS SELECT "sourceId", "tagsStr", "contentCuration", "createdAt", upvotes - downvotes r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval '60 day' and upvotes > downvotes ORDER BY r DESC LIMIT 1000`); - await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","popular_post","SELECT \"sourceId\", \"tagsStr\", \"contentCuration\", \"createdAt\", upvotes - downvotes r FROM \"public\".\"post\" \"p\" WHERE not \"p\".\"private\" and p.\"createdAt\" > now() - interval '60 day' and upvotes > downvotes ORDER BY r DESC LIMIT 1000"]); - await queryRunner.query(`CREATE MATERIALIZED VIEW "popular_tag" AS SELECT unnest(string_to_array("tagsStr", ',')) tag, avg(r) r FROM "public"."popular_post" "base" GROUP BY tag HAVING count(*) > 10 ORDER BY r DESC`); - await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","popular_tag","SELECT unnest(string_to_array(\"tagsStr\", ',')) tag, avg(r) r FROM \"public\".\"popular_post\" \"base\" GROUP BY tag HAVING count(*) > 10 ORDER BY r DESC"]); - await queryRunner.query(`CREATE MATERIALIZED VIEW "popular_video_source" AS SELECT "sourceId", avg(r) r, count(*) posts FROM "public"."popular_video_post" "base" GROUP BY "sourceId" HAVING count(*) > 5 ORDER BY r DESC`); - await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","popular_video_source","SELECT \"sourceId\", avg(r) r, count(*) posts FROM \"public\".\"popular_video_source\" \"base\" GROUP BY \"sourceId\" HAVING count(*) > 5 ORDER BY r DESC"]); - await queryRunner.query(`CREATE MATERIALIZED VIEW "popular_source" AS SELECT "sourceId", avg(r) r FROM "public"."popular_post" "base" GROUP BY "sourceId" HAVING count(*) > 5 ORDER BY r DESC`); - await queryRunner.query(`INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, ["public","MATERIALIZED_VIEW","popular_source","SELECT \"sourceId\", avg(r) r FROM \"public\".\"popular_post\" \"base\" GROUP BY \"sourceId\" HAVING count(*) > 5 ORDER BY r DESC"]); - } - - public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","popular_source","public"]); - await queryRunner.query(`DROP MATERIALIZED VIEW "popular_source"`); - await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","popular_video_source","public"]); - await queryRunner.query(`DROP MATERIALIZED VIEW "popular_video_source"`); - await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","popular_tag","public"]); - await queryRunner.query(`DROP MATERIALIZED VIEW "popular_tag"`); - await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","popular_post","public"]); - await queryRunner.query(`DROP MATERIALIZED VIEW "popular_post"`); - await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","popular_video_post","public"]); - await queryRunner.query(`DROP MATERIALIZED VIEW "popular_video_post"`); - await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","trending_source","public"]); - await queryRunner.query(`DROP MATERIALIZED VIEW "trending_source"`); - await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","trending_tag","public"]); - await queryRunner.query(`DROP MATERIALIZED VIEW "trending_tag"`); - await queryRunner.query(`DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, ["MATERIALIZED_VIEW","trending_post","public"]); - await queryRunner.query(`DROP MATERIALIZED VIEW "trending_post"`); - } + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE MATERIALIZED VIEW "trending_post" AS SELECT "sourceId", "tagsStr", "createdAt", log(10, upvotes - downvotes) + extract(epoch from ("createdAt" - now() + interval '7 days')) / 200000 r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval '7 day' and upvotes > downvotes ORDER BY r DESC LIMIT 100`, + ); + await queryRunner.query( + `INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'trending_post', + 'SELECT "sourceId", "tagsStr", "createdAt", log(10, upvotes - downvotes) + extract(epoch from ("createdAt" - now() + interval \'7 days\')) / 200000 r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval \'7 day\' and upvotes > downvotes ORDER BY r DESC LIMIT 100', + ], + ); + await queryRunner.query( + `CREATE MATERIALIZED VIEW "trending_tag" AS SELECT unnest(string_to_array("tagsStr", ',')) tag, avg(r) r FROM "public"."trending_post" "base" GROUP BY tag HAVING count(*) > 1 ORDER BY r DESC`, + ); + await queryRunner.query( + `INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'trending_tag', + 'SELECT unnest(string_to_array("tagsStr", \',\')) tag, avg(r) r FROM "public"."trending_post" "base" GROUP BY tag HAVING count(*) > 1 ORDER BY r DESC', + ], + ); + await queryRunner.query( + `CREATE MATERIALIZED VIEW "trending_source" AS SELECT "sourceId", avg(r) r FROM "public"."trending_post" "base" GROUP BY "sourceId" HAVING count(*) > 1 ORDER BY r DESC`, + ); + await queryRunner.query( + `INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'trending_source', + 'SELECT "sourceId", avg(r) r FROM "public"."trending_post" "base" GROUP BY "sourceId" HAVING count(*) > 1 ORDER BY r DESC', + ], + ); + await queryRunner.query( + `CREATE MATERIALIZED VIEW "popular_video_post" AS SELECT "sourceId", "tagsStr", "createdAt", upvotes - downvotes r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval '60 day' and upvotes > downvotes and "type" = 'video:youtube' ORDER BY r DESC LIMIT 1000`, + ); + await queryRunner.query( + `INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'popular_video_post', + 'SELECT "sourceId", "tagsStr", "createdAt", upvotes - downvotes r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval \'60 day\' and upvotes > downvotes and "type" = \'video:youtube\' ORDER BY r DESC LIMIT 1000', + ], + ); + await queryRunner.query( + `CREATE MATERIALIZED VIEW "popular_post" AS SELECT "sourceId", "tagsStr", "createdAt", upvotes - downvotes r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval '60 day' and upvotes > downvotes ORDER BY r DESC LIMIT 1000`, + ); + await queryRunner.query( + `INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'popular_post', + 'SELECT "sourceId", "tagsStr", "createdAt", upvotes - downvotes r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval \'60 day\' and upvotes > downvotes ORDER BY r DESC LIMIT 1000', + ], + ); + await queryRunner.query( + `CREATE MATERIALIZED VIEW "popular_tag" AS SELECT unnest(string_to_array("tagsStr", ',')) tag, avg(r) r FROM "public"."popular_post" "base" GROUP BY tag HAVING count(*) > 10 ORDER BY r DESC`, + ); + await queryRunner.query( + `INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'popular_tag', + 'SELECT unnest(string_to_array("tagsStr", \',\')) tag, avg(r) r FROM "public"."popular_post" "base" GROUP BY tag HAVING count(*) > 10 ORDER BY r DESC', + ], + ); + await queryRunner.query( + `CREATE MATERIALIZED VIEW "popular_video_source" AS SELECT "sourceId", avg(r) r, count(*) posts FROM "public"."popular_video_post" "base" GROUP BY "sourceId" HAVING count(*) > 5 ORDER BY r DESC`, + ); + await queryRunner.query( + `INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'popular_video_source', + 'SELECT "sourceId", avg(r) r, count(*) posts FROM "public"."popular_video_source" "base" GROUP BY "sourceId" HAVING count(*) > 5 ORDER BY r DESC', + ], + ); + await queryRunner.query( + `CREATE MATERIALIZED VIEW "popular_source" AS SELECT "sourceId", avg(r) r FROM "public"."popular_post" "base" GROUP BY "sourceId" HAVING count(*) > 5 ORDER BY r DESC`, + ); + await queryRunner.query( + `INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'popular_source', + 'SELECT "sourceId", avg(r) r FROM "public"."popular_post" "base" GROUP BY "sourceId" HAVING count(*) > 5 ORDER BY r DESC', + ], + ); + } + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['MATERIALIZED_VIEW', 'popular_source', 'public'], + ); + await queryRunner.query(`DROP MATERIALIZED VIEW "popular_source"`); + await queryRunner.query( + `DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['MATERIALIZED_VIEW', 'popular_video_source', 'public'], + ); + await queryRunner.query(`DROP MATERIALIZED VIEW "popular_video_source"`); + await queryRunner.query( + `DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['MATERIALIZED_VIEW', 'popular_tag', 'public'], + ); + await queryRunner.query(`DROP MATERIALIZED VIEW "popular_tag"`); + await queryRunner.query( + `DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['MATERIALIZED_VIEW', 'popular_post', 'public'], + ); + await queryRunner.query(`DROP MATERIALIZED VIEW "popular_post"`); + await queryRunner.query( + `DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['MATERIALIZED_VIEW', 'popular_video_post', 'public'], + ); + await queryRunner.query(`DROP MATERIALIZED VIEW "popular_video_post"`); + await queryRunner.query( + `DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['MATERIALIZED_VIEW', 'trending_source', 'public'], + ); + await queryRunner.query(`DROP MATERIALIZED VIEW "trending_source"`); + await queryRunner.query( + `DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['MATERIALIZED_VIEW', 'trending_tag', 'public'], + ); + await queryRunner.query(`DROP MATERIALIZED VIEW "trending_tag"`); + await queryRunner.query( + `DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['MATERIALIZED_VIEW', 'trending_post', 'public'], + ); + await queryRunner.query(`DROP MATERIALIZED VIEW "trending_post"`); + } } diff --git a/src/schema/sources.ts b/src/schema/sources.ts index 67a36efc3..34fe2e8a2 100644 --- a/src/schema/sources.ts +++ b/src/schema/sources.ts @@ -62,6 +62,9 @@ import { validateAndTransformHandle } from '../common/handles'; import { QueryBuilder } from '../graphorm/graphorm'; import type { GQLTagResults } from './tags'; import { SourceTagView } from '../entity/SourceTagView'; +import { TrendingSource } from '../entity/TrendingSource'; +import { PopularSource } from '../entity/PopularSource'; +import { PopularVideoSource } from '../entity/PopularVideoSource'; export interface GQLSource { id: string; @@ -322,6 +325,46 @@ export const typeDefs = /* GraphQL */ ` featured: Boolean ): SourceConnection! + """ + Get the most recent sources + """ + mostRecentSources( + """ + Limit the number of sources returned + """ + limit: Int + ): [Source] @cacheControl(maxAge: 600) + + """ + Get the most trending sources + """ + trendingSources( + """ + Limit the number of sources returned + """ + limit: Int + ): [Source] @cacheControl(maxAge: 600) + + """ + Get the most popular sources + """ + popularSources( + """ + Limit the number of sources returned + """ + limit: Int + ): [Source] @cacheControl(maxAge: 600) + + """ + Get top video sources + """ + topVideoSources( + """ + Limit the number of sources returned + """ + limit: Int + ): [Source] @cacheControl(maxAge: 600) + """ Get the source that matches the feed """ @@ -1199,6 +1242,61 @@ export const resolvers: IResolvers = { }, ); }, + mostRecentSources: async (_, args, ctx, info): Promise => { + const { limit = 10 } = args; + return await graphorm.query(ctx, info, (builder) => { + builder.queryBuilder + .where({ active: true, type: SourceType.Machine }) + .orderBy('"createdAt"', 'DESC') + .limit(limit); + return builder; + }); + }, + trendingSources: async (_, args, ctx): Promise => { + const { limit = 10 } = args; + /* + * This is not best practice, but due to missing join/basetable support for graphORM we decided to leave it like this to ship faster + * If you ever need child entities/graphORM magic this needs to change + */ + return await ctx.con + .createQueryBuilder() + .select('s.*') + .from(TrendingSource, 'ts') + .innerJoin(Source, 's', 'ts."sourceId" = s.id') + .orderBy('r', 'ASC') + .limit(limit) + .getRawMany(); + }, + popularSources: async (_, args, ctx): Promise => { + const { limit = 10 } = args; + /* + * This is not best practice, but due to missing join/basetable support for graphORM we decided to leave it like this to ship faster + * If you ever need child entities/graphORM magic this needs to change + */ + return await ctx.con + .createQueryBuilder() + .select('s.*') + .from(PopularSource, 'ts') + .innerJoin(Source, 's', 'ts."sourceId" = s.id') + .orderBy('r', 'ASC') + .limit(limit) + .getRawMany(); + }, + topVideoSources: async (_, args, ctx): Promise => { + const { limit = 10 } = args; + /* + * This is not best practice, but due to missing join/basetable support for graphORM we decided to leave it like this to ship faster + * If you ever need child entities/graphORM magic this needs to change + */ + return await ctx.con + .createQueryBuilder() + .select('s.*') + .from(PopularVideoSource, 'ts') + .innerJoin(Source, 's', 'ts."sourceId" = s.id') + .orderBy('r', 'ASC') + .limit(limit) + .getRawMany(); + }, sourceByFeed: async ( _, { feed }: { feed: string }, diff --git a/src/schema/tags.ts b/src/schema/tags.ts index 54071fbd2..cca375b3d 100644 --- a/src/schema/tags.ts +++ b/src/schema/tags.ts @@ -8,6 +8,8 @@ import { ValidationError } from 'apollo-server-errors'; import { SubmissionFailErrorMessage } from '../errors'; import graphorm from '../graphorm'; import { GQLKeyword } from './keywords'; +import { TrendingTag } from '../entity/TrendingTag'; +import { PopularTag } from '../entity/PopularTag'; interface GQLTag { name: string; @@ -108,6 +110,16 @@ export const typeDefs = /* GraphQL */ ` } `; +const getFormattedTags = async (entity, args, ctx): Promise => { + const { limit = 10 } = args; + const tags = await ctx.getRepository(entity).find({ + select: ['tag'], + order: { r: 'ASC' }, + take: limit, + }); + return tags.map(({ tag }) => ({ name: tag })); +}; + // eslint-disable-next-line @typescript-eslint/no-explicit-any export const resolvers: IResolvers = traceResolvers({ Query: { @@ -121,28 +133,10 @@ export const resolvers: IResolvers = traceResolvers({ return builder; }), - trendingTags: async (source, args, ctx): Promise => { - // TODO: Implement query based on new MV - const { limit = 10 } = args; - const hits = await ctx.getRepository(Keyword).find({ - select: ['value'], - order: { value: 'ASC' }, - where: { status: 'allow' }, - take: limit, - }); - return hits.map((x) => ({ name: x.value })); - }, - popularTags: async (source, args, ctx): Promise => { - // TODO: Implement query based on new MV - const { limit = 10 } = args; - const hits = await ctx.getRepository(Keyword).find({ - select: ['value'], - order: { value: 'ASC' }, - where: { status: 'allow' }, - take: limit, - }); - return hits.map((x) => ({ name: x.value })); - }, + trendingTags: async (_, args, ctx): Promise => + await getFormattedTags(TrendingTag, args, ctx), + popularTags: async (_, args, ctx): Promise => + await getFormattedTags(PopularTag, args, ctx), searchTags: async ( source, { query }: { query: string }, From 9ce679aada3cd91e97b4aef67ab232a28c5f12fa Mon Sep 17 00:00:00 2001 From: Chris Bongers Date: Thu, 30 May 2024 19:17:35 +0700 Subject: [PATCH 6/7] Update src/cron/updateHighlightedViews.ts Co-authored-by: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> --- src/cron/updateHighlightedViews.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cron/updateHighlightedViews.ts b/src/cron/updateHighlightedViews.ts index 9ceb71bb6..8440ef581 100644 --- a/src/cron/updateHighlightedViews.ts +++ b/src/cron/updateHighlightedViews.ts @@ -31,7 +31,7 @@ const cron: Cron = { } }); - logger.info({}, 'highlighted views updated'); + logger.info('highlighted views updated'); } catch (err) { logger.error({ err }, 'failed to update highlighted views'); } From a2991afb3d0f331263064a2673bbebc202d3123c Mon Sep 17 00:00:00 2001 From: Chris Bongers Date: Thu, 30 May 2024 19:21:10 +0700 Subject: [PATCH 7/7] fix: cleanup the query to one use-case --- src/schema/sources.ts | 67 ++++++++++++++----------------------------- 1 file changed, 22 insertions(+), 45 deletions(-) diff --git a/src/schema/sources.ts b/src/schema/sources.ts index 34fe2e8a2..85f7effe8 100644 --- a/src/schema/sources.ts +++ b/src/schema/sources.ts @@ -1075,6 +1075,22 @@ const togglePinnedPosts = async ( return { _: true }; }; +const getFormattedSources = async (entity, args, ctx): Promise => { + const { limit = 10 } = args; + /* + * This is not best practice, but due to missing join/basetable support for graphORM we decided to leave it like this to ship faster + * If you ever need child entities/graphORM magic this needs to change + */ + return await ctx.con + .createQueryBuilder() + .select('s.*') + .from(entity, 'ts') + .innerJoin(Source, 's', 'ts."sourceId" = s.id') + .orderBy('r', 'ASC') + .limit(limit) + .getRawMany(); +}; + const paginateSourceMembers = ( query: (builder: QueryBuilder, alias: string) => QueryBuilder, args: ConnectionArguments, @@ -1252,51 +1268,12 @@ export const resolvers: IResolvers = { return builder; }); }, - trendingSources: async (_, args, ctx): Promise => { - const { limit = 10 } = args; - /* - * This is not best practice, but due to missing join/basetable support for graphORM we decided to leave it like this to ship faster - * If you ever need child entities/graphORM magic this needs to change - */ - return await ctx.con - .createQueryBuilder() - .select('s.*') - .from(TrendingSource, 'ts') - .innerJoin(Source, 's', 'ts."sourceId" = s.id') - .orderBy('r', 'ASC') - .limit(limit) - .getRawMany(); - }, - popularSources: async (_, args, ctx): Promise => { - const { limit = 10 } = args; - /* - * This is not best practice, but due to missing join/basetable support for graphORM we decided to leave it like this to ship faster - * If you ever need child entities/graphORM magic this needs to change - */ - return await ctx.con - .createQueryBuilder() - .select('s.*') - .from(PopularSource, 'ts') - .innerJoin(Source, 's', 'ts."sourceId" = s.id') - .orderBy('r', 'ASC') - .limit(limit) - .getRawMany(); - }, - topVideoSources: async (_, args, ctx): Promise => { - const { limit = 10 } = args; - /* - * This is not best practice, but due to missing join/basetable support for graphORM we decided to leave it like this to ship faster - * If you ever need child entities/graphORM magic this needs to change - */ - return await ctx.con - .createQueryBuilder() - .select('s.*') - .from(PopularVideoSource, 'ts') - .innerJoin(Source, 's', 'ts."sourceId" = s.id') - .orderBy('r', 'ASC') - .limit(limit) - .getRawMany(); - }, + trendingSources: async (_, args, ctx): Promise => + getFormattedSources(TrendingSource, args, ctx), + popularSources: async (_, args, ctx): Promise => + getFormattedSources(PopularSource, args, ctx), + topVideoSources: async (_, args, ctx): Promise => + getFormattedSources(PopularVideoSource, args, ctx), sourceByFeed: async ( _, { feed }: { feed: string },