From c0758a0c79adf978ca49f5bf4d0ca9d2accfba54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ante=20Bari=C4=87?= Date: Thu, 30 May 2024 14:32:15 +0200 Subject: [PATCH] feat: highlighted views and queries (#1953) --- .infra/crons.ts | 4 + __tests__/__snapshots__/tags.ts.snap | 22 --- __tests__/cron/updateHighlightedViews.ts | 10 + __tests__/sources.ts | 178 ++++++++++++++++++ __tests__/tags.ts | 88 ++++++--- src/cron/index.ts | 2 + src/cron/updateHighlightedViews.ts | 41 ++++ src/entity/PopularPost.ts | 32 ++++ src/entity/PopularSource.ts | 22 +++ src/entity/PopularTag.ts | 22 +++ src/entity/PopularVideoPost.ts | 32 ++++ src/entity/PopularVideoSource.ts | 26 +++ src/entity/TrendingPost.ts | 34 ++++ src/entity/TrendingSource.ts | 22 +++ src/entity/TrendingTag.ts | 22 +++ .../1717019094737-HighlightedViews.ts | 147 +++++++++++++++ src/schema/sources.ts | 75 ++++++++ src/schema/tags.ts | 38 ++-- 18 files changed, 752 insertions(+), 65 deletions(-) create mode 100644 __tests__/cron/updateHighlightedViews.ts create mode 100644 src/cron/updateHighlightedViews.ts create mode 100644 src/entity/PopularPost.ts create mode 100644 src/entity/PopularSource.ts create mode 100644 src/entity/PopularTag.ts create mode 100644 src/entity/PopularVideoPost.ts create mode 100644 src/entity/PopularVideoSource.ts create mode 100644 src/entity/TrendingPost.ts create mode 100644 src/entity/TrendingSource.ts create mode 100644 src/entity/TrendingTag.ts create mode 100644 src/migration/1717019094737-HighlightedViews.ts diff --git a/.infra/crons.ts b/.infra/crons.ts index 33e8fe98f..bed0a5616 100644 --- a/.infra/crons.ts +++ b/.infra/crons.ts @@ -85,4 +85,8 @@ export const crons: Cron[] = [ memory: '1Gi', }, }, + { + name: 'update-highlighted-views', + schedule: '15 4 * * 0', + }, ]; diff --git a/__tests__/__snapshots__/tags.ts.snap b/__tests__/__snapshots__/tags.ts.snap index 6fd149406..0fe45f1a2 100644 --- a/__tests__/__snapshots__/tags.ts.snap +++ b/__tests__/__snapshots__/tags.ts.snap @@ -1,27 +1,5 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`query popularTags should return most popular tags ordered by value 1`] = ` -Object { - "popularTags": Array [ - Object { - "name": "development", - }, - Object { - "name": "fullstack", - }, - Object { - "name": "golang", - }, - Object { - "name": "rust", - }, - Object { - "name": "webdev", - }, - ], -} -`; - exports[`query searchTags should search for tags and order by value 1`] = ` Object { "searchTags": Object { diff --git a/__tests__/cron/updateHighlightedViews.ts b/__tests__/cron/updateHighlightedViews.ts new file mode 100644 index 000000000..d14281ef6 --- /dev/null +++ b/__tests__/cron/updateHighlightedViews.ts @@ -0,0 +1,10 @@ +import { crons } from '../../src/cron/index'; +import cron from '../../src/cron/updateHighlightedViews'; + +describe('updateHighlightedViews cron', () => { + it('should be registered', () => { + const registeredWorker = crons.find((item) => item.name === cron.name); + + expect(registeredWorker).toBeDefined(); + }); +}); diff --git a/__tests__/sources.ts b/__tests__/sources.ts index 4753d4acf..35a1cf074 100644 --- a/__tests__/sources.ts +++ b/__tests__/sources.ts @@ -14,6 +14,7 @@ import { NotificationPreferenceSource, Post, PostKeyword, + PostType, SharePost, Source, SourceFeed, @@ -232,6 +233,183 @@ describe('query sources', () => { }); }); +describe('query mostRecentSources', () => { + const QUERY = ` + query MostRecentSources { + mostRecentSources { + id + name + image + public + } + } + `; + + it('should return most recent sources', async () => { + const res = await client.query(QUERY); + expect(res.errors).toBeFalsy(); + expect(res.data).toMatchObject({ + mostRecentSources: [ + { id: 'a', name: 'A', image: 'http://image.com/a', public: true }, + { id: 'b', name: 'B', image: 'http://image.com/b', public: true }, + ], + }); + }); +}); + +describe('query trendingSources', () => { + const QUERY = ` + query TrendingSources { + trendingSources { + id + name + image + public + } + } + `; + + it('should return most trending sources', async () => { + await con.getRepository(Post).save( + new Array(5).fill('a').map((item, index) => { + return { + id: `post_${index}`, + shortId: `post_${index}`, + title: `Post ${index}`, + tagsStr: 'tag1', + upvotes: 10 + index, + createdAt: new Date(), + sourceId: 'a', + }; + }), + ); + await con.getRepository(Post).save({ + id: `post_6`, + shortId: `post_6`, + title: `Post 6`, + tagsStr: 'tag1', + upvotes: 10, + createdAt: new Date(), + sourceId: 'b', + }); + await con.query(`REFRESH MATERIALIZED VIEW trending_post`); + await con.query(`REFRESH MATERIALIZED VIEW trending_source`); + + const res = await client.query(QUERY); + expect(res.errors).toBeFalsy(); + expect(res.data).toMatchObject({ + trendingSources: [ + { id: 'a', name: 'A', image: 'http://image.com/a', public: null }, + ], + }); + }); +}); + +describe('query popularSources', () => { + const QUERY = ` + query PopularSources { + popularSources { + id + name + image + public + } + } + `; + + it('should return most popular sources', async () => { + await con.getRepository(Post).save( + new Array(6).fill('a').map((item, index) => { + return { + id: `post_${index}`, + shortId: `post_${index}`, + title: `Post ${index}`, + tagsStr: 'tag1', + upvotes: 10 + index, + createdAt: new Date(), + sourceId: 'a', + }; + }), + ); + await con.getRepository(Post).save( + new Array(5).fill('b').map((item, index) => { + return { + id: `post_${index}`, + shortId: `post_${index}`, + title: `Post ${index}`, + tagsStr: 'tag1', + upvotes: 10 + index, + createdAt: new Date(), + sourceId: 'a', + }; + }), + ); + await con.query(`REFRESH MATERIALIZED VIEW popular_post`); + await con.query(`REFRESH MATERIALIZED VIEW popular_source`); + + const res = await client.query(QUERY); + expect(res.errors).toBeFalsy(); + expect(res.data).toMatchObject({ + popularSources: [ + { id: 'a', name: 'A', image: 'http://image.com/a', public: null }, + ], + }); + }); +}); + +describe('query topVideoSources', () => { + const QUERY = ` + query TopVideoSources { + topVideoSources { + id + name + image + public + } + } + `; + + it('should return top video sources', async () => { + await con.getRepository(Post).save( + new Array(6).fill('a').map((item, index) => { + return { + id: `post_a_${index}`, + shortId: `post_a_${index}`, + title: `Post ${index}`, + tagsStr: 'tag1', + upvotes: 10 + index, + createdAt: new Date(), + sourceId: 'a', + type: PostType.VideoYouTube, + }; + }), + ); + await con.getRepository(Post).save( + new Array(6).fill('b').map((item, index) => { + return { + id: `post_b_${index}`, + shortId: `post_b_${index}`, + title: `Post ${index}`, + tagsStr: 'tag1', + upvotes: 10 + index, + createdAt: new Date(), + sourceId: 'b', + }; + }), + ); + await con.query(`REFRESH MATERIALIZED VIEW popular_video_post`); + await con.query(`REFRESH MATERIALIZED VIEW popular_video_source`); + + const res = await client.query(QUERY); + expect(res.errors).toBeFalsy(); + expect(res.data).toMatchObject({ + topVideoSources: [ + { id: 'a', name: 'A', image: 'http://image.com/a', public: null }, + ], + }); + }); +}); + describe('query sourceByFeed', () => { const QUERY = ` query SourceByFeed($data: String!) { diff --git a/__tests__/tags.ts b/__tests__/tags.ts index 76016f269..ea390b0a2 100644 --- a/__tests__/tags.ts +++ b/__tests__/tags.ts @@ -7,7 +7,7 @@ import { saveFixtures, testQueryError, } from './helpers'; -import { ArticlePost, Keyword, PostKeyword, Source } from '../src/entity'; +import { ArticlePost, Keyword, Post, PostKeyword, Source } from '../src/entity'; import { keywordsFixture, postRecommendedKeywordsFixture, @@ -68,27 +68,46 @@ describe('query trendingTags', () => { } }`; + beforeEach(async () => { + await saveFixtures(con, Source, sourcesFixture); + let tags = 'tag1'; + await con.getRepository(Post).save( + new Array(20).fill('tag').map((item, index) => { + tags += `,tag${index + 1}`; + return { + id: `post_${index}`, + shortId: `post_${index}`, + title: `Post ${index}`, + tagsStr: tags, + upvotes: 10 + index, + createdAt: new Date(), + sourceId: 'a', + }; + }), + ); + await con.query(`REFRESH MATERIALIZED VIEW trending_post`); + await con.query(`REFRESH MATERIALIZED VIEW trending_tag`); + }); + it('should return most trending tags ordered by value', async () => { const res = await client.query(QUERY); expect(res.data).toMatchObject({ trendingTags: [ - { name: 'development' }, - { name: 'fullstack' }, - { name: 'golang' }, - { name: 'rust' }, - { name: 'webdev' }, + { name: 'tag1' }, + { name: 'tag2' }, + { name: 'tag3' }, + { name: 'tag4' }, + { name: 'tag5' }, + { name: 'tag6' }, + { name: 'tag7' }, + { name: 'tag8' }, + { name: 'tag9' }, + { name: 'tag10' }, ], }); }); it('should return limit of 10 by default', async () => { - await con.getRepository(Keyword).save( - new Array(20).fill('tag').map((item, index) => ({ - value: item + index, - occurances: 0, - status: 'allow', - })), - ); const res = await client.query(QUERY); expect(res.data.trendingTags.length).toBe(10); }); @@ -101,19 +120,46 @@ describe('query popularTags', () => { } }`; + beforeEach(async () => { + await saveFixtures(con, Source, sourcesFixture); + let tags = 'tag1'; + await con.getRepository(Post).save( + new Array(20).fill('tag').map((item, index) => { + tags += `,tag${index + 1}`; + return { + id: `post_${index}`, + shortId: `post_${index}`, + title: `Post ${index}`, + tagsStr: tags, + upvotes: 10 + index, + createdAt: new Date(), + sourceId: 'a', + }; + }), + ); + await con.query(`REFRESH MATERIALIZED VIEW popular_post`); + await con.query(`REFRESH MATERIALIZED VIEW popular_tag`); + }); + it('should return most popular tags ordered by value', async () => { const res = await client.query(QUERY); - expect(res.data).toMatchSnapshot(); + expect(res.data).toMatchObject({ + popularTags: [ + { name: 'tag1' }, + { name: 'tag2' }, + { name: 'tag3' }, + { name: 'tag4' }, + { name: 'tag5' }, + { name: 'tag6' }, + { name: 'tag7' }, + { name: 'tag8' }, + { name: 'tag9' }, + { name: 'tag10' }, + ], + }); }); it('should return limit of 10 by default', async () => { - await con.getRepository(Keyword).save( - new Array(20).fill('tag').map((item, index) => ({ - value: item + index, - occurances: 0, - status: 'allow', - })), - ); const res = await client.query(QUERY); expect(res.data.popularTags.length).toBe(10); }); diff --git a/src/cron/index.ts b/src/cron/index.ts index 2b90fca3e..a9bebd94c 100644 --- a/src/cron/index.ts +++ b/src/cron/index.ts @@ -11,6 +11,7 @@ import personalizedDigest from './personalizedDigest'; import generateSearchInvites from './generateSearchInvites'; import checkReferralReminder from './checkReferralReminder'; import dailyDigest from './dailyDigest'; +import updateHighlightedViews from './updateHighlightedViews'; export const crons: Cron[] = [ updateViews, @@ -25,4 +26,5 @@ export const crons: Cron[] = [ generateSearchInvites, checkReferralReminder, dailyDigest, + updateHighlightedViews, ]; diff --git a/src/cron/updateHighlightedViews.ts b/src/cron/updateHighlightedViews.ts new file mode 100644 index 000000000..8440ef581 --- /dev/null +++ b/src/cron/updateHighlightedViews.ts @@ -0,0 +1,41 @@ +import { Cron } from './cron'; +import { PopularPost } from '../entity/PopularPost'; +import { PopularSource } from '../entity/PopularSource'; +import { PopularTag } from '../entity/PopularTag'; +import { PopularVideoSource } from '../entity/PopularVideoSource'; +import { TrendingPost } from '../entity/TrendingPost'; +import { TrendingSource } from '../entity/TrendingSource'; +import { TrendingTag } from '../entity/TrendingTag'; +import { PopularVideoPost } from '../entity/PopularVideoPost'; + +const cron: Cron = { + name: 'update-highlighted-views', + handler: async (con, logger) => { + const viewsToRefresh = [ + TrendingPost, + TrendingSource, + TrendingTag, + PopularPost, + PopularSource, + PopularTag, + PopularVideoPost, + PopularVideoSource, + ]; + + try { + await con.transaction(async (manager) => { + for (const viewToRefresh of viewsToRefresh) { + await manager.query( + `REFRESH MATERIALIZED VIEW ${con.getRepository(viewToRefresh).metadata.tableName}`, + ); + } + }); + + logger.info('highlighted views updated'); + } catch (err) { + logger.error({ err }, 'failed to update highlighted views'); + } + }, +}; + +export default cron; diff --git a/src/entity/PopularPost.ts b/src/entity/PopularPost.ts new file mode 100644 index 000000000..5655cb28a --- /dev/null +++ b/src/entity/PopularPost.ts @@ -0,0 +1,32 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; +import { Post } from './posts'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select('"sourceId"') + .addSelect('"tagsStr"') + .addSelect('"createdAt"') + .addSelect('upvotes - downvotes r') + .from(Post, 'p') + .where( + 'not p.private and p."createdAt" > now() - interval \'60 day\' and upvotes > downvotes', + ) + .orderBy('r', 'DESC') + .limit(1000), +}) +export class PopularPost { + @ViewColumn() + sourceId: string; + + @ViewColumn() + tagsStr: string; + + @ViewColumn() + createdAt: Date; + + @ViewColumn() + r: number; +} diff --git a/src/entity/PopularSource.ts b/src/entity/PopularSource.ts new file mode 100644 index 000000000..7aa8a6438 --- /dev/null +++ b/src/entity/PopularSource.ts @@ -0,0 +1,22 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; +import { PopularPost } from './PopularPost'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select('"sourceId"') + .addSelect('avg(r) r') + .from(PopularPost, 'base') + .groupBy('"sourceId"') + .having('count(*) > 5') + .orderBy('r', 'DESC'), +}) +export class PopularSource { + @ViewColumn() + sourceId: string; + + @ViewColumn() + r: number; +} diff --git a/src/entity/PopularTag.ts b/src/entity/PopularTag.ts new file mode 100644 index 000000000..bc96133cf --- /dev/null +++ b/src/entity/PopularTag.ts @@ -0,0 +1,22 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; +import { PopularPost } from './PopularPost'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select('unnest(string_to_array("tagsStr", \',\')) tag') + .addSelect('avg(r) r') + .from(PopularPost, 'base') + .groupBy('tag') + .having('count(*) > 10') + .orderBy('r', 'DESC'), +}) +export class PopularTag { + @ViewColumn() + tag: string; + + @ViewColumn() + r: number; +} diff --git a/src/entity/PopularVideoPost.ts b/src/entity/PopularVideoPost.ts new file mode 100644 index 000000000..2477c6658 --- /dev/null +++ b/src/entity/PopularVideoPost.ts @@ -0,0 +1,32 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; +import { Post } from './posts'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select('"sourceId"') + .addSelect('"tagsStr"') + .addSelect('"createdAt"') + .addSelect('upvotes - downvotes r') + .from(Post, 'p') + .where( + 'not p.private and p."createdAt" > now() - interval \'60 day\' and upvotes > downvotes and "type" = \'video:youtube\'', + ) + .orderBy('r', 'DESC') + .limit(1000), +}) +export class PopularVideoPost { + @ViewColumn() + sourceId: string; + + @ViewColumn() + tagsStr: string; + + @ViewColumn() + createdAt: Date; + + @ViewColumn() + r: number; +} diff --git a/src/entity/PopularVideoSource.ts b/src/entity/PopularVideoSource.ts new file mode 100644 index 000000000..bf3f136af --- /dev/null +++ b/src/entity/PopularVideoSource.ts @@ -0,0 +1,26 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; +import { PopularVideoPost } from './PopularVideoPost'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select('"sourceId"') + .addSelect('avg(r) r') + .addSelect('count(*) posts') + .from(PopularVideoPost, 'base') + .groupBy('"sourceId"') + .having('count(*) > 5') + .orderBy('r', 'DESC'), +}) +export class PopularVideoSource { + @ViewColumn() + sourceId: string; + + @ViewColumn() + r: number; + + @ViewColumn() + posts: number; +} diff --git a/src/entity/TrendingPost.ts b/src/entity/TrendingPost.ts new file mode 100644 index 000000000..349facae7 --- /dev/null +++ b/src/entity/TrendingPost.ts @@ -0,0 +1,34 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; +import { Post } from './posts'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select('"sourceId"') + .addSelect('"tagsStr"') + .addSelect('"createdAt"') + .addSelect( + `log(10, upvotes - downvotes) + extract(epoch from ("createdAt" - now() + interval '7 days')) / 200000 r`, + ) + .from(Post, 'p') + .where( + `not p.private and p."createdAt" > now() - interval '7 day' and upvotes > downvotes`, + ) + .orderBy('r', 'DESC') + .limit(100), +}) +export class TrendingPost { + @ViewColumn() + sourceId: string; + + @ViewColumn() + tagsStr: string; + + @ViewColumn() + createdAt: Date; + + @ViewColumn() + r: number; +} diff --git a/src/entity/TrendingSource.ts b/src/entity/TrendingSource.ts new file mode 100644 index 000000000..0ecbcb510 --- /dev/null +++ b/src/entity/TrendingSource.ts @@ -0,0 +1,22 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; +import { TrendingPost } from './TrendingPost'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select('"sourceId"') + .addSelect('avg(r) r') + .from(TrendingPost, 'base') + .groupBy('"sourceId"') + .having('count(*) > 1') + .orderBy('r', 'DESC'), +}) +export class TrendingSource { + @ViewColumn() + sourceId: string; + + @ViewColumn() + r: number; +} diff --git a/src/entity/TrendingTag.ts b/src/entity/TrendingTag.ts new file mode 100644 index 000000000..0e0d80444 --- /dev/null +++ b/src/entity/TrendingTag.ts @@ -0,0 +1,22 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; +import { TrendingPost } from './TrendingPost'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select('unnest(string_to_array("tagsStr", \',\')) tag') + .addSelect('avg(r) r') + .from(TrendingPost, 'base') + .groupBy('tag') + .having('count(*) > 1') + .orderBy('r', 'DESC'), +}) +export class TrendingTag { + @ViewColumn() + tag: string; + + @ViewColumn() + r: number; +} diff --git a/src/migration/1717019094737-HighlightedViews.ts b/src/migration/1717019094737-HighlightedViews.ts new file mode 100644 index 000000000..f8f78ecc6 --- /dev/null +++ b/src/migration/1717019094737-HighlightedViews.ts @@ -0,0 +1,147 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class HighlightedViews1717019094737 implements MigrationInterface { + name = 'HighlightedViews1717019094737'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE MATERIALIZED VIEW "trending_post" AS SELECT "sourceId", "tagsStr", "createdAt", log(10, upvotes - downvotes) + extract(epoch from ("createdAt" - now() + interval '7 days')) / 200000 r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval '7 day' and upvotes > downvotes ORDER BY r DESC LIMIT 100`, + ); + await queryRunner.query( + `INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'trending_post', + 'SELECT "sourceId", "tagsStr", "createdAt", log(10, upvotes - downvotes) + extract(epoch from ("createdAt" - now() + interval \'7 days\')) / 200000 r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval \'7 day\' and upvotes > downvotes ORDER BY r DESC LIMIT 100', + ], + ); + await queryRunner.query( + `CREATE MATERIALIZED VIEW "trending_tag" AS SELECT unnest(string_to_array("tagsStr", ',')) tag, avg(r) r FROM "public"."trending_post" "base" GROUP BY tag HAVING count(*) > 1 ORDER BY r DESC`, + ); + await queryRunner.query( + `INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'trending_tag', + 'SELECT unnest(string_to_array("tagsStr", \',\')) tag, avg(r) r FROM "public"."trending_post" "base" GROUP BY tag HAVING count(*) > 1 ORDER BY r DESC', + ], + ); + await queryRunner.query( + `CREATE MATERIALIZED VIEW "trending_source" AS SELECT "sourceId", avg(r) r FROM "public"."trending_post" "base" GROUP BY "sourceId" HAVING count(*) > 1 ORDER BY r DESC`, + ); + await queryRunner.query( + `INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'trending_source', + 'SELECT "sourceId", avg(r) r FROM "public"."trending_post" "base" GROUP BY "sourceId" HAVING count(*) > 1 ORDER BY r DESC', + ], + ); + await queryRunner.query( + `CREATE MATERIALIZED VIEW "popular_video_post" AS SELECT "sourceId", "tagsStr", "createdAt", upvotes - downvotes r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval '60 day' and upvotes > downvotes and "type" = 'video:youtube' ORDER BY r DESC LIMIT 1000`, + ); + await queryRunner.query( + `INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'popular_video_post', + 'SELECT "sourceId", "tagsStr", "createdAt", upvotes - downvotes r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval \'60 day\' and upvotes > downvotes and "type" = \'video:youtube\' ORDER BY r DESC LIMIT 1000', + ], + ); + await queryRunner.query( + `CREATE MATERIALIZED VIEW "popular_post" AS SELECT "sourceId", "tagsStr", "createdAt", upvotes - downvotes r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval '60 day' and upvotes > downvotes ORDER BY r DESC LIMIT 1000`, + ); + await queryRunner.query( + `INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'popular_post', + 'SELECT "sourceId", "tagsStr", "createdAt", upvotes - downvotes r FROM "public"."post" "p" WHERE not "p"."private" and p."createdAt" > now() - interval \'60 day\' and upvotes > downvotes ORDER BY r DESC LIMIT 1000', + ], + ); + await queryRunner.query( + `CREATE MATERIALIZED VIEW "popular_tag" AS SELECT unnest(string_to_array("tagsStr", ',')) tag, avg(r) r FROM "public"."popular_post" "base" GROUP BY tag HAVING count(*) > 10 ORDER BY r DESC`, + ); + await queryRunner.query( + `INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'popular_tag', + 'SELECT unnest(string_to_array("tagsStr", \',\')) tag, avg(r) r FROM "public"."popular_post" "base" GROUP BY tag HAVING count(*) > 10 ORDER BY r DESC', + ], + ); + await queryRunner.query( + `CREATE MATERIALIZED VIEW "popular_video_source" AS SELECT "sourceId", avg(r) r, count(*) posts FROM "public"."popular_video_post" "base" GROUP BY "sourceId" HAVING count(*) > 5 ORDER BY r DESC`, + ); + await queryRunner.query( + `INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'popular_video_source', + 'SELECT "sourceId", avg(r) r, count(*) posts FROM "public"."popular_video_source" "base" GROUP BY "sourceId" HAVING count(*) > 5 ORDER BY r DESC', + ], + ); + await queryRunner.query( + `CREATE MATERIALIZED VIEW "popular_source" AS SELECT "sourceId", avg(r) r FROM "public"."popular_post" "base" GROUP BY "sourceId" HAVING count(*) > 5 ORDER BY r DESC`, + ); + await queryRunner.query( + `INSERT INTO "public"."typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES (DEFAULT, $1, DEFAULT, $2, $3, $4)`, + [ + 'public', + 'MATERIALIZED_VIEW', + 'popular_source', + 'SELECT "sourceId", avg(r) r FROM "public"."popular_post" "base" GROUP BY "sourceId" HAVING count(*) > 5 ORDER BY r DESC', + ], + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['MATERIALIZED_VIEW', 'popular_source', 'public'], + ); + await queryRunner.query(`DROP MATERIALIZED VIEW "popular_source"`); + await queryRunner.query( + `DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['MATERIALIZED_VIEW', 'popular_video_source', 'public'], + ); + await queryRunner.query(`DROP MATERIALIZED VIEW "popular_video_source"`); + await queryRunner.query( + `DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['MATERIALIZED_VIEW', 'popular_tag', 'public'], + ); + await queryRunner.query(`DROP MATERIALIZED VIEW "popular_tag"`); + await queryRunner.query( + `DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['MATERIALIZED_VIEW', 'popular_post', 'public'], + ); + await queryRunner.query(`DROP MATERIALIZED VIEW "popular_post"`); + await queryRunner.query( + `DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['MATERIALIZED_VIEW', 'popular_video_post', 'public'], + ); + await queryRunner.query(`DROP MATERIALIZED VIEW "popular_video_post"`); + await queryRunner.query( + `DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['MATERIALIZED_VIEW', 'trending_source', 'public'], + ); + await queryRunner.query(`DROP MATERIALIZED VIEW "trending_source"`); + await queryRunner.query( + `DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['MATERIALIZED_VIEW', 'trending_tag', 'public'], + ); + await queryRunner.query(`DROP MATERIALIZED VIEW "trending_tag"`); + await queryRunner.query( + `DELETE FROM "public"."typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "schema" = $3`, + ['MATERIALIZED_VIEW', 'trending_post', 'public'], + ); + await queryRunner.query(`DROP MATERIALIZED VIEW "trending_post"`); + } +} diff --git a/src/schema/sources.ts b/src/schema/sources.ts index 67a36efc3..85f7effe8 100644 --- a/src/schema/sources.ts +++ b/src/schema/sources.ts @@ -62,6 +62,9 @@ import { validateAndTransformHandle } from '../common/handles'; import { QueryBuilder } from '../graphorm/graphorm'; import type { GQLTagResults } from './tags'; import { SourceTagView } from '../entity/SourceTagView'; +import { TrendingSource } from '../entity/TrendingSource'; +import { PopularSource } from '../entity/PopularSource'; +import { PopularVideoSource } from '../entity/PopularVideoSource'; export interface GQLSource { id: string; @@ -322,6 +325,46 @@ export const typeDefs = /* GraphQL */ ` featured: Boolean ): SourceConnection! + """ + Get the most recent sources + """ + mostRecentSources( + """ + Limit the number of sources returned + """ + limit: Int + ): [Source] @cacheControl(maxAge: 600) + + """ + Get the most trending sources + """ + trendingSources( + """ + Limit the number of sources returned + """ + limit: Int + ): [Source] @cacheControl(maxAge: 600) + + """ + Get the most popular sources + """ + popularSources( + """ + Limit the number of sources returned + """ + limit: Int + ): [Source] @cacheControl(maxAge: 600) + + """ + Get top video sources + """ + topVideoSources( + """ + Limit the number of sources returned + """ + limit: Int + ): [Source] @cacheControl(maxAge: 600) + """ Get the source that matches the feed """ @@ -1032,6 +1075,22 @@ const togglePinnedPosts = async ( return { _: true }; }; +const getFormattedSources = async (entity, args, ctx): Promise => { + const { limit = 10 } = args; + /* + * This is not best practice, but due to missing join/basetable support for graphORM we decided to leave it like this to ship faster + * If you ever need child entities/graphORM magic this needs to change + */ + return await ctx.con + .createQueryBuilder() + .select('s.*') + .from(entity, 'ts') + .innerJoin(Source, 's', 'ts."sourceId" = s.id') + .orderBy('r', 'ASC') + .limit(limit) + .getRawMany(); +}; + const paginateSourceMembers = ( query: (builder: QueryBuilder, alias: string) => QueryBuilder, args: ConnectionArguments, @@ -1199,6 +1258,22 @@ export const resolvers: IResolvers = { }, ); }, + mostRecentSources: async (_, args, ctx, info): Promise => { + const { limit = 10 } = args; + return await graphorm.query(ctx, info, (builder) => { + builder.queryBuilder + .where({ active: true, type: SourceType.Machine }) + .orderBy('"createdAt"', 'DESC') + .limit(limit); + return builder; + }); + }, + trendingSources: async (_, args, ctx): Promise => + getFormattedSources(TrendingSource, args, ctx), + popularSources: async (_, args, ctx): Promise => + getFormattedSources(PopularSource, args, ctx), + topVideoSources: async (_, args, ctx): Promise => + getFormattedSources(PopularVideoSource, args, ctx), sourceByFeed: async ( _, { feed }: { feed: string }, diff --git a/src/schema/tags.ts b/src/schema/tags.ts index 54071fbd2..cca375b3d 100644 --- a/src/schema/tags.ts +++ b/src/schema/tags.ts @@ -8,6 +8,8 @@ import { ValidationError } from 'apollo-server-errors'; import { SubmissionFailErrorMessage } from '../errors'; import graphorm from '../graphorm'; import { GQLKeyword } from './keywords'; +import { TrendingTag } from '../entity/TrendingTag'; +import { PopularTag } from '../entity/PopularTag'; interface GQLTag { name: string; @@ -108,6 +110,16 @@ export const typeDefs = /* GraphQL */ ` } `; +const getFormattedTags = async (entity, args, ctx): Promise => { + const { limit = 10 } = args; + const tags = await ctx.getRepository(entity).find({ + select: ['tag'], + order: { r: 'ASC' }, + take: limit, + }); + return tags.map(({ tag }) => ({ name: tag })); +}; + // eslint-disable-next-line @typescript-eslint/no-explicit-any export const resolvers: IResolvers = traceResolvers({ Query: { @@ -121,28 +133,10 @@ export const resolvers: IResolvers = traceResolvers({ return builder; }), - trendingTags: async (source, args, ctx): Promise => { - // TODO: Implement query based on new MV - const { limit = 10 } = args; - const hits = await ctx.getRepository(Keyword).find({ - select: ['value'], - order: { value: 'ASC' }, - where: { status: 'allow' }, - take: limit, - }); - return hits.map((x) => ({ name: x.value })); - }, - popularTags: async (source, args, ctx): Promise => { - // TODO: Implement query based on new MV - const { limit = 10 } = args; - const hits = await ctx.getRepository(Keyword).find({ - select: ['value'], - order: { value: 'ASC' }, - where: { status: 'allow' }, - take: limit, - }); - return hits.map((x) => ({ name: x.value })); - }, + trendingTags: async (_, args, ctx): Promise => + await getFormattedTags(TrendingTag, args, ctx), + popularTags: async (_, args, ctx): Promise => + await getFormattedTags(PopularTag, args, ctx), searchTags: async ( source, { query }: { query: string },