diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 682c8c2ecbe..a43416f6cca 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -334,9 +334,6 @@ jobs: - name: Build run: pnpm -r build - - name: build - run: pnpm -r lint - - name: Run unit tests run: pnpm -r test -- --allowOnly=false diff --git a/libs/core/src/integration/events.schemas.ts b/libs/core/src/integration/events.schemas.ts index 68574cb1f9b..7f19d05a094 100644 --- a/libs/core/src/integration/events.schemas.ts +++ b/libs/core/src/integration/events.schemas.ts @@ -19,7 +19,7 @@ import { } from './chain-event.schemas'; import { EventMetadata } from './util.schemas'; -export const ThreadCreated = Thread.extend({ +export const ThreadCreated = Thread.omit({ search: true }).extend({ contestManagers: z.array(z.object({ contest_address: z.string() })).nullish(), }); export const ThreadUpvoted = Reaction.omit({ comment_id: true }).extend({ @@ -27,7 +27,7 @@ export const ThreadUpvoted = Reaction.omit({ comment_id: true }).extend({ community_id: z.string(), contestManagers: z.array(z.object({ contest_address: z.string() })).nullish(), }); -export const CommentCreated = Comment.extend({ +export const CommentCreated = Comment.omit({ search: true }).extend({ community_id: z.string(), users_mentioned: z .array(PG_INT) diff --git a/libs/model/src/comment/CreateComment.command.ts b/libs/model/src/comment/CreateComment.command.ts index c0d88853af6..0a93333b6d4 100644 --- a/libs/model/src/comment/CreateComment.command.ts +++ b/libs/model/src/comment/CreateComment.command.ts @@ -1,4 +1,5 @@ import { EventNames, InvalidState, type Command } from '@hicommonwealth/core'; +import { getCommentSearchVector } from '@hicommonwealth/model'; import * as schemas from '@hicommonwealth/schemas'; import { models } from '../database'; import { isAuthorized, type AuthContext } from '../middleware'; @@ -70,6 +71,7 @@ export function CreateComment(): Command< reaction_count: 0, reaction_weights_sum: 0, created_by: '', + search: getCommentSearchVector(text), }, { transaction, diff --git a/libs/model/src/comment/SearchComments.query.ts b/libs/model/src/comment/SearchComments.query.ts index de239fcac49..e61282171bc 100644 --- a/libs/model/src/comment/SearchComments.query.ts +++ b/libs/model/src/comment/SearchComments.query.ts @@ -48,7 +48,7 @@ export function SearchComments(): Query { } const communityWhere = bind.community - ? '"Comments".community_id = $community AND' + ? '"Threads".community_id = $community AND' : ''; const sqlBaseQuery = ` @@ -63,7 +63,7 @@ export function SearchComments(): Query { "Addresses".community_id as address_community_id, "Comments".created_at, "Threads".community_id as community_id, - ts_rank_cd("Comments"._search, query) as rank + ts_rank_cd("Comments".search, query) as rank FROM "Comments" JOIN "Threads" ON "Comments".thread_id = "Threads".id JOIN "Addresses" ON "Comments".address_id = "Addresses".id, @@ -71,7 +71,7 @@ export function SearchComments(): Query { WHERE ${communityWhere} "Comments".deleted_at IS NULL AND - query @@ "Comments"._search + query @@ "Comments".search ${paginationSort} `; @@ -85,7 +85,7 @@ export function SearchComments(): Query { WHERE ${communityWhere} "Comments".deleted_at IS NULL AND - query @@ "Comments"._search + query @@ "Comments".search `; // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/libs/model/src/comment/UpdateComment.command.ts b/libs/model/src/comment/UpdateComment.command.ts index d7033940731..c2fa679205a 100644 --- a/libs/model/src/comment/UpdateComment.command.ts +++ b/libs/model/src/comment/UpdateComment.command.ts @@ -3,6 +3,7 @@ import * as schemas from '@hicommonwealth/schemas'; import { models } from '../database'; import { isAuthorized, type AuthContext } from '../middleware'; import { mustBeAuthorized } from '../middleware/guards'; +import { getCommentSearchVector } from '../models'; import { emitMentions, findMentionDiff, @@ -47,7 +48,7 @@ export function UpdateComment(): Command< // == mutation transaction boundary == await models.sequelize.transaction(async (transaction) => { await models.Comment.update( - { text, plaintext }, + { text, plaintext, search: getCommentSearchVector(text) }, { where: { id: comment.id }, transaction }, ); diff --git a/libs/model/src/models/comment.ts b/libs/model/src/models/comment.ts index aff1e5f3f77..151828a498f 100644 --- a/libs/model/src/models/comment.ts +++ b/libs/model/src/models/comment.ts @@ -60,6 +60,10 @@ export default ( allowNull: false, defaultValue: 0, }, + search: { + type: Sequelize.TSVECTOR, + allowNull: false, + }, }, { hooks: { @@ -114,3 +118,14 @@ export default ( ], }, ); + +export function getCommentSearchVector(body: string) { + let decodedBody = body; + + try { + decodedBody = decodeURIComponent(body); + // eslint-disable-next-line no-empty + } catch {} + + return Sequelize.fn('to_tsvector', 'english', decodedBody); +} diff --git a/libs/model/src/models/thread.ts b/libs/model/src/models/thread.ts index 03e73887642..8786af806aa 100644 --- a/libs/model/src/models/thread.ts +++ b/libs/model/src/models/thread.ts @@ -98,6 +98,10 @@ export default ( allowNull: true, defaultValue: new Date(), }, + search: { + type: Sequelize.TSVECTOR, + allowNull: false, + }, }, { timestamps: true, @@ -164,3 +168,22 @@ export default ( }, }, ); + +export function getThreadSearchVector(title: string, body: string) { + let decodedTitle = title; + let decodedBody = body; + try { + decodedTitle = decodeURIComponent(title); + // eslint-disable-next-line no-empty + } catch {} + + try { + decodedBody = decodeURIComponent(body); + // eslint-disable-next-line no-empty + } catch {} + return Sequelize.fn( + 'to_tsvector', + 'english', + decodedTitle + ' ' + decodedBody, + ); +} diff --git a/libs/model/src/tester/e2eSeeds.ts b/libs/model/src/tester/e2eSeeds.ts index 047d4e8d1a5..55d05befb1d 100644 --- a/libs/model/src/tester/e2eSeeds.ts +++ b/libs/model/src/tester/e2eSeeds.ts @@ -11,6 +11,7 @@ import type { TopicAttributes, UserInstance, } from '../../src'; +import { getCommentSearchVector, getThreadSearchVector } from '../models'; export type E2E_TestEntities = { testThreads: ThreadInstance[]; @@ -189,6 +190,8 @@ export const e2eTestEntities = async function ( await testDb.Thread.findOrCreate({ where: { id: -i - 1, + }, + defaults: { address_id: -1, title: `testThread Title ${-i - 1}`, body: `testThread Body ${-i - 1}`, @@ -196,6 +199,15 @@ export const e2eTestEntities = async function ( topic_id: -1, kind: 'discussion', plaintext: 'text', + stage: 'discussion', + view_count: 0, + reaction_count: 0, + reaction_weights_sum: 0, + comment_count: 0, + search: getThreadSearchVector( + `testThread Title ${-i - 1}`, + `testThread Body ${-i - 1}`, + ), }, }) )[0], @@ -211,6 +223,8 @@ export const e2eTestEntities = async function ( await testDb.Thread.findOrCreate({ where: { id: -i - 1 - 2, + }, + defaults: { address_id: -2, title: `testThread Title ${-i - 1 - 2}`, body: `testThread Body ${-i - 1 - 2}`, @@ -218,6 +232,15 @@ export const e2eTestEntities = async function ( topic_id: -2, kind: 'discussion', plaintext: 'text', + stage: 'discussion', + view_count: 0, + reaction_count: 0, + reaction_weights_sum: 0, + comment_count: 0, + search: getThreadSearchVector( + `testThread Title ${-i - 1 - 2}`, + `testThread Body ${-i - 1 - 2}`, + ), }, }) )[0], @@ -249,10 +272,14 @@ export const e2eTestEntities = async function ( await testDb.Comment.findOrCreate({ where: { id: -i - 1, + }, + defaults: { address_id: -1, text: '', thread_id: -1, plaintext: '', + reaction_count: 0, + search: getCommentSearchVector(''), }, }) )[0], @@ -268,10 +295,14 @@ export const e2eTestEntities = async function ( await testDb.Comment.findOrCreate({ where: { id: -i - 1 - 2, + }, + defaults: { address_id: -2, text: '', thread_id: -2, plaintext: '', + reaction_count: 0, + search: getCommentSearchVector(''), }, }) )[0], diff --git a/libs/model/src/thread/CreateThread.command.ts b/libs/model/src/thread/CreateThread.command.ts index 62101c10fe5..85fdc438644 100644 --- a/libs/model/src/thread/CreateThread.command.ts +++ b/libs/model/src/thread/CreateThread.command.ts @@ -14,6 +14,7 @@ import { models } from '../database'; import { isAuthorized, type AuthContext } from '../middleware'; import { verifyThreadSignature } from '../middleware/canvas'; import { mustBeAuthorized } from '../middleware/guards'; +import { getThreadSearchVector } from '../models/thread'; import { tokenBalanceCache } from '../services'; import { emitMentions, @@ -131,6 +132,7 @@ export function CreateThread(): Command< comment_count: 0, reaction_count: 0, reaction_weights_sum: 0, + search: getThreadSearchVector(rest.title, body), }, { transaction, diff --git a/libs/model/test/seed/model.spec.ts b/libs/model/test/seed/model.spec.ts index e25307e92d2..e5e5efd0b7e 100644 --- a/libs/model/test/seed/model.spec.ts +++ b/libs/model/test/seed/model.spec.ts @@ -39,8 +39,8 @@ const generateSchemas = async () => { const migration_schema = await get_info_schema(migration, { ignore_columns: { // Missing in model - migrations with backups - Comments: ['body_backup', 'text_backup', 'root_id', '_search'], - Threads: ['body_backup', '_search'], + Comments: ['body_backup', 'text_backup', 'root_id'], + Threads: ['body_backup'], Topics: ['default_offchain_template_backup'], GroupPermissions: ['allowed_actions'], }, diff --git a/libs/model/test/util-tests/getCommentDepth.spec.ts b/libs/model/test/util-tests/getCommentDepth.spec.ts index 51ddeab343c..12d7590e4e4 100644 --- a/libs/model/test/util-tests/getCommentDepth.spec.ts +++ b/libs/model/test/util-tests/getCommentDepth.spec.ts @@ -1,7 +1,13 @@ import { dispose } from '@hicommonwealth/core'; import { expect } from 'chai'; import { afterAll, beforeAll, describe, test } from 'vitest'; -import { CommentInstance, models, tester } from '../../src'; +import { + CommentInstance, + getCommentSearchVector, + getThreadSearchVector, + models, + tester, +} from '../../src'; import { getCommentDepth } from '../../src/utils/getCommentDepth'; describe('getCommentDepth', () => { @@ -23,6 +29,7 @@ describe('getCommentDepth', () => { title: 'Testing', plaintext: '', kind: 'discussion', + search: getThreadSearchVector('Testing', ''), }); let comment: CommentInstance; for (let i = 0; i < maxDepth; i++) { @@ -35,6 +42,7 @@ describe('getCommentDepth', () => { // @ts-expect-error StrictNullChecks address_id: address.id, text: String(i), + search: getCommentSearchVector(String(i)), }); comments.push(result); comment = result; diff --git a/libs/schemas/src/entities/comment.schemas.ts b/libs/schemas/src/entities/comment.schemas.ts index ab857e34131..aea745e76a1 100644 --- a/libs/schemas/src/entities/comment.schemas.ts +++ b/libs/schemas/src/entities/comment.schemas.ts @@ -35,6 +35,8 @@ export const Comment = z.object({ reaction_count: PG_INT, reaction_weights_sum: PG_INT.optional(), + search: z.union([z.string(), z.record(z.any())]), + Address: Address.nullish(), Thread: Thread.nullish(), Reaction: Reaction.nullish(), diff --git a/libs/schemas/src/entities/thread.schemas.ts b/libs/schemas/src/entities/thread.schemas.ts index 4984c1fc871..4c596c459bc 100644 --- a/libs/schemas/src/entities/thread.schemas.ts +++ b/libs/schemas/src/entities/thread.schemas.ts @@ -45,6 +45,8 @@ export const Thread = z.object({ created_by: z.string().nullish(), profile_name: z.string().nullish(), + search: z.union([z.string(), z.record(z.any())]), + // associations Address: Address.nullish(), topic: Topic.nullish(), diff --git a/packages/commonwealth/server/controllers/server_comments_methods/search_comments.ts b/packages/commonwealth/server/controllers/server_comments_methods/search_comments.ts index 8c3745f869f..14b97cae03b 100644 --- a/packages/commonwealth/server/controllers/server_comments_methods/search_comments.ts +++ b/packages/commonwealth/server/controllers/server_comments_methods/search_comments.ts @@ -79,7 +79,7 @@ export async function __searchComments( } const communityWhere = bind.community - ? '"Comments".community_id = $community AND' + ? '"Threads".community_id = $community AND' : ''; const sqlBaseQuery = ` @@ -94,7 +94,7 @@ export async function __searchComments( "Addresses".community_id as address_community_id, "Comments".created_at, "Threads".community_id as community_id, - ts_rank_cd("Comments"._search, query) as rank + ts_rank_cd("Comments".search, query) as rank FROM "Comments" JOIN "Threads" ON "Comments".thread_id = "Threads".id JOIN "Addresses" ON "Comments".address_id = "Addresses".id, @@ -102,7 +102,7 @@ export async function __searchComments( WHERE ${communityWhere} "Comments".deleted_at IS NULL AND - query @@ "Comments"._search + query @@ "Comments".search ${paginationSort} `; @@ -116,7 +116,7 @@ export async function __searchComments( WHERE ${communityWhere} "Comments".deleted_at IS NULL AND - query @@ "Comments"._search + query @@ "Comments".search `; const [results, [{ count }]]: [any[], any[]] = await Promise.all([ diff --git a/packages/commonwealth/server/controllers/server_threads_methods/search_threads.ts b/packages/commonwealth/server/controllers/server_threads_methods/search_threads.ts index d93577e34d8..08903fe1916 100644 --- a/packages/commonwealth/server/controllers/server_threads_methods/search_threads.ts +++ b/packages/commonwealth/server/controllers/server_threads_methods/search_threads.ts @@ -82,7 +82,7 @@ export async function __searchThreads( let searchWhere = `"Threads".title ILIKE '%' || $searchTerm || '%'`; if (!threadTitleOnly) { - searchWhere = `("Threads".title ILIKE '%' || $searchTerm || '%' OR query @@ "Threads"._search)`; + searchWhere = `("Threads".title ILIKE '%' || $searchTerm || '%' OR query @@ "Threads".search)`; } const sqlBaseQuery = ` @@ -97,7 +97,7 @@ export async function __searchThreads( "Addresses".community_id as address_community_id, "Threads".created_at, "Threads".community_id as community_id, - ts_rank_cd("Threads"._search, query) as rank + ts_rank_cd("Threads".search, query) as rank FROM "Threads" JOIN "Addresses" ON "Threads".address_id = "Addresses".id, websearch_to_tsquery('english', $searchTerm) as query diff --git a/packages/commonwealth/server/controllers/server_threads_methods/update_thread.ts b/packages/commonwealth/server/controllers/server_threads_methods/update_thread.ts index eca7e1144f8..338cd8aacd3 100644 --- a/packages/commonwealth/server/controllers/server_threads_methods/update_thread.ts +++ b/packages/commonwealth/server/controllers/server_threads_methods/update_thread.ts @@ -9,6 +9,7 @@ import { UserInstance, emitMentions, findMentionDiff, + getThreadSearchVector, parseUserMentions, } from '@hicommonwealth/model'; import { renderQuillDeltaToText } from '@hicommonwealth/shared'; @@ -114,11 +115,12 @@ export async function __updateThread( // check if thread is part of a contest topic const contestManagers = await this.models.sequelize.query( ` - SELECT cm.contest_address FROM "Threads" t - JOIN "ContestTopics" ct on ct.topic_id = t.topic_id - JOIN "ContestManagers" cm on cm.contest_address = ct.contest_address - WHERE t.id = :thread_id - `, + SELECT cm.contest_address + FROM "Threads" t + JOIN "ContestTopics" ct on ct.topic_id = t.topic_id + JOIN "ContestManagers" cm on cm.contest_address = ct.contest_address + WHERE t.id = :thread_id + `, { type: QueryTypes.SELECT, replacements: { @@ -225,6 +227,7 @@ export async function __updateThread( await thread.update( { ...toUpdate, + search: getThreadSearchVector(title || thread.title, body), last_edited: Sequelize.literal('CURRENT_TIMESTAMP'), }, { transaction }, diff --git a/packages/commonwealth/server/migrations/20240910064652-drop-search-triggers.js b/packages/commonwealth/server/migrations/20240910064652-drop-search-triggers.js new file mode 100644 index 00000000000..41a5134a5b5 --- /dev/null +++ b/packages/commonwealth/server/migrations/20240910064652-drop-search-triggers.js @@ -0,0 +1,50 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.sequelize.transaction(async (transaction) => { + await queryInterface.sequelize.query( + ` + DROP TRIGGER IF EXISTS "OffchainThreads_vector_update" ON "Threads"; + `, + { transaction }, + ); + await queryInterface.renameColumn('Threads', '_search', 'search', { + transaction, + }); + await queryInterface.sequelize.query( + ` + ALTER TABLE "Threads" + ALTER COLUMN search SET NOT NULL; + `, + { transaction }, + ); + await queryInterface.sequelize.query( + ` + DROP TRIGGER IF EXISTS "OffchainComments_vector_update" ON "Comments"; + `, + { transaction }, + ); + await queryInterface.renameColumn('Comments', '_search', 'search', { + transaction, + }); + await queryInterface.sequelize.query( + ` + ALTER TABLE "Comments" + ALTER COLUMN search SET NOT NULL; + `, + { transaction }, + ); + }); + }, + + async down(queryInterface, Sequelize) { + /** + * Add reverting commands here. + * + * Example: + * await queryInterface.dropTable('users'); + */ + }, +}; diff --git a/packages/commonwealth/test/integration/api/threads-query.spec.ts b/packages/commonwealth/test/integration/api/threads-query.spec.ts index 73bb9455c77..63079445d11 100644 --- a/packages/commonwealth/test/integration/api/threads-query.spec.ts +++ b/packages/commonwealth/test/integration/api/threads-query.spec.ts @@ -1,8 +1,15 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { dispose } from '@hicommonwealth/core'; -import { tester, type DB } from '@hicommonwealth/model'; +import { + ThreadAttributes, + getThreadSearchVector, + tester, + type DB, +} from '@hicommonwealth/model'; import chai from 'chai'; import chaiHttp from 'chai-http'; +import { Optional } from 'sequelize'; +import { NullishPropertiesOf } from 'sequelize/lib/utils'; import { afterAll, beforeAll, describe, test } from 'vitest'; chai.use(chaiHttp); @@ -38,13 +45,18 @@ describe('Thread queries', () => { const thread = ( await models.Thread.findOrCreate({ where: { - // @ts-expect-error StrictNullChecks - community_id: chain.id, + community_id: chain!.id, address_id: address.id, title: 'title', kind: 'kind', stage: 'stage', }, + defaults: { + search: getThreadSearchVector('title', ''), + } as unknown as Optional< + ThreadAttributes, + NullishPropertiesOf + >, }) )[0]; expect(thread.id).to.be.greaterThan(0); diff --git a/packages/commonwealth/test/integration/databaseCleaner.spec.ts b/packages/commonwealth/test/integration/databaseCleaner.spec.ts index 95ec675d564..914820db2fe 100644 --- a/packages/commonwealth/test/integration/databaseCleaner.spec.ts +++ b/packages/commonwealth/test/integration/databaseCleaner.spec.ts @@ -1,5 +1,10 @@ import { dispose } from '@hicommonwealth/core'; -import { tester, type DB } from '@hicommonwealth/model'; +import { + getCommentSearchVector, + getThreadSearchVector, + tester, + type DB, +} from '@hicommonwealth/model'; import chai from 'chai'; import chaiHttp from 'chai-http'; import { Sequelize } from 'sequelize'; @@ -184,6 +189,7 @@ describe('DatabaseCleaner Tests', async () => { stage: 'discussion', view_count: 0, comment_count: 0, + search: getThreadSearchVector('Testing', ''), }); const comment = await models.Comment.create({ @@ -193,6 +199,7 @@ describe('DatabaseCleaner Tests', async () => { reaction_count: 0, reaction_weights_sum: 0, plaintext: 'Testing', + search: getCommentSearchVector('Testing'), }); await models.ThreadSubscription.create({