Remove _search dependency on plaintext columns

.github/workflows/CI.yml

@@ -334,9 +334,6 @@ jobs:
       - name: Build
         run: pnpm -r build
 
-      - name: build
-        run: pnpm -r lint
-
       - name: Run unit tests
         run: pnpm -r test -- --allowOnly=false
 

libs/core/src/integration/events.schemas.ts

@@ -19,15 +19,15 @@ import {
 } from './chain-event.schemas';
 import { EventMetadata } from './util.schemas';
 
-export const ThreadCreated = Thread.extend({
+export const ThreadCreated = Thread.omit({ search: true }).extend({
   contestManagers: z.array(z.object({ contest_address: z.string() })).nullish(),
 });
 export const ThreadUpvoted = Reaction.omit({ comment_id: true }).extend({
   thread_id: PG_INT,
   community_id: z.string(),
   contestManagers: z.array(z.object({ contest_address: z.string() })).nullish(),
 });
-export const CommentCreated = Comment.extend({
+export const CommentCreated = Comment.omit({ search: true }).extend({
   community_id: z.string(),
   users_mentioned: z
     .array(PG_INT)

libs/model/src/comment/CreateComment.command.ts

@@ -1,4 +1,5 @@
 import { EventNames, InvalidState, type Command } from '@hicommonwealth/core';
+import { getCommentSearchVector } from '@hicommonwealth/model';
 import * as schemas from '@hicommonwealth/schemas';
 import { models } from '../database';
 import { isAuthorized, type AuthContext } from '../middleware';
@@ -70,6 +71,7 @@ export function CreateComment(): Command<
               reaction_count: 0,
               reaction_weights_sum: 0,
               created_by: '',
+              search: getCommentSearchVector(text),
             },
             {
               transaction,

libs/model/src/comment/SearchComments.query.ts

@@ -48,7 +48,7 @@ export function SearchComments(): Query<typeof schemas.SearchComments> {
       }
 
       const communityWhere = bind.community
-        ? '"Comments".community_id = $community AND'
+        ? '"Threads".community_id = $community AND'
         : '';
 
       const sqlBaseQuery = `
@@ -63,15 +63,15 @@ export function SearchComments(): Query<typeof schemas.SearchComments> {
       "Addresses".community_id as address_community_id,
       "Comments".created_at,
       "Threads".community_id as community_id,
-      ts_rank_cd("Comments"._search, query) as rank
+      ts_rank_cd("Comments".search, query) as rank
     FROM "Comments"
     JOIN "Threads" ON "Comments".thread_id = "Threads".id
     JOIN "Addresses" ON "Comments".address_id = "Addresses".id,
     websearch_to_tsquery('english', $searchTerm) as query
     WHERE
       ${communityWhere}
       "Comments".deleted_at IS NULL AND
-      query @@ "Comments"._search
+      query @@ "Comments".search
     ${paginationSort}
   `;
 
@@ -85,7 +85,7 @@ export function SearchComments(): Query<typeof schemas.SearchComments> {
     WHERE
       ${communityWhere}
       "Comments".deleted_at IS NULL AND
-      query @@ "Comments"._search
+      query @@ "Comments".search
   `;
 
       // eslint-disable-next-line @typescript-eslint/no-explicit-any

libs/model/src/comment/UpdateComment.command.ts

@@ -3,6 +3,7 @@ import * as schemas from '@hicommonwealth/schemas';
 import { models } from '../database';
 import { isAuthorized, type AuthContext } from '../middleware';
 import { mustBeAuthorized } from '../middleware/guards';
+import { getCommentSearchVector } from '../models';
 import {
   emitMentions,
   findMentionDiff,
@@ -47,7 +48,7 @@ export function UpdateComment(): Command<
         // == mutation transaction boundary ==
         await models.sequelize.transaction(async (transaction) => {
           await models.Comment.update(
-            { text, plaintext },
+            { text, plaintext, search: getCommentSearchVector(text) },
             { where: { id: comment.id }, transaction },
           );
 

libs/model/src/models/comment.ts

@@ -60,6 +60,10 @@ export default (
         allowNull: false,
         defaultValue: 0,
       },
+      search: {
+        type: Sequelize.TSVECTOR,
+        allowNull: false,
+      },
     },
     {
       hooks: {
@@ -114,3 +118,14 @@ export default (
       ],
     },
   );
+
+export function getCommentSearchVector(body: string) {
+  let decodedBody = body;
+
+  try {
+    decodedBody = decodeURIComponent(body);
+    // eslint-disable-next-line no-empty
+  } catch {}
+
+  return Sequelize.fn('to_tsvector', 'english', decodedBody);
+}

libs/model/src/models/thread.ts

@@ -98,6 +98,10 @@ export default (
         allowNull: true,
         defaultValue: new Date(),
       },
+      search: {
+        type: Sequelize.TSVECTOR,
+        allowNull: false,
+      },
     },
     {
       timestamps: true,
@@ -164,3 +168,22 @@ export default (
       },
     },
   );
+
+export function getThreadSearchVector(title: string, body: string) {
+  let decodedTitle = title;
+  let decodedBody = body;
+  try {
+    decodedTitle = decodeURIComponent(title);
+    // eslint-disable-next-line no-empty
+  } catch {}
+
+  try {
+    decodedBody = decodeURIComponent(body);
+    // eslint-disable-next-line no-empty
+  } catch {}
+  return Sequelize.fn(
+    'to_tsvector',
+    'english',
+    decodedTitle + ' ' + decodedBody,
+  );
+}

libs/model/src/tester/e2eSeeds.ts

@@ -11,6 +11,7 @@ import type {
   TopicAttributes,
   UserInstance,
 } from '../../src';
+import { getCommentSearchVector, getThreadSearchVector } from '../models';
 
 export type E2E_TestEntities = {
   testThreads: ThreadInstance[];
@@ -189,13 +190,24 @@ export const e2eTestEntities = async function (
               await testDb.Thread.findOrCreate({
                 where: {
                   id: -i - 1,
+                },
+                defaults: {
                   address_id: -1,
                   title: `testThread Title ${-i - 1}`,
                   body: `testThread Body ${-i - 1}`,
                   community_id: 'cmntest',
                   topic_id: -1,
                   kind: 'discussion',
                   plaintext: 'text',
+                  stage: 'discussion',
+                  view_count: 0,
+                  reaction_count: 0,
+                  reaction_weights_sum: 0,
+                  comment_count: 0,
+                  search: getThreadSearchVector(
+                    `testThread Title ${-i - 1}`,
+                    `testThread Body ${-i - 1}`,
+                  ),
                 },
               })
             )[0],
@@ -211,13 +223,24 @@ export const e2eTestEntities = async function (
               await testDb.Thread.findOrCreate({
                 where: {
                   id: -i - 1 - 2,
+                },
+                defaults: {
                   address_id: -2,
                   title: `testThread Title ${-i - 1 - 2}`,
                   body: `testThread Body ${-i - 1 - 2}`,
                   community_id: 'cmntest',
                   topic_id: -2,
                   kind: 'discussion',
                   plaintext: 'text',
+                  stage: 'discussion',
+                  view_count: 0,
+                  reaction_count: 0,
+                  reaction_weights_sum: 0,
+                  comment_count: 0,
+                  search: getThreadSearchVector(
+                    `testThread Title ${-i - 1 - 2}`,
+                    `testThread Body ${-i - 1 - 2}`,
+                  ),
                 },
               })
             )[0],
@@ -249,10 +272,14 @@ export const e2eTestEntities = async function (
               await testDb.Comment.findOrCreate({
                 where: {
                   id: -i - 1,
+                },
+                defaults: {
                   address_id: -1,
                   text: '',
                   thread_id: -1,
                   plaintext: '',
+                  reaction_count: 0,
+                  search: getCommentSearchVector(''),
                 },
               })
             )[0],
@@ -268,10 +295,14 @@ export const e2eTestEntities = async function (
               await testDb.Comment.findOrCreate({
                 where: {
                   id: -i - 1 - 2,
+                },
+                defaults: {
                   address_id: -2,
                   text: '',
                   thread_id: -2,
                   plaintext: '',
+                  reaction_count: 0,
+                  search: getCommentSearchVector(''),
                 },
               })
             )[0],

libs/model/src/thread/CreateThread.command.ts

@@ -14,6 +14,7 @@ import { models } from '../database';
 import { isAuthorized, type AuthContext } from '../middleware';
 import { verifyThreadSignature } from '../middleware/canvas';
 import { mustBeAuthorized } from '../middleware/guards';
+import { getThreadSearchVector } from '../models/thread';
 import { tokenBalanceCache } from '../services';
 import {
   emitMentions,
@@ -131,6 +132,7 @@ export function CreateThread(): Command<
               comment_count: 0,
               reaction_count: 0,
               reaction_weights_sum: 0,
+              search: getThreadSearchVector(rest.title, body),
             },
             {
               transaction,

libs/model/test/seed/model.spec.ts

@@ -39,8 +39,8 @@ const generateSchemas = async () => {
   const migration_schema = await get_info_schema(migration, {
     ignore_columns: {
       // Missing in model - migrations with backups
-      Comments: ['body_backup', 'text_backup', 'root_id', '_search'],
-      Threads: ['body_backup', '_search'],
+      Comments: ['body_backup', 'text_backup', 'root_id'],
+      Threads: ['body_backup'],
       Topics: ['default_offchain_template_backup'],
       GroupPermissions: ['allowed_actions'],
     },

libs/model/test/util-tests/getCommentDepth.spec.ts

@@ -1,7 +1,13 @@
 import { dispose } from '@hicommonwealth/core';
 import { expect } from 'chai';
 import { afterAll, beforeAll, describe, test } from 'vitest';
-import { CommentInstance, models, tester } from '../../src';
+import {
+  CommentInstance,
+  getCommentSearchVector,
+  getThreadSearchVector,
+  models,
+  tester,
+} from '../../src';
 import { getCommentDepth } from '../../src/utils/getCommentDepth';
 
 describe('getCommentDepth', () => {
@@ -23,6 +29,7 @@ describe('getCommentDepth', () => {
       title: 'Testing',
       plaintext: '',
       kind: 'discussion',
+      search: getThreadSearchVector('Testing', ''),
     });
     let comment: CommentInstance;
     for (let i = 0; i < maxDepth; i++) {
@@ -35,6 +42,7 @@ describe('getCommentDepth', () => {
         // @ts-expect-error StrictNullChecks
         address_id: address.id,
         text: String(i),
+        search: getCommentSearchVector(String(i)),
       });
       comments.push(result);
       comment = result;

libs/schemas/src/entities/comment.schemas.ts

@@ -35,6 +35,8 @@ export const Comment = z.object({
   reaction_count: PG_INT,
   reaction_weights_sum: PG_INT.optional(),
 
+  search: z.union([z.string(), z.record(z.any())]),
+
   Address: Address.nullish(),
   Thread: Thread.nullish(),
   Reaction: Reaction.nullish(),

libs/schemas/src/entities/thread.schemas.ts

@@ -45,6 +45,8 @@ export const Thread = z.object({
   created_by: z.string().nullish(),
   profile_name: z.string().nullish(),
 
+  search: z.union([z.string(), z.record(z.any())]),
+
   // associations
   Address: Address.nullish(),
   topic: Topic.nullish(),

packages/commonwealth/server/controllers/server_comments_methods/search_comments.ts

@@ -79,7 +79,7 @@ export async function __searchComments(
   }
 
   const communityWhere = bind.community
-    ? '"Comments".community_id = $community AND'
+    ? '"Threads".community_id = $community AND'
     : '';
 
   const sqlBaseQuery = `
@@ -94,15 +94,15 @@ export async function __searchComments(
       "Addresses".community_id as address_community_id,
       "Comments".created_at,
       "Threads".community_id as community_id,
-      ts_rank_cd("Comments"._search, query) as rank
+      ts_rank_cd("Comments".search, query) as rank
     FROM "Comments"
     JOIN "Threads" ON "Comments".thread_id = "Threads".id
     JOIN "Addresses" ON "Comments".address_id = "Addresses".id,
     websearch_to_tsquery('english', $searchTerm) as query
     WHERE
       ${communityWhere}
       "Comments".deleted_at IS NULL AND
-      query @@ "Comments"._search
+      query @@ "Comments".search
     ${paginationSort}
   `;
 
@@ -116,7 +116,7 @@ export async function __searchComments(
     WHERE
       ${communityWhere}
       "Comments".deleted_at IS NULL AND
-      query @@ "Comments"._search
+      query @@ "Comments".search
   `;
 
   const [results, [{ count }]]: [any[], any[]] = await Promise.all([

packages/commonwealth/server/controllers/server_threads_methods/search_threads.ts

@@ -82,7 +82,7 @@ export async function __searchThreads(
 
   let searchWhere = `"Threads".title ILIKE '%' || $searchTerm || '%'`;
   if (!threadTitleOnly) {
-    searchWhere = `("Threads".title ILIKE '%' || $searchTerm || '%' OR query @@ "Threads"._search)`;
+    searchWhere = `("Threads".title ILIKE '%' || $searchTerm || '%' OR query @@ "Threads".search)`;
   }
 
   const sqlBaseQuery = `
@@ -97,7 +97,7 @@ export async function __searchThreads(
       "Addresses".community_id as address_community_id,
       "Threads".created_at,
       "Threads".community_id as community_id,
-      ts_rank_cd("Threads"._search, query) as rank
+      ts_rank_cd("Threads".search, query) as rank
     FROM "Threads"
     JOIN "Addresses" ON "Threads".address_id = "Addresses".id,
     websearch_to_tsquery('english', $searchTerm) as query