From 9031a35eaa08c192c2da28ab023816eec27ca6ed Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Thu, 24 Oct 2024 13:29:29 +0300 Subject: [PATCH 01/13] truncate bodies async script --- .../async-migrate-comment-version-history.ts | 90 ----------- .../async-migrate-thread-version-history.ts | 122 -------------- .../commonwealth/scripts/truncate-bodies.ts | 153 ++++++++++++++++++ 3 files changed, 153 insertions(+), 212 deletions(-) delete mode 100644 packages/commonwealth/scripts/async-migrate-comment-version-history.ts delete mode 100644 packages/commonwealth/scripts/async-migrate-thread-version-history.ts create mode 100644 packages/commonwealth/scripts/truncate-bodies.ts diff --git a/packages/commonwealth/scripts/async-migrate-comment-version-history.ts b/packages/commonwealth/scripts/async-migrate-comment-version-history.ts deleted file mode 100644 index 2e67ca9c57d..00000000000 --- a/packages/commonwealth/scripts/async-migrate-comment-version-history.ts +++ /dev/null @@ -1,90 +0,0 @@ -//TODO: This should be deleted after comment version histories are fixed -import { dispose } from '@hicommonwealth/core'; -import { CommentVersionHistoryInstance, models } from '@hicommonwealth/model'; -import { QueryTypes } from 'sequelize'; - -async function run() { - const commentCount = ( - await models.sequelize.query( - `SELECT COUNT(*) FROM "Comments" WHERE version_history_updated = false`, - { - raw: true, - type: QueryTypes.SELECT, - }, - ) - )[0]; - - const count = parseInt(commentCount['count']); - let i = 0; - while (i < count) { - try { - await models.sequelize.transaction(async (transaction) => { - const commentVersionHistory: { - id: number; - versionHistories: { timestamp: string; body: string }[]; - }[] = ( - await models.sequelize.query( - `SELECT id, version_history FROM "Comments" where version_history_updated = false - FOR UPDATE SKIP LOCKED LIMIT 1`, - { - raw: true, - type: QueryTypes.SELECT, - transaction, - }, - ) - ).map((c) => ({ - id: parseInt(c['id']), - versionHistories: c['version_history'].map((v) => JSON.parse(v)), - })); - - if (commentVersionHistory.length === 0) { - return; - } - - for (const versionHistory of commentVersionHistory) { - console.log( - `${i}/${count} Updating comment version_histories for id ${versionHistory.id}`, - ); - - const formattedValues = versionHistory.versionHistories.map((v) => { - const { body, ...rest } = v; - return { - comment_id: versionHistory.id, - ...rest, - text: body, - }; - }) as unknown as CommentVersionHistoryInstance[]; - - await models.sequelize.query( - `UPDATE "Comments" SET version_history_updated = true WHERE id = $id`, - { - bind: { id: versionHistory.id }, - transaction, - }, - ); - return await models.CommentVersionHistory.bulkCreate( - formattedValues, - { - transaction, - }, - ); - } - }); - } catch (error) { - console.error('Error:', error.message); - throw error; - } - - i += 1; - } - - console.log('Finished migration'); -} - -run() - .then(() => { - void dispose()('EXIT', true); - }) - .catch((error) => { - console.error('Failed to migrate community counts:', error); - }); diff --git a/packages/commonwealth/scripts/async-migrate-thread-version-history.ts b/packages/commonwealth/scripts/async-migrate-thread-version-history.ts deleted file mode 100644 index a130364c284..00000000000 --- a/packages/commonwealth/scripts/async-migrate-thread-version-history.ts +++ /dev/null @@ -1,122 +0,0 @@ -//TODO: This should be deleted after thread version histories are fixed -import { dispose } from '@hicommonwealth/core'; -import { models, ThreadVersionHistoryAttributes } from '@hicommonwealth/model'; -import { QueryTypes } from 'sequelize'; - -async function run() { - const threadCount = ( - await models.sequelize.query( - `SELECT COUNT(*) FROM "Threads" WHERE version_history_updated = false AND address_id is not NULL`, - { - raw: true, - type: QueryTypes.SELECT, - }, - ) - )[0]; - - const count = parseInt(threadCount['count']); - let i = 0; - while (i < count) { - try { - await models.sequelize.transaction(async (transaction) => { - const threadVersionHistory: { - id: number; - addressId: number; - versionHistories: { - timestamp: string; - body: string; - author: string; - }[]; - }[] = ( - await models.sequelize.query( - `SELECT id, address_id, version_history FROM "Threads" - WHERE version_history_updated = false - AND address_id IS NOT NULL FOR UPDATE SKIP LOCKED LIMIT 1`, - { - raw: true, - type: QueryTypes.SELECT, - transaction, - }, - ) - ).map((c) => ({ - id: parseInt(c['id']), - addressId: parseInt(c['address_id']), - versionHistories: c['version_history'].map((v) => JSON.parse(v)), - })); - - if (threadVersionHistory.length === 0) { - return; - } - - for (const versionHistory of threadVersionHistory) { - console.log( - `${i}/${count} Updating thread version_histories for id ${versionHistory.id}`, - ); - - const formattedValues = (await Promise.all( - versionHistory.versionHistories.map(async (v) => { - const { author, ...rest } = v; - let address = author?.['address']; - - // Address could not have been JSON decoded correctly. - if (!address) { - try { - address = JSON.parse(author)?.address; - } catch (e) { - // do nothing, because we will try to query from database - } - } - - // If still undefined, as last resort get from database - if (!address) { - const result = await models.sequelize.query( - `SELECT address FROM "Addresses" WHERE id = ${versionHistory.addressId}`, - { - type: QueryTypes.SELECT, - raw: true, - }, - ); - - if (result.length > 0) { - address = result[0]['address']; - } - } - - return { - thread_id: versionHistory.id, - ...rest, - address, - }; - }), - )) as unknown as ThreadVersionHistoryAttributes[]; - - await models.sequelize.query( - `UPDATE "Threads" SET version_history_updated = true WHERE id = $id`, - { - bind: { id: versionHistory.id }, - transaction, - }, - ); - return await models.ThreadVersionHistory.bulkCreate(formattedValues, { - transaction, - }); - } - }); - } catch (error) { - console.error('Error:', error.message); - throw error; - } - - i += 1; - } - - console.log('Finished migration'); -} - -run() - .then(() => { - void dispose()('EXIT', true); - }) - .catch((error) => { - console.error('Failed to migrate community counts:', error); - }); diff --git a/packages/commonwealth/scripts/truncate-bodies.ts b/packages/commonwealth/scripts/truncate-bodies.ts new file mode 100644 index 00000000000..e174ad1557d --- /dev/null +++ b/packages/commonwealth/scripts/truncate-bodies.ts @@ -0,0 +1,153 @@ +import { dispose, logger } from '@hicommonwealth/core'; +import { models } from '@hicommonwealth/model'; +import { safeTruncateBody } from '@hicommonwealth/shared'; +import { QueryTypes } from 'sequelize'; + +const log = logger(import.meta); + +const BATCH_SIZE = 10; +const queryCase = 'WHEN id = ? THEN ? '; + +async function truncateText( + { + tableName, + columnName, + }: + | { + tableName: 'Threads' | 'ThreadVersionHistories'; + columnName: 'bodies'; + } + | { + tableName: 'Comments' | 'CommentVersionHistories'; + columnName: 'text'; + }, + lastId = 0, +) { + let lastProcessedId = lastId; + while (true) { + const transaction = await models.sequelize.transaction(); + try { + const records = await models.sequelize.query<{ + id: number; + content: string; + }>( + ` + SELECT id, ${columnName} as content + FROM "${tableName}" + WHERE id > :lastId + AND content_url IS NOT NULL + ORDER BY id + LIMIT :batchSize FOR UPDATE; + `, + { + transaction, + replacements: { + lastId: lastProcessedId, + batchSize: BATCH_SIZE, + }, + type: QueryTypes.SELECT, + }, + ); + + if (records.length === 0) { + await transaction.rollback(); + break; + } + + lastProcessedId = records.at(-1)!.id!; + + let queryCases = ''; + const replacements: (number | string)[] = []; + const recordIds: number[] = []; + for (const { id, content } of records) { + const truncatedContent = safeTruncateBody(content, 2000); + if (content === truncatedContent) continue; + queryCases += queryCase; + replacements.push(id!, truncatedContent); + recordIds.push(id!); + } + + if (replacements.length > 0) { + await models.sequelize.query( + ` + UPDATE "${tableName}" + SET ${columnName} = CASE + ${queryCases} + END + WHERE id IN (?); + `, + { + replacements: [...replacements, recordIds], + type: QueryTypes.BULKUPDATE, + transaction, + }, + ); + } + await transaction.commit(); + log.info( + `Successfully truncated ${tableName} ` + + `${records[0].id} to ${records.at(-1)!.id!}`, + ); + } catch (e) { + log.error('Failed to update', e); + await transaction.rollback(); + break; + } + } +} + +async function main() { + const acceptedArgs = [ + 'threads', + 'comments', + 'thread-versions', + 'comment-versions', + ]; + if (!acceptedArgs.includes(process.argv[2])) { + throw new Error(`Must provide one of: ${JSON.stringify(acceptedArgs)}`); + } + + let lastId = 0; + if (process.argv[3]) { + lastId = parseInt(process.argv[3]); + } + + switch (process.argv[2]) { + case 'threads': + await truncateText( + { tableName: 'Threads', columnName: 'bodies' }, + lastId, + ); + break; + case 'comments': + await truncateText({ tableName: 'Comments', columnName: 'text' }, lastId); + break; + case 'thread-versions': + await truncateText( + { tableName: 'ThreadVersionHistories', columnName: 'bodies' }, + lastId, + ); + break; + case 'comment-versions': + await truncateText( + { tableName: 'CommentVersionHistories', columnName: 'text' }, + lastId, + ); + break; + default: + log.error('Invalid argument!'); + } +} + +if (import.meta.url.endsWith(process.argv[1])) { + main() + .then(() => { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + dispose()('EXIT', true); + }) + .catch((err) => { + console.error(err); + // eslint-disable-next-line @typescript-eslint/no-floating-promises + dispose()('ERROR', true); + }); +} From 9b71dd0659576311d3b1302fa407a6d2ad4fcca8 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Thu, 24 Oct 2024 13:57:43 +0300 Subject: [PATCH 02/13] fix script + delete old script --- .../scripts/migrate-existing-content.ts | 242 ------------------ .../commonwealth/scripts/truncate-bodies.ts | 9 +- 2 files changed, 3 insertions(+), 248 deletions(-) delete mode 100644 packages/commonwealth/scripts/migrate-existing-content.ts diff --git a/packages/commonwealth/scripts/migrate-existing-content.ts b/packages/commonwealth/scripts/migrate-existing-content.ts deleted file mode 100644 index 0a97ed0a1ac..00000000000 --- a/packages/commonwealth/scripts/migrate-existing-content.ts +++ /dev/null @@ -1,242 +0,0 @@ -import { R2BlobStorage } from '@hicommonwealth/adapters'; -import { blobStorage, dispose, logger } from '@hicommonwealth/core'; -import { R2_ADAPTER_KEY, models, uploadIfLarge } from '@hicommonwealth/model'; -import { QueryTypes } from 'sequelize'; - -const log = logger(import.meta); -const BATCH_SIZE = 10; -const queryCase = 'WHEN id = ? THEN ? '; - -async function migrateCommentVersionHistory(lastId = 0) { - let lastVersionHistoryId = lastId; - while (true) { - const transaction = await models.sequelize.transaction(); - try { - const commentVersions = await models.sequelize.query<{ - id: number; - text: string; - }>( - ` - SELECT id, text - FROM "CommentVersionHistories" - WHERE id > :lastId - AND LENGTH(text) > 2000 AND content_url IS NULL - ORDER BY id - LIMIT :batchSize FOR UPDATE; - `, - { - transaction, - replacements: { - lastId: lastVersionHistoryId, - batchSize: BATCH_SIZE, - }, - type: QueryTypes.SELECT, - }, - ); - - if (commentVersions.length === 0) { - await transaction.rollback(); - break; - } - - lastVersionHistoryId = commentVersions.at(-1)!.id!; - - let queryCases = ''; - const replacements: (number | string)[] = []; - const commentVersionIds: number[] = []; - for (const { id, text } of commentVersions) { - const { contentUrl } = await uploadIfLarge('comments', text); - if (!contentUrl) continue; - queryCases += queryCase; - replacements.push(id!, contentUrl); - commentVersionIds.push(id!); - } - - if (replacements.length > 0) { - await models.sequelize.query( - ` - UPDATE "CommentVersionHistories" - SET content_url = CASE - ${queryCases} - END - WHERE id IN (?); - `, - { - replacements: [...replacements, commentVersionIds], - type: QueryTypes.BULKUPDATE, - transaction, - }, - ); - } - await transaction.commit(); - log.info( - 'Successfully uploaded comment version histories ' + - `${commentVersions[0].id} to ${commentVersions.at(-1)!.id!}`, - ); - } catch (e) { - log.error('Failed to update', e); - await transaction.rollback(); - break; - } - break; - } -} - -async function updateComments() { - await models.sequelize.query( - ` - WITH latest_version as (SELECT DISTINCT ON (comment_id) id, comment_id, content_url - FROM "CommentVersionHistories" - WHERE content_url IS NOT NULL - ORDER BY comment_id, timestamp DESC) - UPDATE "Comments" C - SET content_url = LV.content_url - FROM latest_version LV - WHERE C.id = LV.comment_id - `, - { - type: QueryTypes.BULKUPDATE, - }, - ); -} - -async function migrateThreadVersionHistory(lastId: number = 0) { - let lastVersionHistoryId = lastId; - while (true) { - const transaction = await models.sequelize.transaction(); - try { - const threadVersions = await models.sequelize.query<{ - id: number; - body: string; - }>( - ` - SELECT id, body - FROM "ThreadVersionHistories" - WHERE id > :lastId - AND LENGTH(body) > 2000 AND content_url IS NULL - ORDER BY id - LIMIT :batchSize FOR UPDATE; - `, - { - transaction, - replacements: { - lastId: lastVersionHistoryId, - batchSize: BATCH_SIZE, - }, - type: QueryTypes.SELECT, - }, - ); - - if (threadVersions.length === 0) { - await transaction.rollback(); - break; - } - - lastVersionHistoryId = threadVersions.at(-1)!.id!; - - let queryCases = ''; - const replacements: (number | string)[] = []; - const threadVersionIds: number[] = []; - for (const { id, body } of threadVersions) { - const { contentUrl } = await uploadIfLarge('threads', body); - if (!contentUrl) continue; - queryCases += queryCase; - replacements.push(id!, contentUrl); - threadVersionIds.push(id!); - } - - if (replacements.length > 0) { - await models.sequelize.query( - ` - UPDATE "ThreadVersionHistories" - SET content_url = CASE - ${queryCases} - END - WHERE id IN (?); - `, - { - replacements: [...replacements, threadVersionIds], - type: QueryTypes.BULKUPDATE, - transaction, - }, - ); - } - await transaction.commit(); - log.info( - 'Successfully uploaded thread version histories ' + - `${threadVersions[0].id} to ${threadVersions.at(-1)!.id!}`, - ); - } catch (e) { - log.error('Failed to update', e); - await transaction.rollback(); - break; - } - } -} - -/** - * Copies the content_url (if it exists) from the latest thread version history - * to the thread itself - */ -async function updateThreads() { - await models.sequelize.query( - ` - WITH latest_version as (SELECT DISTINCT ON (thread_id) id, thread_id, content_url - FROM "ThreadVersionHistories" - WHERE content_url IS NOT NULL - ORDER BY thread_id, timestamp DESC) - UPDATE "Threads" T - SET content_url = LV.content_url - FROM latest_version LV - WHERE T.id = LV.thread_id - `, - { - type: QueryTypes.BULKUPDATE, - }, - ); -} - -async function main() { - blobStorage({ - key: R2_ADAPTER_KEY, - adapter: R2BlobStorage(), - isDefault: false, - }); - - const acceptedArgs = ['threads', 'comments']; - - if (!acceptedArgs.includes(process.argv[2])) { - log.error(`Must provide one of: ${JSON.stringify(acceptedArgs)}`); - } - - let lastId = 0; - if (process.argv[3]) { - lastId = parseInt(process.argv[3]); - } - - switch (process.argv[2]) { - case 'threads': - await migrateThreadVersionHistory(lastId); - await updateThreads(); - break; - case 'comments': - await migrateCommentVersionHistory(lastId); - await updateComments(); - break; - default: - log.error('Invalid argument!'); - } -} - -if (import.meta.url.endsWith(process.argv[1])) { - main() - .then(() => { - // eslint-disable-next-line @typescript-eslint/no-floating-promises - dispose()('EXIT', true); - }) - .catch((err) => { - console.error(err); - // eslint-disable-next-line @typescript-eslint/no-floating-promises - dispose()('ERROR', true); - }); -} diff --git a/packages/commonwealth/scripts/truncate-bodies.ts b/packages/commonwealth/scripts/truncate-bodies.ts index e174ad1557d..d8803c9291d 100644 --- a/packages/commonwealth/scripts/truncate-bodies.ts +++ b/packages/commonwealth/scripts/truncate-bodies.ts @@ -15,7 +15,7 @@ async function truncateText( }: | { tableName: 'Threads' | 'ThreadVersionHistories'; - columnName: 'bodies'; + columnName: 'body'; } | { tableName: 'Comments' | 'CommentVersionHistories'; @@ -114,17 +114,14 @@ async function main() { switch (process.argv[2]) { case 'threads': - await truncateText( - { tableName: 'Threads', columnName: 'bodies' }, - lastId, - ); + await truncateText({ tableName: 'Threads', columnName: 'body' }, lastId); break; case 'comments': await truncateText({ tableName: 'Comments', columnName: 'text' }, lastId); break; case 'thread-versions': await truncateText( - { tableName: 'ThreadVersionHistories', columnName: 'bodies' }, + { tableName: 'ThreadVersionHistories', columnName: 'body' }, lastId, ); break; From 07007973e63f83d1f9b161aade7bfb7dfd201c0e Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Thu, 24 Oct 2024 17:03:51 +0200 Subject: [PATCH 03/13] truncate thread body --- libs/model/src/models/thread.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/libs/model/src/models/thread.ts b/libs/model/src/models/thread.ts index ab7fa6174d9..08281308f9f 100644 --- a/libs/model/src/models/thread.ts +++ b/libs/model/src/models/thread.ts @@ -1,6 +1,6 @@ import { EventNames } from '@hicommonwealth/core'; import { Thread } from '@hicommonwealth/schemas'; -import { getDecodedString } from '@hicommonwealth/shared'; +import { getDecodedString, safeTruncateBody } from '@hicommonwealth/shared'; import Sequelize from 'sequelize'; import { z } from 'zod'; import { emitEvent, getThreadContestManagers } from '../utils/utils'; @@ -115,6 +115,15 @@ export default ( { fields: ['canvas_msg_id'] }, ], hooks: { + beforeValidate(instance: ThreadInstance) { + if (!instance.body || instance.body.length <= 2_000) return; + + if (!instance.content_url) { + throw new Error( + `content_url must be defined if body length is greater than ${2_000}`, + ); + } else instance.body = safeTruncateBody(instance.body, 2_000); + }, afterCreate: async ( thread: ThreadInstance, options: Sequelize.CreateOptions, From c8ee744fbda0856781d4c8d0c1457a3a9a6712f0 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Thu, 24 Oct 2024 17:48:21 +0200 Subject: [PATCH 04/13] add NOT NULL constraint to thread bodies --- .../20241024154316-not-null-thread-body.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 packages/commonwealth/server/migrations/20241024154316-not-null-thread-body.js diff --git a/packages/commonwealth/server/migrations/20241024154316-not-null-thread-body.js b/packages/commonwealth/server/migrations/20241024154316-not-null-thread-body.js new file mode 100644 index 00000000000..8222d353db5 --- /dev/null +++ b/packages/commonwealth/server/migrations/20241024154316-not-null-thread-body.js @@ -0,0 +1,18 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.sequelize.query(` + ALTER TABLE "Threads" + ALTER COLUMN "body" SET NOT NULL; + `); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.sequelize.query(` + ALTER TABLE "Threads" + ALTER COLUMN "body" DROP NOT NULL; + `); + }, +}; From 8ccb96a414bbddda1dbd53957c9c15f08062ea5a Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Thu, 24 Oct 2024 17:50:17 +0200 Subject: [PATCH 05/13] add beforeValidate hook to thread models --- libs/model/src/models/thread.ts | 13 +++------- .../src/models/thread_version_history.ts | 6 +++++ libs/model/src/models/utils.ts | 26 ++++++++++++++++++- libs/schemas/src/entities/thread.schemas.ts | 2 +- libs/shared/src/constants.ts | 5 ++++ 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/libs/model/src/models/thread.ts b/libs/model/src/models/thread.ts index 08281308f9f..cb1e26aa706 100644 --- a/libs/model/src/models/thread.ts +++ b/libs/model/src/models/thread.ts @@ -1,12 +1,13 @@ import { EventNames } from '@hicommonwealth/core'; import { Thread } from '@hicommonwealth/schemas'; -import { getDecodedString, safeTruncateBody } from '@hicommonwealth/shared'; +import { getDecodedString } from '@hicommonwealth/shared'; import Sequelize from 'sequelize'; import { z } from 'zod'; import { emitEvent, getThreadContestManagers } from '../utils/utils'; import type { CommunityAttributes } from './community'; import type { ThreadSubscriptionAttributes } from './thread_subscriptions'; import type { ModelInstance } from './types'; +import { beforeValidateThreadsHook } from './utils'; export type ThreadAttributes = z.infer & { // associations @@ -25,7 +26,7 @@ export default ( address_id: { type: Sequelize.INTEGER, allowNull: true }, created_by: { type: Sequelize.STRING, allowNull: true }, title: { type: Sequelize.TEXT, allowNull: false }, - body: { type: Sequelize.TEXT, allowNull: true }, + body: { type: Sequelize.TEXT, allowNull: false }, kind: { type: Sequelize.STRING, allowNull: false }, stage: { type: Sequelize.TEXT, @@ -116,13 +117,7 @@ export default ( ], hooks: { beforeValidate(instance: ThreadInstance) { - if (!instance.body || instance.body.length <= 2_000) return; - - if (!instance.content_url) { - throw new Error( - `content_url must be defined if body length is greater than ${2_000}`, - ); - } else instance.body = safeTruncateBody(instance.body, 2_000); + beforeValidateThreadsHook(instance); }, afterCreate: async ( thread: ThreadInstance, diff --git a/libs/model/src/models/thread_version_history.ts b/libs/model/src/models/thread_version_history.ts index 912f4cee62c..41d25fab973 100644 --- a/libs/model/src/models/thread_version_history.ts +++ b/libs/model/src/models/thread_version_history.ts @@ -3,6 +3,7 @@ import Sequelize from 'sequelize'; import { z } from 'zod'; import { ThreadAttributes } from './thread'; import type { ModelInstance } from './types'; +import { beforeValidateThreadsHook } from './utils'; export type ThreadVersionHistoryAttributes = z.infer< typeof ThreadVersionHistory @@ -31,5 +32,10 @@ export default ( tableName: 'ThreadVersionHistories', timestamps: false, indexes: [{ fields: ['thread_id'] }], + hooks: { + beforeValidate(instance: ThreadVersionHistoryInstance) { + beforeValidateThreadsHook(instance); + }, + }, }, ); diff --git a/libs/model/src/models/utils.ts b/libs/model/src/models/utils.ts index 3acfcb6fc57..b4de2a25c99 100644 --- a/libs/model/src/models/utils.ts +++ b/libs/model/src/models/utils.ts @@ -1,4 +1,8 @@ -import { decamelize } from '@hicommonwealth/shared'; +import { + decamelize, + MAX_TRUNCATED_CONTENT_LENGTH, + safeTruncateBody, +} from '@hicommonwealth/shared'; import { Model, Sequelize, @@ -278,3 +282,23 @@ export const syncHooks = { options.logging = false; }, }; + +export const beforeValidateThreadsHook = (instance: { + body: string; + content_url?: string | null | undefined; +}) => { + if (!instance.body || instance.body.length <= MAX_TRUNCATED_CONTENT_LENGTH) + return; + + if (!instance.content_url) { + throw new Error( + 'content_url must be defined if body ' + + `length is greater than ${MAX_TRUNCATED_CONTENT_LENGTH}`, + ); + } else + instance.body = safeTruncateBody( + instance.body, + MAX_TRUNCATED_CONTENT_LENGTH, + ); + return instance; +}; diff --git a/libs/schemas/src/entities/thread.schemas.ts b/libs/schemas/src/entities/thread.schemas.ts index b3f249f56a7..a7a8dd3c2c5 100644 --- a/libs/schemas/src/entities/thread.schemas.ts +++ b/libs/schemas/src/entities/thread.schemas.ts @@ -21,7 +21,7 @@ export const Thread = z.object({ title: z.string(), kind: z.string(), stage: z.string().optional(), - body: z.string().nullish(), + body: z.string(), url: z.string().nullish(), topic_id: PG_INT.nullish(), pinned: z.boolean().nullish(), diff --git a/libs/shared/src/constants.ts b/libs/shared/src/constants.ts index 345f21a5688..d8411400cfd 100644 --- a/libs/shared/src/constants.ts +++ b/libs/shared/src/constants.ts @@ -16,3 +16,8 @@ export const DISCORD_BOT_ADDRESS = '0xdiscordbot'; export const DEFAULT_NAME = 'Anonymous'; export const MAX_RECIPIENTS_PER_WORKFLOW_TRIGGER = 1_000; + +// The maximum number of characters allowed in 'body' and 'text' +// columns of Threads, Comments, and version history models. +// Full content found by fetching from 'content_url'. +export const MAX_TRUNCATED_CONTENT_LENGTH = 2_000; From b1e9b2a95d9e54701359567c899c2b580d0fdf51 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Thu, 24 Oct 2024 18:00:14 +0200 Subject: [PATCH 06/13] add beforeValidate hook to comment models --- libs/model/src/models/comment.ts | 5 ++++- .../src/models/comment_version_history.ts | 6 +++++ libs/model/src/models/utils.ts | 22 +++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/libs/model/src/models/comment.ts b/libs/model/src/models/comment.ts index abd1a1aa24c..234b9f56cd8 100644 --- a/libs/model/src/models/comment.ts +++ b/libs/model/src/models/comment.ts @@ -9,6 +9,7 @@ import type { ReactionAttributes, ThreadInstance, } from '.'; +import { beforeValidateCommentsHook } from './utils'; export type CommentAttributes = z.infer & { // associations @@ -68,6 +69,9 @@ export default ( }, { hooks: { + beforeValidate(instance: CommentInstance) { + beforeValidateCommentsHook(instance); + }, afterCreate: async (comment, options) => { await ( sequelize.models.Thread as Sequelize.ModelStatic @@ -85,7 +89,6 @@ export default ( thread_id: String(comment.thread_id), }); }, - afterDestroy: async ({ thread_id }, options) => { await ( sequelize.models.Thread as Sequelize.ModelStatic diff --git a/libs/model/src/models/comment_version_history.ts b/libs/model/src/models/comment_version_history.ts index aa3bb624550..115a91515fd 100644 --- a/libs/model/src/models/comment_version_history.ts +++ b/libs/model/src/models/comment_version_history.ts @@ -3,6 +3,7 @@ import Sequelize from 'sequelize'; import { z } from 'zod'; import { CommentAttributes } from './comment'; import type { ModelInstance } from './types'; +import { beforeValidateCommentsHook } from './utils'; export type CommentVersionHistoryAttributes = z.infer< typeof CommentVersionHistory @@ -30,5 +31,10 @@ export default ( tableName: 'CommentVersionHistories', timestamps: false, indexes: [{ fields: ['comment_id'] }], + hooks: { + beforeValidate(instance: CommentVersionHistoryInstance) { + beforeValidateCommentsHook(instance); + }, + }, }, ); diff --git a/libs/model/src/models/utils.ts b/libs/model/src/models/utils.ts index b4de2a25c99..4448ff67370 100644 --- a/libs/model/src/models/utils.ts +++ b/libs/model/src/models/utils.ts @@ -302,3 +302,25 @@ export const beforeValidateThreadsHook = (instance: { ); return instance; }; + +// TODO: merge with beforeValidateThreadsHook after +// https://github.com/hicommonwealth/commonwealth/issues/9673 +export const beforeValidateCommentsHook = (instance: { + text: string; + content_url?: string | null | undefined; +}) => { + if (!instance.text || instance.text.length <= MAX_TRUNCATED_CONTENT_LENGTH) + return; + + if (!instance.content_url) { + throw new Error( + 'content_url must be defined if body ' + + `length is greater than ${MAX_TRUNCATED_CONTENT_LENGTH}`, + ); + } else + instance.text = safeTruncateBody( + instance.text, + MAX_TRUNCATED_CONTENT_LENGTH, + ); + return instance; +}; From aea7ce5c607372181b1779bbdf5f255df4d23196 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Fri, 25 Oct 2024 12:38:02 +0200 Subject: [PATCH 07/13] test fixes --- .../test/thread/thread-lifecycle.spec.ts | 20 ++++++++++++------- .../test/util-tests/getCommentDepth.spec.ts | 1 + 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/libs/model/test/thread/thread-lifecycle.spec.ts b/libs/model/test/thread/thread-lifecycle.spec.ts index 4cbeb340cfe..b9ec9f6518a 100644 --- a/libs/model/test/thread/thread-lifecycle.spec.ts +++ b/libs/model/test/thread/thread-lifecycle.spec.ts @@ -283,7 +283,6 @@ describe('Thread lifecycle', () => { payload: await signPayload(actors[role].address!, instancePayload), }); expect(_thread?.title).to.equal(instancePayload.title); - expect(_thread?.body).to.equal(instancePayload.body); expect(_thread?.stage).to.equal(instancePayload.stage); // capture as admin author for other tests if (!thread) thread = _thread!; @@ -296,6 +295,10 @@ describe('Thread lifecycle', () => { key: _thread!.content_url!.split('/').pop()!, }), ).toBeTruthy(); + + expect(_thread?.body).to.equal(instancePayload.body.slice(0, 2000)); + } else { + expect(_thread?.body).to.equal(instancePayload.body); } }); } else { @@ -313,7 +316,7 @@ describe('Thread lifecycle', () => { describe('updates', () => { test('should patch content', async () => { - const body = { + const payloadContent = { title: 'hello', body: chance.paragraph({ sentences: 50 }), canvas_msg_id: '', @@ -323,10 +326,13 @@ describe('Thread lifecycle', () => { actor: actors.admin, payload: { thread_id: thread.id!, - ...body, + ...payloadContent, }, }); - expect(updated).to.contain(body); + expect(updated).to.contain({ + ...payloadContent, + body: payloadContent.body.slice(0, 2000), + }); expect(updated?.content_url).toBeTruthy(); expect( await blobStorage({ key: R2_ADAPTER_KEY }).exists({ @@ -336,15 +342,15 @@ describe('Thread lifecycle', () => { ).toBeTruthy(); expect(updated?.ThreadVersionHistories?.length).to.equal(2); - body.body = 'wasup'; + payloadContent.body = 'wasup'; updated = await command(UpdateThread(), { actor: actors.admin, payload: { thread_id: thread.id!, - ...body, + ...payloadContent, }, }); - expect(updated).to.contain(body); + expect(updated).to.contain(payloadContent); expect(updated?.content_url).toBeFalsy(); expect(updated!.ThreadVersionHistories?.length).to.equal(3); const sortedHistory = updated!.ThreadVersionHistories!.sort( diff --git a/libs/model/test/util-tests/getCommentDepth.spec.ts b/libs/model/test/util-tests/getCommentDepth.spec.ts index 09c438b1938..1adcd2e3fdc 100644 --- a/libs/model/test/util-tests/getCommentDepth.spec.ts +++ b/libs/model/test/util-tests/getCommentDepth.spec.ts @@ -24,6 +24,7 @@ describe('getCommentDepth', () => { }); const thread = await models.Thread.create({ community_id, + body: 'test', address_id: address!.id!, title: 'Testing', kind: 'discussion', From 884c040d89f26ef8ed9c7489b0c5ed52c2893062 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Fri, 25 Oct 2024 12:42:50 +0200 Subject: [PATCH 08/13] more test/type fixes --- .../test/contest-worker/contest-worker-policy.spec.ts | 1 + libs/model/test/thread/thread-lifecycle.spec.ts | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/libs/model/test/contest-worker/contest-worker-policy.spec.ts b/libs/model/test/contest-worker/contest-worker-policy.spec.ts index 47a3b759b2f..c3aa97efbb2 100644 --- a/libs/model/test/contest-worker/contest-worker-policy.spec.ts +++ b/libs/model/test/contest-worker/contest-worker-policy.spec.ts @@ -97,6 +97,7 @@ describe('Contest Worker Policy', () => { canvas_msg_id: '', kind: '', stage: '', + body: '', view_count: 0, reaction_count: 0, reaction_weights_sum: '0', diff --git a/libs/model/test/thread/thread-lifecycle.spec.ts b/libs/model/test/thread/thread-lifecycle.spec.ts index b9ec9f6518a..7865600a1d4 100644 --- a/libs/model/test/thread/thread-lifecycle.spec.ts +++ b/libs/model/test/thread/thread-lifecycle.spec.ts @@ -21,6 +21,7 @@ import * as schemas from '@hicommonwealth/schemas'; import { TopicWeightedVoting } from '@hicommonwealth/schemas'; import { CANVAS_TOPIC, + MAX_TRUNCATED_CONTENT_LENGTH, getTestSigner, sign, toCanvasSignedDataApiArgs, @@ -296,7 +297,9 @@ describe('Thread lifecycle', () => { }), ).toBeTruthy(); - expect(_thread?.body).to.equal(instancePayload.body.slice(0, 2000)); + expect(_thread?.body).to.equal( + instancePayload.body.slice(0, MAX_TRUNCATED_CONTENT_LENGTH), + ); } else { expect(_thread?.body).to.equal(instancePayload.body); } @@ -331,7 +334,7 @@ describe('Thread lifecycle', () => { }); expect(updated).to.contain({ ...payloadContent, - body: payloadContent.body.slice(0, 2000), + body: payloadContent.body.slice(0, MAX_TRUNCATED_CONTENT_LENGTH), }); expect(updated?.content_url).toBeTruthy(); expect( @@ -578,7 +581,7 @@ describe('Thread lifecycle', () => { }); expect(firstComment).to.include({ thread_id: thread!.id, - text, + text: text.slice(0, MAX_TRUNCATED_CONTENT_LENGTH), community_id: thread!.community_id, }); expect(firstComment?.content_url).toBeTruthy(); From d9c5a88e785cd13efcc79b910c6e7c2efb84d5b1 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Fri, 25 Oct 2024 12:46:39 +0200 Subject: [PATCH 09/13] type fix --- packages/commonwealth/test/integration/databaseCleaner.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/commonwealth/test/integration/databaseCleaner.spec.ts b/packages/commonwealth/test/integration/databaseCleaner.spec.ts index 14bb2506f88..90b876c3e0a 100644 --- a/packages/commonwealth/test/integration/databaseCleaner.spec.ts +++ b/packages/commonwealth/test/integration/databaseCleaner.spec.ts @@ -182,6 +182,7 @@ describe('DatabaseCleaner Tests', async () => { const thread = await models.Thread.create({ address_id: address.id!, title: 'Testing', + body: 'test', community_id: 'ethereum', reaction_count: 0, reaction_weights_sum: '0', From b7b224c3f2ba64dca002ec8d7b3e35d87e5b14f6 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Fri, 25 Oct 2024 12:47:24 +0200 Subject: [PATCH 10/13] test fix --- libs/model/test/thread/thread-lifecycle.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/model/test/thread/thread-lifecycle.spec.ts b/libs/model/test/thread/thread-lifecycle.spec.ts index 7865600a1d4..6d86e3e8ad1 100644 --- a/libs/model/test/thread/thread-lifecycle.spec.ts +++ b/libs/model/test/thread/thread-lifecycle.spec.ts @@ -722,7 +722,7 @@ describe('Thread lifecycle', () => { }); expect(updated).to.include({ thread_id: thread!.id, - text, + text: text.slice(0, MAX_TRUNCATED_CONTENT_LENGTH), community_id: thread!.community_id, }); expect(updated?.content_url).toBeTruthy(); From 905e244cc4dbce0a6b17f3f8caebf55e51993141 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Fri, 25 Oct 2024 12:55:30 +0200 Subject: [PATCH 11/13] integration test fix --- packages/commonwealth/test/integration/api/threads-query.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/commonwealth/test/integration/api/threads-query.spec.ts b/packages/commonwealth/test/integration/api/threads-query.spec.ts index 63079445d11..36e2f95af6e 100644 --- a/packages/commonwealth/test/integration/api/threads-query.spec.ts +++ b/packages/commonwealth/test/integration/api/threads-query.spec.ts @@ -50,6 +50,7 @@ describe('Thread queries', () => { title: 'title', kind: 'kind', stage: 'stage', + body: '', }, defaults: { search: getThreadSearchVector('title', ''), From 9f5b1590afdb094a97ff6a13ee1d431a27b989f2 Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Fri, 1 Nov 2024 11:42:06 +0100 Subject: [PATCH 12/13] merge utils --- libs/model/src/models/comment.ts | 4 ++-- .../src/models/comment_version_history.ts | 4 ++-- libs/model/src/models/thread.ts | 4 ++-- .../src/models/thread_version_history.ts | 4 ++-- libs/model/src/models/utils.ts | 24 +------------------ .../src/utils/validateGroupMembership.ts | 1 + 6 files changed, 10 insertions(+), 31 deletions(-) diff --git a/libs/model/src/models/comment.ts b/libs/model/src/models/comment.ts index e62e56902ea..d299ebe1eb1 100644 --- a/libs/model/src/models/comment.ts +++ b/libs/model/src/models/comment.ts @@ -9,7 +9,7 @@ import type { ReactionAttributes, ThreadInstance, } from '.'; -import { beforeValidateCommentsHook } from './utils'; +import { beforeValidateBodyHook } from './utils'; export type CommentAttributes = z.infer & { // associations @@ -70,7 +70,7 @@ export default ( { hooks: { beforeValidate(instance: CommentInstance) { - beforeValidateCommentsHook(instance); + beforeValidateBodyHook(instance); }, afterCreate: async (comment, options) => { await ( diff --git a/libs/model/src/models/comment_version_history.ts b/libs/model/src/models/comment_version_history.ts index 4a17d139642..c074d324674 100644 --- a/libs/model/src/models/comment_version_history.ts +++ b/libs/model/src/models/comment_version_history.ts @@ -3,7 +3,7 @@ import Sequelize from 'sequelize'; import { z } from 'zod'; import { CommentAttributes } from './comment'; import type { ModelInstance } from './types'; -import { beforeValidateCommentsHook } from './utils'; +import { beforeValidateBodyHook } from './utils'; export type CommentVersionHistoryAttributes = z.infer< typeof CommentVersionHistory @@ -33,7 +33,7 @@ export default ( indexes: [{ fields: ['comment_id'] }], hooks: { beforeValidate(instance: CommentVersionHistoryInstance) { - beforeValidateCommentsHook(instance); + beforeValidateBodyHook(instance); }, }, }, diff --git a/libs/model/src/models/thread.ts b/libs/model/src/models/thread.ts index 64098485b7c..6df44779640 100644 --- a/libs/model/src/models/thread.ts +++ b/libs/model/src/models/thread.ts @@ -8,7 +8,7 @@ import { AddressAttributes } from './address'; import type { CommunityAttributes } from './community'; import type { ThreadSubscriptionAttributes } from './thread_subscriptions'; import type { ModelInstance } from './types'; -import { beforeValidateThreadsHook } from './utils'; +import { beforeValidateBodyHook } from './utils'; export type ThreadAttributes = z.infer & { // associations @@ -118,7 +118,7 @@ export default ( ], hooks: { beforeValidate(instance: ThreadInstance) { - beforeValidateThreadsHook(instance); + beforeValidateBodyHook(instance); }, afterCreate: async ( thread: ThreadInstance, diff --git a/libs/model/src/models/thread_version_history.ts b/libs/model/src/models/thread_version_history.ts index 41d25fab973..11b0234e42f 100644 --- a/libs/model/src/models/thread_version_history.ts +++ b/libs/model/src/models/thread_version_history.ts @@ -3,7 +3,7 @@ import Sequelize from 'sequelize'; import { z } from 'zod'; import { ThreadAttributes } from './thread'; import type { ModelInstance } from './types'; -import { beforeValidateThreadsHook } from './utils'; +import { beforeValidateBodyHook } from './utils'; export type ThreadVersionHistoryAttributes = z.infer< typeof ThreadVersionHistory @@ -34,7 +34,7 @@ export default ( indexes: [{ fields: ['thread_id'] }], hooks: { beforeValidate(instance: ThreadVersionHistoryInstance) { - beforeValidateThreadsHook(instance); + beforeValidateBodyHook(instance); }, }, }, diff --git a/libs/model/src/models/utils.ts b/libs/model/src/models/utils.ts index 4448ff67370..ed37f5fddb6 100644 --- a/libs/model/src/models/utils.ts +++ b/libs/model/src/models/utils.ts @@ -283,7 +283,7 @@ export const syncHooks = { }, }; -export const beforeValidateThreadsHook = (instance: { +export const beforeValidateBodyHook = (instance: { body: string; content_url?: string | null | undefined; }) => { @@ -302,25 +302,3 @@ export const beforeValidateThreadsHook = (instance: { ); return instance; }; - -// TODO: merge with beforeValidateThreadsHook after -// https://github.com/hicommonwealth/commonwealth/issues/9673 -export const beforeValidateCommentsHook = (instance: { - text: string; - content_url?: string | null | undefined; -}) => { - if (!instance.text || instance.text.length <= MAX_TRUNCATED_CONTENT_LENGTH) - return; - - if (!instance.content_url) { - throw new Error( - 'content_url must be defined if body ' + - `length is greater than ${MAX_TRUNCATED_CONTENT_LENGTH}`, - ); - } else - instance.text = safeTruncateBody( - instance.text, - MAX_TRUNCATED_CONTENT_LENGTH, - ); - return instance; -}; diff --git a/libs/model/src/utils/validateGroupMembership.ts b/libs/model/src/utils/validateGroupMembership.ts index 4a030456b8d..f4d2d136697 100644 --- a/libs/model/src/utils/validateGroupMembership.ts +++ b/libs/model/src/utils/validateGroupMembership.ts @@ -19,6 +19,7 @@ export type ValidateGroupMembershipResponse = { * @param userAddress address of user * @param requirements An array of requirement types to be validated against * @param balances address balances + * @param numRequiredRequirements * @returns ValidateGroupMembershipResponse validity and messages on requirements that failed */ export function validateGroupMembership( From 6f40581869cb999e06e10553c402b5f2909d5bca Mon Sep 17 00:00:00 2001 From: Timothee Legros Date: Fri, 1 Nov 2024 18:01:03 +0100 Subject: [PATCH 13/13] fix script comment.text -> comment.body --- .../commonwealth/scripts/truncate-bodies.ts | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/packages/commonwealth/scripts/truncate-bodies.ts b/packages/commonwealth/scripts/truncate-bodies.ts index d8803c9291d..fae5e35ebcd 100644 --- a/packages/commonwealth/scripts/truncate-bodies.ts +++ b/packages/commonwealth/scripts/truncate-bodies.ts @@ -12,15 +12,14 @@ async function truncateText( { tableName, columnName, - }: - | { - tableName: 'Threads' | 'ThreadVersionHistories'; - columnName: 'body'; - } - | { - tableName: 'Comments' | 'CommentVersionHistories'; - columnName: 'text'; - }, + }: { + tableName: + | 'Threads' + | 'ThreadVersionHistories' + | 'Comments' + | 'CommentVersionHistories'; + columnName: 'body'; + }, lastId = 0, ) { let lastProcessedId = lastId; @@ -117,7 +116,7 @@ async function main() { await truncateText({ tableName: 'Threads', columnName: 'body' }, lastId); break; case 'comments': - await truncateText({ tableName: 'Comments', columnName: 'text' }, lastId); + await truncateText({ tableName: 'Comments', columnName: 'body' }, lastId); break; case 'thread-versions': await truncateText( @@ -127,7 +126,7 @@ async function main() { break; case 'comment-versions': await truncateText( - { tableName: 'CommentVersionHistories', columnName: 'text' }, + { tableName: 'CommentVersionHistories', columnName: 'body' }, lastId, ); break;