-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #123 from TogetherCrew/122-capture-webhook-messages
122 capture webhook messages
- Loading branch information
Showing
8 changed files
with
1,836 additions
and
1,204 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,6 @@ | ||
| # env vars | ||
| src/config/*.env | ||
| *.env | ||
|
|
||
| # compiled output | ||
| /lib | ||
|
|
||
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
36 changes: 36 additions & 0 deletions
36
src/migrations/db/1695198084420-add-isgeneratedbyweebhook-to-rawinfo-schema.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| import 'dotenv/config'; | ||
| import { Client, GatewayIntentBits, } from 'discord.js'; | ||
| import { guildService } from '../../database/services'; | ||
| import { connectDB } from '../../database'; | ||
| import { databaseService } from '@togethercrew.dev/db'; | ||
| import config from '../../config'; | ||
| import { closeConnection } from '../../database/connection'; | ||
| import webhookLogic from '../utils/webhookLogic'; | ||
|
|
||
| const { | ||
| Guilds, | ||
| GuildMembers, | ||
| GuildMessages, | ||
| GuildPresences, | ||
| DirectMessages | ||
| } = GatewayIntentBits; | ||
|
|
||
|
|
||
| export const up = async () => { | ||
| const client = new Client({ | ||
| intents: [Guilds, GuildMembers, GuildMessages, GuildPresences, DirectMessages], | ||
| }); | ||
|
|
||
| await client.login(config.discord.botToken); | ||
| await connectDB(); | ||
| const guilds = await guildService.getGuilds({}); | ||
| for (let i = 0; i < guilds.length; i++) { | ||
| const connection = databaseService.connectionFactory(guilds[i].guildId, config.mongoose.dbURL); | ||
| await webhookLogic(connection, client, guilds[i].guildId); | ||
| await closeConnection(connection); | ||
| } | ||
| }; | ||
|
|
||
| export const down = async () => { | ||
| // TODO: Implement rollback logic if needed | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| import { connectDB } from '../../database'; | ||
| import { databaseService } from '@togethercrew.dev/db'; | ||
| import 'dotenv/config'; | ||
| import config from '../../config'; | ||
|
|
||
| export const up = async () => { | ||
| await connectDB(); | ||
| const connection = databaseService.connectionFactory("681946187490000803", config.mongoose.dbURL); | ||
| await connection.createCollection('my_collection'); | ||
| }; | ||
|
|
||
| export const down = async () => { | ||
| await connectDB() | ||
|
|
||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| // eslint-disable-next-line @typescript-eslint/no-var-requires | ||
| const tsNode = require('ts-node'); | ||
| module.exports = tsNode.register; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,194 @@ | ||
| import { TextChannel, Message, ThreadChannel, Snowflake, Client } from 'discord.js'; | ||
| import { IRawInfo } from '@togethercrew.dev/db'; | ||
| import { Connection } from 'mongoose'; | ||
| import parentLogger from '../../config/logger'; | ||
| import { rawInfoService, channelService } from '../../database/services'; | ||
|
|
||
| const logger = parentLogger.child({ module: 'Migration' }); | ||
|
|
||
| interface FetchOptions { | ||
| limit: number; | ||
| before?: Snowflake; | ||
| after?: Snowflake; | ||
| } | ||
|
|
||
| async function fetchMessagesBetweenOldestAndNewest( | ||
| connection: Connection, | ||
| channel: TextChannel | ThreadChannel, | ||
| oldestRawInfo: IRawInfo, | ||
| newestRawInfo: IRawInfo | ||
| ) { | ||
| try { | ||
| let allMessages: Message[] = []; | ||
| logger.info( | ||
| { guild_id: connection.name, channel_id: channel.id }, | ||
| 'Fetching channel messages is running' | ||
| ); | ||
| const options: FetchOptions = { limit: 100 }; | ||
| options.after = oldestRawInfo.messageId; | ||
| let fetchedMessages = await channel.messages.fetch(options); | ||
| while (fetchedMessages.size > 0) { | ||
| allMessages = allMessages.concat(Array.from(fetchedMessages.values())); | ||
| if (fetchedMessages.has(newestRawInfo.messageId)) { | ||
| break; | ||
| } | ||
| options.after = fetchedMessages.first()?.id; | ||
| fetchedMessages = await channel.messages.fetch(options); | ||
| } | ||
| return allMessages; | ||
| } catch (err) { | ||
| logger.error( | ||
| { guild_id: connection.name, channel_id: channel.id, err }, | ||
| 'Fetching channel messages failed' | ||
| ); | ||
| } | ||
| logger.info( | ||
| { guild_id: connection.name, channel_id: channel.id }, | ||
| 'Fetching channel messages is done' | ||
| ); | ||
| } | ||
|
|
||
| async function migrateIsGeneratedByWebhook(connection: Connection, channel: TextChannel) { | ||
| try { | ||
| logger.info({ guild_id: connection.name, channel_id: channel.id }, 'Migration for isGeneratedByWebhook is running'); | ||
|
|
||
| // Fetch oldest rawInfo from DB | ||
| const oldestChannelRawInfo = await rawInfoService.getOldestRawInfo(connection, { | ||
| channelId: channel?.id, | ||
| threadId: null, | ||
| }); | ||
|
|
||
| // Fetch newest rawInfo from DB | ||
| const newestChannelRawInfo = await rawInfoService.getNewestRawInfo(connection, { | ||
| channelId: channel?.id, | ||
| threadId: null, | ||
| }); | ||
|
|
||
| if (!oldestChannelRawInfo || !newestChannelRawInfo) { | ||
| logger.info({ guild_id: connection.name, channel_id: channel.id }, 'No oldest rawInfo found, skipping migration'); | ||
| return; | ||
| } | ||
|
|
||
|
|
||
|
|
||
| const fetchedMessages = await fetchMessagesBetweenOldestAndNewest(connection, channel, oldestChannelRawInfo, newestChannelRawInfo); | ||
| const messagesToUpdateTrue = []; | ||
| const messagesToUpdateFalse = []; | ||
|
|
||
| const oldestMessage = await channel.messages.fetch(oldestChannelRawInfo.messageId); | ||
| const newestMessage = await channel.messages.fetch(newestChannelRawInfo.messageId); | ||
|
|
||
|
|
||
| if (oldestMessage.webhookId) messagesToUpdateTrue.push(oldestMessage.id); | ||
| else messagesToUpdateFalse.push(oldestMessage.id); | ||
|
|
||
| if (newestMessage.webhookId) messagesToUpdateTrue.push(newestMessage.id); | ||
| else messagesToUpdateFalse.push(newestMessage.id); | ||
|
|
||
| if (fetchedMessages) { | ||
| for (const message of fetchedMessages) { | ||
| if (message.webhookId) { | ||
| messagesToUpdateTrue.push(message.id); | ||
| } else { | ||
| messagesToUpdateFalse.push(message.id); | ||
| } | ||
| } | ||
|
|
||
| } | ||
|
|
||
| if (messagesToUpdateTrue.length > 0) { | ||
| await rawInfoService.updateManyRawInfo(connection, { messageId: { $in: messagesToUpdateTrue } }, { isGeneratedByWebhook: true }); | ||
| } | ||
|
|
||
| if (messagesToUpdateFalse.length > 0) { | ||
|
|
||
| await rawInfoService.updateManyRawInfo(connection, { messageId: { $in: messagesToUpdateFalse } }, { isGeneratedByWebhook: false }); | ||
| } | ||
|
|
||
| const threads = channel.threads.cache.values(); | ||
|
|
||
| // Handle threads of the channel | ||
| for (const thread of threads) { | ||
| const oldestThreadRawInfo = await rawInfoService.getOldestRawInfo(connection, { | ||
| channelId: channel?.id, | ||
| threadId: thread.id, | ||
| }); | ||
|
|
||
| const newestThreadRawInfo = await rawInfoService.getNewestRawInfo(connection, { | ||
| channelId: channel?.id, | ||
| threadId: thread.id, | ||
| }); | ||
|
|
||
| if (!oldestThreadRawInfo || !newestThreadRawInfo) { | ||
| continue; // No data to migrate for this thread | ||
| } | ||
|
|
||
| const fetchedThreadMessages = await fetchMessagesBetweenOldestAndNewest(connection, thread, oldestThreadRawInfo, newestThreadRawInfo); | ||
|
|
||
| const threadMessagesToUpdateTrue = []; | ||
| const threadMessagesToUpdateFalse = []; | ||
|
|
||
|
|
||
| const oldestThreadMessage = await thread.messages.fetch(oldestThreadRawInfo.messageId); | ||
| const newestThreadMessage = await thread.messages.fetch(newestThreadRawInfo.messageId); | ||
|
|
||
| if (oldestThreadMessage.webhookId) threadMessagesToUpdateTrue.push(oldestThreadMessage.id); | ||
| else threadMessagesToUpdateFalse.push(oldestThreadMessage.id); | ||
|
|
||
| if (newestThreadMessage.webhookId) threadMessagesToUpdateTrue.push(newestThreadMessage.id); | ||
| else threadMessagesToUpdateFalse.push(newestThreadMessage.id); | ||
|
|
||
|
|
||
|
|
||
| if (fetchedThreadMessages) { | ||
| for (const message of fetchedThreadMessages) { | ||
| if (message.webhookId) { | ||
| threadMessagesToUpdateTrue.push(message.id); | ||
| } else { | ||
| threadMessagesToUpdateFalse.push(message.id); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| if (threadMessagesToUpdateTrue.length > 0) { | ||
| await rawInfoService.updateManyRawInfo(connection, { messageId: { $in: threadMessagesToUpdateTrue } }, { isGeneratedByWebhook: true }); | ||
| } | ||
|
|
||
| if (threadMessagesToUpdateFalse.length > 0) { | ||
| await rawInfoService.updateManyRawInfo(connection, { messageId: { $in: threadMessagesToUpdateFalse } }, { isGeneratedByWebhook: false }); | ||
| } | ||
| } | ||
|
|
||
| logger.info({ guild_id: connection.name, channel_id: channel.id }, 'Migration for isGeneratedByWebhook is done'); | ||
|
|
||
| } catch (err) { | ||
| logger.error({ guild_id: connection.name, channel_id: channel.id, err }, 'Migration for isGeneratedByWebhook failed'); | ||
| } | ||
| } | ||
|
|
||
|
|
||
| /** | ||
| * | ||
| * @param {Connection} connection - Mongoose connection object for the database. | ||
| * @param {Client} client - The discord.js client object used to fetch the guild. | ||
| * @param {Snowflake} guildId - The identifier of the guild to extract information from. | ||
| */ | ||
| async function runRawInfoMigration(connection: Connection, client: Client, guildId: Snowflake) { | ||
| logger.info({ guild_id: guildId }, 'Migration is running'); | ||
| try { | ||
| const guild = await client.guilds.fetch(guildId); | ||
| const channels = await channelService.getChannels(connection, {}); | ||
| for (let i = 0; i < channels.length; i++) { | ||
| const channel = await guild.channels.fetch(channels[i].channelId); | ||
| if (channel) { | ||
| if (channel.type !== 0) continue; | ||
| await migrateIsGeneratedByWebhook(connection, channel); | ||
| } | ||
| } | ||
| } catch (err) { | ||
| logger.error({ guild_id: guildId, err }, 'Migration is failed'); | ||
| } | ||
| logger.info({ guild_id: guildId }, 'Migration is done'); | ||
| } | ||
|
|
||
| export default runRawInfoMigration; |