Skip to content
This repository has been archived by the owner on Jul 15, 2024. It is now read-only.

Commit

Permalink
feat: Use bulk rows insert (#358)
Browse files Browse the repository at this point in the history 
Co-authored-by: KagChi <[email protected]>
Co-authored-by: Vann <[email protected]>
  • Loading branch information
@Hazmi35 @KagChi @Vann-Dev
3 people authored Feb 20, 2024
1 parent 7191fc4 commit a9d5448
Show file tree
Hide file tree
Showing 3 changed files with 320 additions and 280 deletions.
72 changes: 36 additions & 36 deletions services/kanao-gateway/.example.env
View file
Original file line number Diff line number Diff line change
@@ -1,43 +1,43 @@
GATEWAY_LARGE_TRESHOLD = 250
GATEWAY_PRESENCE_NAME = Standalone Gateway
GATEWAY_PRESENCE_TYPE = 0
GATEWAY_INTENTS = 0
GATEWAY_SHARD_START = 0
GATEWAY_SHARD_END = 1
GATEWAY_SHARD_COUNT = 1
GATEWAY_SHARD_COUNT_PER_REPLICA = 1
GATEWAY_SHARDS_PERWORKERS = 14
GATEWAY_HELLO_TIMEOUT = 600000
GATEWAY_READY_TIMEOUT = 600000
GATEWAY_HANDSHAKE_TIMEOUT =
GATEWAY_GUILDS_PER_SHARD =
GATEWAY_RESUME = true
GATEWAY_LARGE_TRESHOLD=250
GATEWAY_PRESENCE_NAME=Standalone Gateway
GATEWAY_PRESENCE_TYPE=0
GATEWAY_INTENTS=0
GATEWAY_SHARD_START=0
GATEWAY_SHARD_END=1
GATEWAY_SHARD_COUNT=1
GATEWAY_SHARD_COUNT_PER_REPLICA=1
GATEWAY_SHARDS_PERWORKERS=14
GATEWAY_HELLO_TIMEOUT=600000
GATEWAY_READY_TIMEOUT=600000
GATEWAY_HANDSHAKE_TIMEOUT=
GATEWAY_GUILDS_PER_SHARD=
GATEWAY_RESUME=true

DISCORD_TOKEN =
HTTP_PROXY =
DISCORD_TOKEN=
HTTP_PROXY=

DATABASE_URL =
DATABASE_URL=

AMQP_HOST = "amqp://guest:guest@localhost"
AMQP_HOST="amqp://guest:guest@localhost"

STATE_MESSAGE =
STATE_MEMBER =
STATE_VOICE =
STATE_CHANNEL =
STATE_PRESENCE =
STATE_STICKER =
STATE_REACTION =
STATE_EMOJI =
STATE_THREAD =
STATE_USER =
STATE_THREAD_MEMBER =
STATE_ROLE =
STATE_MESSAGE=
STATE_MEMBER=
STATE_VOICE=
STATE_CHANNEL=
STATE_PRESENCE=
STATE_STICKER=
STATE_REACTION=
STATE_EMOJI=
STATE_THREAD=
STATE_USER=
STATE_THREAD_MEMBER=
STATE_ROLE=

PROMETHEUS_PORT = 9090
PROMETHEUS_ENABLED = false
PROMETHEUS_PORT=9090
PROMETHEUS_ENABLED=false

STORE_LOGS = true
LOKI_HOST = http://localhost:3100 # (optional)
STORE_LOGS=true
LOKI_HOST=http://localhost:3100 # (optional)

REPLICA_ID = 0
REPLICA_COUNT = 1
REPLICA_ID=0
REPLICA_COUNT=1
121 changes: 81 additions & 40 deletions services/kanao-gateway/src/Listeners/Caches/GuildMembers/GuildMembersChunkListener.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ import { Buffer } from "node:buffer";
import { RabbitMQ } from "@nezuchan/constants";
import { memberRoles, members, users } from "@nezuchan/kanao-schema";
import { RoutingKey } from "@nezuchan/utilities";
import { chunk } from "@sapphire/utilities";
import type { GatewayGuildMembersChunkDispatch } from "discord-api-types/v10";
import { GatewayDispatchEvents } from "discord-api-types/v10";
import { sql } from "drizzle-orm";
import type { ListenerContext } from "../../../Stores/Listener.js";
import { Listener } from "../../../Stores/Listener.js";
import { clientId, stateMembers, stateUsers } from "../../../config.js";
Expand All @@ -16,48 +18,87 @@ export class GuildMembersChunkListener extends Listener {
}

public async run(payload: { data: GatewayGuildMembersChunkDispatch; shardId: number; }): Promise<void> {
if (stateMembers || stateUsers) {
for (const member of payload.data.d.members) {
if (stateUsers) {
await this.store.drizzle.insert(users).values({
id: member.user!.id,
username: member.user!.username,
discriminator: member.user?.discriminator ?? null,
globalName: member.user?.global_name ?? null,
avatar: member.user?.avatar ?? null,
bot: member.user?.bot ?? false,
flags: member.user?.flags,
accentColor: member.user?.accent_color,
avatarDecoration: member.user?.avatar_decoration,
banner: member.user?.banner,
locale: member.user?.locale,
mfaEnabled: member.user?.mfa_enabled,
premiumType: member.user?.premium_type,
publicFlags: member.user?.public_flags
}).onConflictDoNothing({ target: users.id });
}
const chunks = chunk(payload.data.d.members, 1_000);

if (stateMembers) {
await this.store.drizzle.insert(members).values({
id: member.user!.id,
avatar: member.avatar,
communicationDisabledUntil: member.premium_since,
deaf: member.deaf,
flags: member.flags,
joinedAt: member.joined_at,
mute: member.mute,
nick: member.nick,
pending: member.pending,
premiumSince: member.premium_since
}).onConflictDoNothing({ target: members.id });
}
for (const memberChunk of chunks) {
if (stateUsers) {
await this.store.drizzle.insert(users)
.values(
memberChunk.map(member => ({
id: member.user!.id,
username: member.user!.username,
discriminator: member.user?.discriminator ?? null,
globalName: member.user?.global_name ?? null,
avatar: member.user?.avatar ?? null,
bot: member.user?.bot ?? false,
flags: member.user?.flags,
accentColor: member.user?.accent_color,
avatarDecoration: member.user?.avatar_decoration,
banner: member.user?.banner,
locale: member.user?.locale,
mfaEnabled: member.user?.mfa_enabled,
premiumType: member.user?.premium_type,
publicFlags: member.user?.public_flags
}))
).onConflictDoUpdate({
target: users.id,
set: {
username: sql`EXCLUDED.username`,
discriminator: sql`EXCLUDED.discriminator`,
globalName: sql`EXCLUDED.global_name`,
avatar: sql`EXCLUDED.avatar`,
bot: sql`EXCLUDED.bot`,
flags: sql`EXCLUDED.flags`,
accentColor: sql`EXCLUDED.accent_color`,
avatarDecoration: sql`EXCLUDED.avatar_decoration`,
banner: sql`EXCLUDED.banner`,
locale: sql`EXCLUDED.locale`,
mfaEnabled: sql`EXCLUDED.mfa_enabled`,
premiumType: sql`EXCLUDED.premium_type`,
publicFlags: sql`EXCLUDED.public_flags`
}
});
}

if (stateMembers) {
await this.store.drizzle.insert(members)
.values(
memberChunk.map(member => ({
id: member.user!.id,
avatar: member.avatar,
communicationDisabledUntil: member.premium_since,
deaf: member.deaf,
flags: member.flags,
joinedAt: member.joined_at,
mute: member.mute,
nick: member.nick,
pending: member.pending,
premiumSince: member.premium_since
}))
).onConflictDoUpdate({
target: members.id,
set: {
avatar: sql`EXCLUDED.avatar`,
communicationDisabledUntil: sql`EXCLUDED.premium_since`,
deaf: sql`EXCLUDED.deaf`,
flags: sql`EXCLUDED.flags`,
joinedAt: sql`EXCLUDED.joined_at`,
mute: sql`EXCLUDED.mute`,
nick: sql`EXCLUDED.nick`,
pending: sql`EXCLUDED.pending`,
premiumSince: sql`EXCLUDED.premium_since`
}
});

if (member.roles.length > 0) {
await this.store.drizzle.insert(memberRoles).values(member.roles.map(role => ({
memberId: member.user!.id,
roleId: role,
guildId: payload.data.d.guild_id
}))).onConflictDoNothing({ target: [memberRoles.memberId, memberRoles.roleId] });
for (const member of memberChunk) {
if (member.roles.length > 0) {
await this.store.drizzle.insert(memberRoles)
.values(member.roles.map(role => ({
memberId: member.user!.id,
roleId: role,
guildId: payload.data.d.guild_id
}))).onConflictDoNothing({ target: [memberRoles.memberId, memberRoles.roleId] });
}
}
}
}
Expand Down
Loading

0 comments on commit a9d5448

Please sign in to comment.