From a13cc5bb8af6362e22a5ce806e65703f8e1ce1aa Mon Sep 17 00:00:00 2001 From: enxtur Date: Sun, 5 May 2024 18:37:53 +0800 Subject: [PATCH 1/2] Add batchLimit for db config --- config/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/config/index.ts b/config/index.ts index 7e53e078..7a750419 100644 --- a/config/index.ts +++ b/config/index.ts @@ -23,6 +23,7 @@ export const CONFIG = { name: process.env.NODE_ENV, authBaseUrl: process.env.AUTHBASE_URL, db: { + batchLimit: 32_768, // 2^15, connection: process.env.POSTGRES_SERVER, logging: !!parseInt(process.env.POSTGRES_LOGGING ?? ''), poolMin: 2, From a33d70286886a3436a6c766d1ef164959ad09c6a Mon Sep 17 00:00:00 2001 From: enxtur Date: Sun, 5 May 2024 18:38:23 +0800 Subject: [PATCH 2/2] HPC-9551: Chunk versionModel update by batch limit --- src/common-libs/plan/versioning.ts | 35 ++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/common-libs/plan/versioning.ts b/src/common-libs/plan/versioning.ts index 5cec9d1b..8c3b5dc5 100644 --- a/src/common-libs/plan/versioning.ts +++ b/src/common-libs/plan/versioning.ts @@ -40,6 +40,8 @@ import { createBrandedValue, type Brand, } from '@unocha/hpc-api-core/src/util/types'; +import { chunk } from 'lodash'; +import { CONFIG } from '../../../config'; // eslint-disable-next-line @typescript-eslint/no-explicit-any type AnyModelId = Brand; @@ -257,18 +259,29 @@ const updateBaseAndVersionModelTags = async ( }, skipValidation: skipAttachmentsAndMeasurementsValidation(tableName), }); - await versionModel.update({ - values: { - latestTaggedVersion: false, - ...(tag.public ? { currentVersion: false } : {}), - }, - where: { - id: { - [models.Op.IN]: oldVersions.map((v) => v.id as AnyModelId), + /** + * We have to limit the number of parameters that can be passed to a query. + * Limit is 32768 (2^15) which is batchLimit in the config file. + * latestTaggedVersion, currentVersion, and updatedAt is also included in the parameters + * so we subtract 3 from the limit to get the maximum number of parameters that can be passed to a query + */ + for (const chunkedOldVersionModelIDs of chunk( + oldVersions.map((v) => v.id as AnyModelId), + CONFIG.db.batchLimit - 3 + )) { + await versionModel.update({ + values: { + latestTaggedVersion: false, + ...(tag.public ? { currentVersion: false } : {}), }, - }, - skipValidation: skipAttachmentsAndMeasurementsValidation(tableName), - }); + where: { + id: { + [models.Op.IN]: chunkedOldVersionModelIDs, + }, + }, + skipValidation: skipAttachmentsAndMeasurementsValidation(tableName), + }); + } }; const updateBaseModelTags = async (