From a13cc5bb8af6362e22a5ce806e65703f8e1ce1aa Mon Sep 17 00:00:00 2001 From: enxtur Date: Sun, 5 May 2024 18:37:53 +0800 Subject: [PATCH 1/3] Add batchLimit for db config --- config/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/config/index.ts b/config/index.ts index 7e53e078..7a750419 100644 --- a/config/index.ts +++ b/config/index.ts @@ -23,6 +23,7 @@ export const CONFIG = { name: process.env.NODE_ENV, authBaseUrl: process.env.AUTHBASE_URL, db: { + batchLimit: 32_768, // 2^15, connection: process.env.POSTGRES_SERVER, logging: !!parseInt(process.env.POSTGRES_LOGGING ?? ''), poolMin: 2, From a33d70286886a3436a6c766d1ef164959ad09c6a Mon Sep 17 00:00:00 2001 From: enxtur Date: Sun, 5 May 2024 18:38:23 +0800 Subject: [PATCH 2/3] HPC-9551: Chunk versionModel update by batch limit --- src/common-libs/plan/versioning.ts | 35 ++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/common-libs/plan/versioning.ts b/src/common-libs/plan/versioning.ts index 5cec9d1b..8c3b5dc5 100644 --- a/src/common-libs/plan/versioning.ts +++ b/src/common-libs/plan/versioning.ts @@ -40,6 +40,8 @@ import { createBrandedValue, type Brand, } from '@unocha/hpc-api-core/src/util/types'; +import { chunk } from 'lodash'; +import { CONFIG } from '../../../config'; // eslint-disable-next-line @typescript-eslint/no-explicit-any type AnyModelId = Brand; @@ -257,18 +259,29 @@ const updateBaseAndVersionModelTags = async ( }, skipValidation: skipAttachmentsAndMeasurementsValidation(tableName), }); - await versionModel.update({ - values: { - latestTaggedVersion: false, - ...(tag.public ? { currentVersion: false } : {}), - }, - where: { - id: { - [models.Op.IN]: oldVersions.map((v) => v.id as AnyModelId), + /** + * We have to limit the number of parameters that can be passed to a query. + * Limit is 32768 (2^15) which is batchLimit in the config file. + * latestTaggedVersion, currentVersion, and updatedAt is also included in the parameters + * so we subtract 3 from the limit to get the maximum number of parameters that can be passed to a query + */ + for (const chunkedOldVersionModelIDs of chunk( + oldVersions.map((v) => v.id as AnyModelId), + CONFIG.db.batchLimit - 3 + )) { + await versionModel.update({ + values: { + latestTaggedVersion: false, + ...(tag.public ? { currentVersion: false } : {}), }, - }, - skipValidation: skipAttachmentsAndMeasurementsValidation(tableName), - }); + where: { + id: { + [models.Op.IN]: chunkedOldVersionModelIDs, + }, + }, + skipValidation: skipAttachmentsAndMeasurementsValidation(tableName), + }); + } }; const updateBaseModelTags = async ( From 4738f87a005be4d90a5738f5fa1d3a8f25d28afb Mon Sep 17 00:00:00 2001 From: enxtur Date: Tue, 7 May 2024 20:15:58 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=94=96=20Bump=20version=20to=20v4.6.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 80b0ff70..73537e57 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hpc-api", - "version": "4.6.4", + "version": "4.6.5", "description": "api for HPC applications", "main": "src/server.ts", "license": "MIT",