From 3fd02bbd93590b28b4ef130680bba7cf6197c546 Mon Sep 17 00:00:00 2001
From: Jaden VanEckhout <jadenv@users.noreply.github.com>
Date: Tue, 2 Jan 2024 12:25:24 -0600
Subject: [PATCH] fix: upload to s3 in chunks (#190)

* fix: upload to s3 in chunks

* fix: increase chunk size to 100
---
 src/lambdas/nextjs-bucket-deployment.ts | 37 +++++++++++++++----------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/src/lambdas/nextjs-bucket-deployment.ts b/src/lambdas/nextjs-bucket-deployment.ts
index 0039ec78..1ae6e82e 100644
--- a/src/lambdas/nextjs-bucket-deployment.ts
+++ b/src/lambdas/nextjs-bucket-deployment.ts
@@ -259,7 +259,13 @@ function createS3Key({ keyPrefix, path, baseLocalDir }: { keyPrefix?: string; pa
   return join(...objectKeyParts);
 }
 
-function uploadObjects({
+async function* chunkArray(array: string[], chunkSize: number) {
+  for (let i = 0; i < array.length; i += chunkSize) {
+    yield array.slice(i, i + chunkSize);
+  }
+}
+
+async function uploadObjects({
   bucket,
   keyPrefix,
   filePaths,
@@ -272,19 +278,22 @@ function uploadObjects({
   baseLocalDir: string;
   putConfig: CustomResourceProperties['putConfig'];
 }) {
-  const putObjectInputs: PutObjectCommandInput[] = filePaths.map((path) => {
-    const contentType = mime.lookup(path) || undefined;
-    const putObjectOptions = getPutObjectOptions({ path, putConfig });
-    const key = createS3Key({ keyPrefix, path, baseLocalDir });
-    return {
-      ContentType: contentType,
-      ...putObjectOptions,
-      Bucket: bucket,
-      Key: key,
-      Body: createReadStream(path),
-    };
-  });
-  return Promise.all(putObjectInputs.map((input) => s3.send(new PutObjectCommand(input))));
+  for await (const filePathChunk of chunkArray(filePaths, 100)) {
+    const putObjectInputs: PutObjectCommandInput[] = filePathChunk.map((path) => {
+      const contentType = mime.lookup(path) || undefined;
+      const putObjectOptions = getPutObjectOptions({ path, putConfig });
+      const key = createS3Key({ keyPrefix, path, baseLocalDir });
+      return {
+        ContentType: contentType,
+        ...putObjectOptions,
+        Bucket: bucket,
+        Key: key,
+        Body: createReadStream(path),
+      };
+    });
+
+    await Promise.all(putObjectInputs.map((input) => s3.send(new PutObjectCommand(input))));
+  }
 }
 
 /**