Skip to content

Commit

Permalink
fix: delete from s3 in chunks (#197)
Browse files Browse the repository at this point in the history
  • Loading branch information
jadenv authored Jan 10, 2024
1 parent 05a77d5 commit d669ff8
Showing 1 changed file with 20 additions and 6 deletions.
26 changes: 20 additions & 6 deletions src/lambdas/nextjs-bucket-deployment.ts
Original file line number Diff line number Diff line change
Expand Up @@ -379,12 +379,26 @@ async function pruneBucket({
}
}
if (oldObjectKeysToBeDeleted.length) {
await s3.send(
new DeleteObjectsCommand({
Bucket: bucketName,
Delete: { Objects: oldObjectKeysToBeDeleted.map((k) => ({ Key: k })) },
})
);
const deletePromises = [];

// AWS limits S3 delete commands to 1000 keys per call
const deleteCommandLimit = 1000;

for (let i = 0; i < oldObjectKeysToBeDeleted.length; i += deleteCommandLimit) {
const objectChunk = oldObjectKeysToBeDeleted.slice(i, i + deleteCommandLimit);

deletePromises.push(
s3.send(
new DeleteObjectsCommand({
Bucket: bucketName,
Delete: { Objects: objectChunk.map((k) => ({ Key: k })) },
})
)
);
}

await Promise.all(deletePromises);

debug(`Objects pruned in ${bucketName}: ${oldObjectKeysToBeDeleted.join(', ')}`);
} else {
debug(`No objects to prune`);
Expand Down

0 comments on commit d669ff8

Please sign in to comment.