From 786e4e9e3ece1410cb77983baa3a887e198d69b2 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Mon, 6 Nov 2023 16:28:14 +1300 Subject: [PATCH] fix: reduce the number of cloudfront invalidations (#2991) #### Motivation Fixing a bug where cloudfront would rate limit us from invalidations #### Modification - Invalidate only top level directories - upload 10 files at a time #### Checklist _If not applicable, provide explanation of why._ - [ ] Tests updated - [ ] Docs updated - [ ] Issue linked in Title --- packages/cli/src/cli/util.ts | 24 +++++----- packages/landing/scripts/deploy.js | 71 ++++++++++++++++++++---------- 2 files changed, 61 insertions(+), 34 deletions(-) diff --git a/packages/cli/src/cli/util.ts b/packages/cli/src/cli/util.ts index 2865b6432..c0060959f 100644 --- a/packages/cli/src/cli/util.ts +++ b/packages/cli/src/cli/util.ts @@ -57,7 +57,7 @@ export const HashKey = 'linz-hash'; export async function getHash(Bucket: string, Key: string): Promise { try { - const obj = await s3.getObject({ Bucket, Key }).promise(); + const obj = await s3.headObject({ Bucket, Key }).promise(); return obj.Metadata?.[HashKey] ?? null; } catch (e: any) { if (e.code === 'NoSuchKey') return null; @@ -69,14 +69,19 @@ export async function getHash(Bucket: string, Key: string): Promise { - if (staticBucket != null) return staticBucket; - // Since the bucket is generated inside of CDK lets look up the bucket name - const stackInfo = await cloudFormation.describeStacks({ StackName: 'Edge' }).promise(); - const bucket = stackInfo.Stacks?.[0]?.Outputs?.find((f) => f.OutputKey === 'CloudFrontBucket'); - if (bucket == null) throw new Error('Failed to find EdgeBucket'); - staticBucket = bucket.OutputValue ?? null; +let staticBucket: Promise | undefined; +export function getStaticBucket(): Promise { + if (staticBucket == null) { + // Since the bucket is generated inside of CDK lets look up the bucket name + staticBucket = cloudFormation + .describeStacks({ StackName: 'Edge' }) + .promise() + .then((stackInfo) => { + const val = stackInfo.Stacks?.[0]?.Outputs?.find((f) => f.OutputKey === 'CloudFrontBucket')?.OutputValue; + if (val == null) throw new Error('Failed to find EdgeBucket'); + return val; + }); + } return staticBucket; } @@ -103,7 +108,6 @@ export async function uploadStaticFile( // S3 keys should not start with a `/` if (target.startsWith('/')) target = target.slice(1); - const bucket = await getStaticBucket(); if (bucket == null) throw new Error('Unable to find static bucket'); diff --git a/packages/landing/scripts/deploy.js b/packages/landing/scripts/deploy.js index 80ca50e5a..b8297cc31 100644 --- a/packages/landing/scripts/deploy.js +++ b/packages/landing/scripts/deploy.js @@ -1,7 +1,10 @@ import mime from 'mime-types'; -import { extname, basename, resolve } from 'path'; +import { extname, basename, resolve, dirname } from 'path'; import { invalidateCache, uploadStaticFile } from '@basemaps/cli/build/cli/util.js'; import { fsa } from '@basemaps/shared'; +import pLimit from 'p-limit'; + +const Q = pLimit(10); const DistDir = './dist'; @@ -10,32 +13,52 @@ const HasVersionRe = /-\d+\.\d+\.\d+-/; /** * Deploy the built s3 assets into the Edge bucket - * - * TODO there does not appear to be a easy way to do this with aws-cdk yet */ async function deploy() { const basePath = resolve(DistDir); - for await (const filePath of fsa.list(basePath)) { - const targetKey = filePath.slice(basePath.length); - const isVersioned = HasVersionRe.test(basename(filePath)); - const contentType = mime.contentType(extname(filePath)); - - const cacheControl = isVersioned - ? // Set cache control for versioned files to immutable - 'public, max-age=604800, immutable' - : // Set cache control for non versioned files to be short lived - 'public, max-age=60, stale-while-revalidate=300'; - - if (targetKey.endsWith('index.html') && targetKey !== '/index.html') { - await uploadStaticFile(filePath, targetKey.replace('/index.html', ''), contentType, cacheControl); - await uploadStaticFile(filePath, targetKey.replace('/index.html', '/'), contentType, cacheControl); - } - - const isUploaded = await uploadStaticFile(filePath, targetKey, contentType, cacheControl); - if (isUploaded) { - console.log('Uploaded', { targetKey }); - if (!isVersioned) await invalidateCache(targetKey, true); - } + + const invalidationPaths = new Set(); + + const fileList = await fsa.toArray(fsa.list(basePath)); + const promises = fileList.map((filePath) => + Q(async () => { + const targetKey = filePath.slice(basePath.length); + const isVersioned = HasVersionRe.test(basename(filePath)); + const contentType = mime.contentType(extname(filePath)); + + const cacheControl = isVersioned + ? // Set cache control for versioned files to immutable + 'public, max-age=604800, immutable' + : // Set cache control for non versioned files to be short lived + 'public, max-age=60, stale-while-revalidate=300'; + + if (targetKey.endsWith('index.html') && targetKey !== '/index.html') { + await uploadStaticFile(filePath, targetKey.replace('/index.html', ''), contentType, cacheControl); + await uploadStaticFile(filePath, targetKey.replace('/index.html', '/'), contentType, cacheControl); + } + + const isUploaded = await uploadStaticFile(filePath, targetKey, contentType, cacheControl); + if (!isUploaded) return; // No need to invalidate objects not uploaded + console.log('FileUpload', targetKey, { isVersioned }); + if (isVersioned) return; // No need to invalidate versioned objects + + // Invalidate the top level directory only + // or the base file if it exists + if (targetKey.includes('/', 1)) { + const dir = dirname(targetKey); + invalidationPaths.add(dir + '/*'); + } else { + invalidationPaths.add(targetKey); + } + }), + ); + + await Promise.all(promises); + + if (invalidationPaths.size > 0) { + const toInvalidate = [...invalidationPaths]; + console.log('Invalidate', toInvalidate); + await invalidateCache(toInvalidate, true); } }