fix: reduce the number of cloudfront invalidations (#2991)

#### Motivation

Fixing a bug where cloudfront would rate limit us from invalidations

#### Modification

- Invalidate only top level directories
- upload 10 files at a time

#### Checklist

_If not applicable, provide explanation of why._

- [ ] Tests updated
- [ ] Docs updated
- [ ] Issue linked in Title

packages/cli/src/cli/util.ts

@@ -57,7 +57,7 @@ export const HashKey = 'linz-hash';
 
 export async function getHash(Bucket: string, Key: string): Promise<string | null> {
   try {
-    const obj = await s3.getObject({ Bucket, Key }).promise();
+    const obj = await s3.headObject({ Bucket, Key }).promise();
     return obj.Metadata?.[HashKey] ?? null;
   } catch (e: any) {
     if (e.code === 'NoSuchKey') return null;
@@ -69,14 +69,19 @@ export async function getHash(Bucket: string, Key: string): Promise<string | nul
  * Lookup the static bucket from cloudformation
  * @returns
  */
-let staticBucket: string | null = null;
-export async function getStaticBucket(): Promise<string | null> {
-  if (staticBucket != null) return staticBucket;
-  // Since the bucket is generated inside of CDK lets look up the bucket name
-  const stackInfo = await cloudFormation.describeStacks({ StackName: 'Edge' }).promise();
-  const bucket = stackInfo.Stacks?.[0]?.Outputs?.find((f) => f.OutputKey === 'CloudFrontBucket');
-  if (bucket == null) throw new Error('Failed to find EdgeBucket');
-  staticBucket = bucket.OutputValue ?? null;
+let staticBucket: Promise<string> | undefined;
+export function getStaticBucket(): Promise<string> {
+  if (staticBucket == null) {
+    // Since the bucket is generated inside of CDK lets look up the bucket name
+    staticBucket = cloudFormation
+      .describeStacks({ StackName: 'Edge' })
+      .promise()
+      .then((stackInfo) => {
+        const val = stackInfo.Stacks?.[0]?.Outputs?.find((f) => f.OutputKey === 'CloudFrontBucket')?.OutputValue;
+        if (val == null) throw new Error('Failed to find EdgeBucket');
+        return val;
+      });
+  }
   return staticBucket;
 }
 
@@ -103,7 +108,6 @@ export async function uploadStaticFile(
 
   // S3 keys should not start with a `/`
   if (target.startsWith('/')) target = target.slice(1);
-
   const bucket = await getStaticBucket();
   if (bucket == null) throw new Error('Unable to find static bucket');
 

packages/landing/scripts/deploy.js

@@ -1,7 +1,10 @@
 import mime from 'mime-types';
-import { extname, basename, resolve } from 'path';
+import { extname, basename, resolve, dirname } from 'path';
 import { invalidateCache, uploadStaticFile } from '@basemaps/cli/build/cli/util.js';
 import { fsa } from '@basemaps/shared';
+import pLimit from 'p-limit';
+
+const Q = pLimit(10);
 
 const DistDir = './dist';
 
@@ -10,32 +13,52 @@ const HasVersionRe = /-\d+\.\d+\.\d+-/;
 
 /**
  * Deploy the built s3 assets into the Edge bucket
- *
- * TODO there does not appear to be a easy way to do this with aws-cdk yet
  */
 async function deploy() {
   const basePath = resolve(DistDir);
-  for await (const filePath of fsa.list(basePath)) {
-    const targetKey = filePath.slice(basePath.length);
-    const isVersioned = HasVersionRe.test(basename(filePath));
-    const contentType = mime.contentType(extname(filePath));
-
-    const cacheControl = isVersioned
-      ? // Set cache control for versioned files to immutable
-        'public, max-age=604800, immutable'
-      : // Set cache control for non versioned files to be short lived
-        'public, max-age=60, stale-while-revalidate=300';
-
-    if (targetKey.endsWith('index.html') && targetKey !== '/index.html') {
-      await uploadStaticFile(filePath, targetKey.replace('/index.html', ''), contentType, cacheControl);
-      await uploadStaticFile(filePath, targetKey.replace('/index.html', '/'), contentType, cacheControl);
-    }
-
-    const isUploaded = await uploadStaticFile(filePath, targetKey, contentType, cacheControl);
-    if (isUploaded) {
-      console.log('Uploaded', { targetKey });
-      if (!isVersioned) await invalidateCache(targetKey, true);
-    }
+
+  const invalidationPaths = new Set();
+
+  const fileList = await fsa.toArray(fsa.list(basePath));
+  const promises = fileList.map((filePath) =>
+    Q(async () => {
+      const targetKey = filePath.slice(basePath.length);
+      const isVersioned = HasVersionRe.test(basename(filePath));
+      const contentType = mime.contentType(extname(filePath));
+
+      const cacheControl = isVersioned
+        ? // Set cache control for versioned files to immutable
+          'public, max-age=604800, immutable'
+        : // Set cache control for non versioned files to be short lived
+          'public, max-age=60, stale-while-revalidate=300';
+
+      if (targetKey.endsWith('index.html') && targetKey !== '/index.html') {
+        await uploadStaticFile(filePath, targetKey.replace('/index.html', ''), contentType, cacheControl);
+        await uploadStaticFile(filePath, targetKey.replace('/index.html', '/'), contentType, cacheControl);
+      }
+
+      const isUploaded = await uploadStaticFile(filePath, targetKey, contentType, cacheControl);
+      if (!isUploaded) return; // No need to invalidate objects not uploaded
+      console.log('FileUpload', targetKey, { isVersioned });
+      if (isVersioned) return; // No need to invalidate versioned objects
+
+      // Invalidate the top level directory only
+      // or the base file if it exists
+      if (targetKey.includes('/', 1)) {
+        const dir = dirname(targetKey);
+        invalidationPaths.add(dir + '/*');
+      } else {
+        invalidationPaths.add(targetKey);
+      }
+    }),
+  );
+
+  await Promise.all(promises);
+
+  if (invalidationPaths.size > 0) {
+    const toInvalidate = [...invalidationPaths];
+    console.log('Invalidate', toInvalidate);
+    await invalidateCache(toInvalidate, true);
   }
 }