feat: allow reads from linz "-scratch" buckets TDE-906 (#2982)

#### Motivation LINZ is starting to suffix s3 buckets with a rough indication to the quality of data inside of the bucket a "scratch" bucket is a temporary storage location and should have a data lifecycle rule set to delete data after 90/180 days. #### Modification Adds Workflows and Basemaps scratch buckets to allowed locations #### Checklist _If not applicable, provide explanation of why._ - [ ] Tests updated - [ ] Docs updated - [ ] Issue linked in Title
linz · Nov 1, 2023 · 615334f · 615334f
1 parent eb62388
commit 615334f
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 2 deletions.
diff --git a/packages/_infra/src/config.ts b/packages/_infra/src/config.ts
@@ -15,7 +15,14 @@ export interface BaseMapsConfig {
 }
 
 export const BaseMapsProdConfig: BaseMapsConfig = {
-  CogBucket: ['linz-basemaps', 'linz-basemaps-vector', `linz-basemaps-staging`, 'linz-workflow-artifacts'],
+  CogBucket: [
+    'linz-basemaps',
+    'linz-basemaps-vector',
+    `linz-basemaps-staging`,
+    `linz-basemaps-scratch`,
+    'linz-workflow-artifacts',
+    'linz-workflows-scratch',
+  ],
   CloudFrontDns: ['basemaps.linz.govt.nz', 'tiles.basemaps.linz.govt.nz'],
   PublicUrlBase: 'https://basemaps.linz.govt.nz',
   AwsRoleConfigBucket: 'linz-bucket-config',

diff --git a/packages/config/src/json/__tests__/config.loader.test.ts b/packages/config/src/json/__tests__/config.loader.test.ts
@@ -124,5 +124,8 @@ o.spec('config import', () => {
     o(
       getImageryName(new URL('s3://linz-workflow-artifacts/2023-09/05-ecan-banks-peninsula-original-9mjdj/flat/')),
     ).equals('05-ecan-banks-peninsula-original-9mjdj');
+    o(
+      getImageryName(new URL('s3://linz-workflows-scratch/2023-09/05-ecan-banks-peninsula-original-9mjdj/flat/')),
+    ).equals('05-ecan-banks-peninsula-original-9mjdj');
   });
 });
diff --git a/packages/lambda-tiler/src/util/config.loader.ts b/packages/lambda-tiler/src/util/config.loader.ts
@@ -6,7 +6,13 @@ import { CachedConfig } from './config.cache.js';
 import { getDefaultConfig } from '@basemaps/shared';
 
 // FIXME load this from process.env COG BUCKETS?
-const SafeBuckets = new Set(['linz-workflow-artifacts', 'linz-basemaps', 'linz-basemaps-staging']);
+const SafeBuckets = new Set([
+  'linz-workflows-scratch',
+  'linz-workflow-artifacts',
+  'linz-basemaps',
+  'linz-basemaps-staging',
+  'linz-basemaps-scratch',
+]);
 const SafeProtocols = new Set(['s3', 'memory']);
 
 export class ConfigLoader {