From 385971dc23c0b1cd9c354c5028d47d01ab5ae584 Mon Sep 17 00:00:00 2001
From: Blayne Chard <bchard@linz.govt.nz>
Date: Mon, 4 Nov 2024 15:05:19 +1300
Subject: [PATCH] fix: register file systems correctly when using duplicate
 roleArns BM-1055 (#3367)

### Motivation

We have been seeing some tiff files intermitently fail to load when
loading across AWS accounts, this was tracked down a failure to register
a AWS roleArn against the target bucket when the roleArn was used in
multiple different buckets

### Modifications

Use the more general "onFileSystemFound" as "onFileSystemCreated" only
happens once per roleArn.

### Verification

Tested locally
---
 package-lock.json                  | 16 ++++++++--------
 packages/shared/package.json       |  2 +-
 packages/shared/src/file.system.ts | 14 +++++++++-----
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 1a87b76f5..71892f5ae 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1482,15 +1482,15 @@
       }
     },
     "node_modules/@chunkd/fs-aws": {
-      "version": "11.2.2",
-      "resolved": "https://registry.npmjs.org/@chunkd/fs-aws/-/fs-aws-11.2.2.tgz",
-      "integrity": "sha512-zTF0sVXHkzHIOZWwav4Qe1AcdTCCHgY5fyd9IWd7XjZtOjgYwlakwKV6GnkRkhD5QqAjTW/SVJRLca8Z7T1snw==",
+      "version": "11.3.0",
+      "resolved": "https://registry.npmjs.org/@chunkd/fs-aws/-/fs-aws-11.3.0.tgz",
+      "integrity": "sha512-Oq3knOYibBvBTGpjNrfol0x9w/H0Gyf0WbasX5NL6q8fJBrynR1vbBUnlT7s/DfSgu9AZLuMcD1dQa7t2ouNlA==",
       "dependencies": {
         "@aws-sdk/client-s3": "*",
         "@aws-sdk/credential-providers": "*",
         "@aws-sdk/lib-storage": "*",
         "@chunkd/fs": "11.2.0",
-        "@chunkd/source-aws": "11.0.3"
+        "@chunkd/source-aws": "11.0.4"
       },
       "engines": {
         "node": ">=16.0.0"
@@ -1516,9 +1516,9 @@
       }
     },
     "node_modules/@chunkd/source-aws": {
-      "version": "11.0.3",
-      "resolved": "https://registry.npmjs.org/@chunkd/source-aws/-/source-aws-11.0.3.tgz",
-      "integrity": "sha512-zWWV39cOKdTmgiSW1PiNJLRM9PoxpaBkMD/z2Wn+8c8JnLt4NNILsdne0qrl722xc4vcFar0xXFdAxy9Stlolg==",
+      "version": "11.0.4",
+      "resolved": "https://registry.npmjs.org/@chunkd/source-aws/-/source-aws-11.0.4.tgz",
+      "integrity": "sha512-HvQ44z6Zz0f/tRFMyvfF4sQDCNXlorp5d6skei1mC0vZyMTn8BJfMkZKgvpDDUpyt5lGCxFREd3/LI140E0rZg==",
       "dependencies": {
         "@aws-sdk/client-s3": "*",
         "@chunkd/source": "^11.1.0"
@@ -19726,7 +19726,7 @@
         "@basemaps/config": "^7.11.0",
         "@basemaps/geo": "^7.11.0",
         "@chunkd/fs": "^11.2.0",
-        "@chunkd/fs-aws": "^11.2.2",
+        "@chunkd/fs-aws": "^11.3.0",
         "@chunkd/middleware": "^11.1.0",
         "@cogeotiff/core": "^9.0.3",
         "@cotar/core": "^6.0.1",
diff --git a/packages/shared/package.json b/packages/shared/package.json
index 9de1322e3..67a34e764 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -29,7 +29,7 @@
     "@basemaps/config": "^7.11.0",
     "@basemaps/geo": "^7.11.0",
     "@chunkd/fs": "^11.2.0",
-    "@chunkd/fs-aws": "^11.2.2",
+    "@chunkd/fs-aws": "^11.3.0",
     "@chunkd/middleware": "^11.1.0",
     "@cogeotiff/core": "^9.0.3",
     "@cotar/core": "^6.0.1",
diff --git a/packages/shared/src/file.system.ts b/packages/shared/src/file.system.ts
index c48b9a38c..9ce91628c 100644
--- a/packages/shared/src/file.system.ts
+++ b/packages/shared/src/file.system.ts
@@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url';
 
 import { S3Client } from '@aws-sdk/client-s3';
 import { sha256base58 } from '@basemaps/config';
-import { FileSystem, fsa, FsHttp } from '@chunkd/fs';
+import { fsa, FsHttp } from '@chunkd/fs';
 import { AwsCredentialConfig, AwsS3CredentialProvider, FsAwsS3 } from '@chunkd/fs-aws';
 import { SourceCache, SourceChunk } from '@chunkd/middleware';
 import type { SourceCallback, SourceRequest } from '@chunkd/source';
@@ -36,7 +36,10 @@ export const s3FsPublic = new FsAwsS3(
 /** Ensure middleware are added to all s3 clients that are created */
 function applyS3MiddleWare(fs: FsAwsS3): void {
   if (fs.s3 == null) return;
-  fs.s3.middlewareStack.add(Fqdn.middleware, { name: 'FQDN', step: 'finalizeRequest' });
+  const stacks = fs.s3.middlewareStack.identify();
+  if (stacks.find((f) => f.startsWith('FQDN - ')) == null) {
+    fs.s3.middlewareStack.add(Fqdn.middleware, { name: 'FQDN', step: 'finalizeRequest' });
+  }
 }
 
 applyS3MiddleWare(s3FsPublic);
@@ -44,9 +47,10 @@ applyS3MiddleWare(s3Fs);
 
 const credentials = new AwsS3CredentialProvider();
 
-credentials.onFileSystemCreated = (acc: AwsCredentialConfig, fs: FileSystem): void => {
-  LogConfig.get().info({ prefix: acc.prefix, roleArn: acc.roleArn }, 'FileSystem:Register');
-  applyS3MiddleWare(fs as FsAwsS3);
+credentials.onFileSystemFound = (acc: AwsCredentialConfig, fs?: FsAwsS3, path?: URL): void => {
+  if (fs == null) return;
+  LogConfig.get().info({ prefix: acc.prefix, roleArn: acc.roleArn, path: path?.href }, 'FileSystem:Register');
+  applyS3MiddleWare(fs);
   fsa.register(acc.prefix, fs);
 };