Skip to content

Commit

Permalink
fix: register file systems correctly when using duplicate roleArns BM…
Browse files Browse the repository at this point in the history
…-1055 (#3367)

### Motivation

We have been seeing some tiff files intermitently fail to load when
loading across AWS accounts, this was tracked down a failure to register
a AWS roleArn against the target bucket when the roleArn was used in
multiple different buckets

### Modifications

Use the more general "onFileSystemFound" as "onFileSystemCreated" only
happens once per roleArn.

### Verification

Tested locally
  • Loading branch information
blacha authored Nov 4, 2024
1 parent af656bf commit 385971d
Show file tree
Hide file tree
Showing 3 changed files with 18 additions and 14 deletions.
16 changes: 8 additions & 8 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion packages/shared/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
"@basemaps/config": "^7.11.0",
"@basemaps/geo": "^7.11.0",
"@chunkd/fs": "^11.2.0",
"@chunkd/fs-aws": "^11.2.2",
"@chunkd/fs-aws": "^11.3.0",
"@chunkd/middleware": "^11.1.0",
"@cogeotiff/core": "^9.0.3",
"@cotar/core": "^6.0.1",
Expand Down
14 changes: 9 additions & 5 deletions packages/shared/src/file.system.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url';

import { S3Client } from '@aws-sdk/client-s3';
import { sha256base58 } from '@basemaps/config';
import { FileSystem, fsa, FsHttp } from '@chunkd/fs';
import { fsa, FsHttp } from '@chunkd/fs';
import { AwsCredentialConfig, AwsS3CredentialProvider, FsAwsS3 } from '@chunkd/fs-aws';
import { SourceCache, SourceChunk } from '@chunkd/middleware';
import type { SourceCallback, SourceRequest } from '@chunkd/source';
Expand Down Expand Up @@ -36,17 +36,21 @@ export const s3FsPublic = new FsAwsS3(
/** Ensure middleware are added to all s3 clients that are created */
function applyS3MiddleWare(fs: FsAwsS3): void {
if (fs.s3 == null) return;
fs.s3.middlewareStack.add(Fqdn.middleware, { name: 'FQDN', step: 'finalizeRequest' });
const stacks = fs.s3.middlewareStack.identify();
if (stacks.find((f) => f.startsWith('FQDN - ')) == null) {
fs.s3.middlewareStack.add(Fqdn.middleware, { name: 'FQDN', step: 'finalizeRequest' });
}
}

applyS3MiddleWare(s3FsPublic);
applyS3MiddleWare(s3Fs);

const credentials = new AwsS3CredentialProvider();

credentials.onFileSystemCreated = (acc: AwsCredentialConfig, fs: FileSystem): void => {
LogConfig.get().info({ prefix: acc.prefix, roleArn: acc.roleArn }, 'FileSystem:Register');
applyS3MiddleWare(fs as FsAwsS3);
credentials.onFileSystemFound = (acc: AwsCredentialConfig, fs?: FsAwsS3, path?: URL): void => {
if (fs == null) return;
LogConfig.get().info({ prefix: acc.prefix, roleArn: acc.roleArn, path: path?.href }, 'FileSystem:Register');
applyS3MiddleWare(fs);
fsa.register(acc.prefix, fs);
};

Expand Down

0 comments on commit 385971d

Please sign in to comment.