diff --git a/package-lock.json b/package-lock.json index 31d6ac3..196af7c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16579,9 +16579,9 @@ } }, "node_modules/serverless-offline/node_modules/ws": { - "version": "8.17.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz", - "integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "dev": true, "engines": { "node": ">=10.0.0" @@ -19286,9 +19286,9 @@ "dev": true }, "node_modules/ws": { - "version": "7.5.9", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", - "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "version": "7.5.10", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", + "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==", "dev": true, "engines": { "node": ">=8.3.0" diff --git a/src/infrastructure/s3BucketService.ts b/src/infrastructure/s3BucketService.ts index 338c89f..3c0b74f 100644 --- a/src/infrastructure/s3BucketService.ts +++ b/src/infrastructure/s3BucketService.ts @@ -1,10 +1,11 @@ import { GetObjectCommand, - GetObjectCommandInput, + GetObjectCommandOutput, PutObjectCommand, S3Client, } from '@aws-sdk/client-s3'; import logger from '../utils/logger'; +import { Readable } from "stream"; export async function uploadToS3(processedData: string, fileName: string, callback: () => void): Promise { const s3: S3Client = configureS3(); @@ -21,11 +22,21 @@ export async function uploadToS3(processedData: string, fileName: string, callba export async function getItemFromS3(key: string): Promise { logger.info(`Reading contents of file ${key}`); const s3 = configureS3(); - const params: GetObjectCommandInput = { Bucket: process.env.AWS_S3_BUCKET_NAME ?? '', Key: key }; + const command: GetObjectCommand = new GetObjectCommand( + { + Bucket: process.env.AWS_S3_BUCKET_NAME ?? '', + Key: key, + } + ); + try { - const body = (await s3.send(new GetObjectCommand(params))).Body?.toString(); - logger.info(`File contents retrieved: ${body}`); - return body; + const response: GetObjectCommandOutput = await s3.send(command); + + if (response.Body instanceof Readable) { + const bufferedString = await streamToString(response.Body); + logger.info(`File contents retrieved: ${bufferedString}`); + return bufferedString; + } } catch (err) { logger.error(`Error reading file from S3 ${JSON.stringify(err)}`); throw err; @@ -78,3 +89,13 @@ function configureS3() { } return new S3Client({}); } + +async function streamToString(stream: Readable): Promise { + const chunks: Uint8Array[] = []; + + for await (const chunk of stream) { + chunks.push(chunk); + } + + return Buffer.concat(chunks).toString('utf-8'); +} diff --git a/tests/unit/infrastructure/s3BucketService.test.ts b/tests/unit/infrastructure/s3BucketService.test.ts index 31f1cd5..8688de3 100644 --- a/tests/unit/infrastructure/s3BucketService.test.ts +++ b/tests/unit/infrastructure/s3BucketService.test.ts @@ -1,4 +1,6 @@ /* eslint-disable import/first */ +import { Readable } from "stream"; + process.env.LOG_LEVEL = 'error'; const mockPromise = jest.fn(); const mockGetObject = jest.fn(() => ({ @@ -32,7 +34,7 @@ describe('readAndUpsert', () => { const originalFileContents = 'the original content of the file'; const newFileContents = 'new content for the file'; - client.on(GetObjectCommand).resolves({ Body: Buffer.from(originalFileContents) } as unknown as GetObjectCommandOutput); + client.on(GetObjectCommand).resolves({ Body: Readable.from(Buffer.from(originalFileContents)) } as unknown as GetObjectCommandOutput); client.on(PutObjectCommand).callsFake(mockUpload); const contents = await readAndUpsert(fileName, newFileContents);