From 1f822e3be617a6e5b8061081d84a3e6cfef350be Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 11 Oct 2024 15:14:36 +0100 Subject: [PATCH 1/4] feat(CB2-14232): update doc-gen-upload to remove snowball SQS pattern --- src/handler.ts | 76 +++++++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 41 deletions(-) diff --git a/src/handler.ts b/src/handler.ts index 3176f25..8d89636 100644 --- a/src/handler.ts +++ b/src/handler.ts @@ -1,4 +1,4 @@ -import { Handler, SQSBatchResponse, SQSEvent } from 'aws-lambda'; +import {Handler, SQSBatchItemFailure, SQSBatchResponse, SQSEvent} from 'aws-lambda'; import { TextDecoder } from 'util'; import { DocumentModel } from './models/documentModel'; import { getDocumentFromRequest } from './models/documentModel.factory'; @@ -13,50 +13,44 @@ export const handler: Handler = async (event: SQSEvent): Promise { - const request = JSON.parse(sqsRecord.body) as Request; - - const document: DocumentModel = getDocumentFromRequest(request); - return generateAndUpload(document, request); - }); - - const results = await Promise.allSettled(altPromiseArray); - const ids = results - .map((result, index) => (result.status === 'fulfilled' ? null : event.Records[index].messageId)) - .filter((item) => item !== null); - return { - batchItemFailures: ids.map((id) => ({ itemIdentifier: id })), - }; + const batchItemFailures: SQSBatchItemFailure[] = []; + // eslint-disable-next-line no-restricted-syntax + for (const sqsRecord of event.Records) { + try { + const request = JSON.parse(sqsRecord.body) as Request; + const document: DocumentModel = getDocumentFromRequest(request); + // eslint-disable-next-line no-await-in-loop + await generateAndUpload(document, request); + } catch (error) { + console.error(error); + batchItemFailures.push({ itemIdentifier: sqsRecord.messageId }); + } + } + return { batchItemFailures }; }; const generateAndUpload = async (document: DocumentModel, request: Request) => { - try { - logger.info(`Starting lambda to lambda invoke (data): ${JSON.stringify(document)}`); - const response = await invokePdfGenLambda(document, request.documentName); - logger.info('Finished lambda to lambda invoke, checking response'); + logger.info(`Starting lambda to lambda invoke (data): ${JSON.stringify(document)}`); + const response = await invokePdfGenLambda(document, request.documentName); + logger.info('Finished lambda to lambda invoke, checking response'); - if (response.StatusCode !== 200) { - logger.error(`Error invoking doc gen (lambda call failed with ${response.StatusCode}`); - throw new Error(`Error invoking doc gen (lambda call failed with ${response.StatusCode}`); - } - const responseString: string = new TextDecoder().decode(response.Payload); - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const responseJson: any = JSON.parse(responseString); - if (responseJson.statusCode !== 200) { - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions - throw new Error(`Error returned from doc gen (${responseJson.statusCode}): ${responseJson.body}`); - } + if (response.StatusCode !== 200) { + logger.error(`Error invoking doc gen (lambda call failed with ${response.StatusCode}`); + throw new Error(`Error invoking doc gen (lambda call failed with ${response.StatusCode}`); + } + const responseString: string = new TextDecoder().decode(response.Payload); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const responseJson: any = JSON.parse(responseString); + if (responseJson.statusCode !== 200) { + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + throw new Error(`Error returned from doc gen (${responseJson.statusCode}): ${responseJson.body}`); + } - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - const responseBuffer: Buffer = Buffer.from(responseJson.body, 'base64'); - document.setFileSize(responseBuffer.byteLength); + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + const responseBuffer: Buffer = Buffer.from(responseJson.body, 'base64'); + document.setFileSize(responseBuffer.byteLength); - logger.info(`Starting s3 upload for file: ${process.env.BRANCH}/${document.filename}`); - await uploadPdfToS3(responseBuffer, document.metaData, document.filename); - logger.info('Finished s3 upload'); - } catch (error) { - logger.error(error.message); - throw error; - } + logger.info(`Starting s3 upload for file: ${process.env.BRANCH}/${document.filename}`); + await uploadPdfToS3(responseBuffer, document.metaData, document.filename); + logger.info('Finished s3 upload'); }; From 88264e85937e4d1c139f04f2fe3da796656933c7 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 16 Oct 2024 11:29:35 +0100 Subject: [PATCH 2/4] feat(CB2-14232): unit test, lint, and logging --- src/handler.ts | 7 ++++--- tests/unit/handler.test.ts | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/handler.ts b/src/handler.ts index 8d89636..94111e9 100644 --- a/src/handler.ts +++ b/src/handler.ts @@ -1,4 +1,6 @@ -import {Handler, SQSBatchItemFailure, SQSBatchResponse, SQSEvent} from 'aws-lambda'; +import { + Handler, SQSBatchItemFailure, SQSBatchResponse, SQSEvent, +} from 'aws-lambda'; import { TextDecoder } from 'util'; import { DocumentModel } from './models/documentModel'; import { getDocumentFromRequest } from './models/documentModel.factory'; @@ -22,7 +24,7 @@ export const handler: Handler = async (event: SQSEvent): Promise { logger.info('Finished lambda to lambda invoke, checking response'); if (response.StatusCode !== 200) { - logger.error(`Error invoking doc gen (lambda call failed with ${response.StatusCode}`); throw new Error(`Error invoking doc gen (lambda call failed with ${response.StatusCode}`); } const responseString: string = new TextDecoder().decode(response.Payload); diff --git a/tests/unit/handler.test.ts b/tests/unit/handler.test.ts index 59a1a50..c9ce98f 100644 --- a/tests/unit/handler.test.ts +++ b/tests/unit/handler.test.ts @@ -1,5 +1,6 @@ import { InvokeCommandOutput, LambdaClient } from '@aws-sdk/client-lambda'; import { PutObjectCommandOutput, S3 } from '@aws-sdk/client-s3'; +import { SQSBatchResponse } from 'aws-lambda'; import { DocumentName } from '../../src/enums/documentName.enum'; import { ReasonForIssue } from '../../src/enums/reasonForIssue.enum'; import * as Handler from '../../src/handler'; @@ -38,6 +39,13 @@ describe('handler tests', () => { } }); + it('should return failed records', async () => { + const sqsEvent = pass; + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const batchFails : SQSBatchResponse = await Handler.handler(sqsEvent, undefined, () => true); + expect(batchFails.batchItemFailures[0].itemIdentifier).toEqual(sqsEvent.Records[0].messageId); + }); + it('should throw an error if document type not supported', async () => { const sqsEvent = pass; sqsEvent.Records[0].body = JSON.stringify({ From 1d60467571a8cd60ffd930f147680fab7f0b61e8 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 16 Oct 2024 13:59:59 +0100 Subject: [PATCH 3/4] feat(CB2-14232): update gitignore --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 83b3b09..1e63a3f 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,8 @@ reports *.map # npm audit results -npm-audit.html \ No newline at end of file +npm-audit.html + +# idea specific hidden files +.idea/** +*.iml From 9cb2029a6f72d417fd738a47383f61eb3df1c23e Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 18 Oct 2024 11:20:35 +0100 Subject: [PATCH 4/4] feat(CB2-14232): logging --- src/handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/handler.ts b/src/handler.ts index 94111e9..95e5c60 100644 --- a/src/handler.ts +++ b/src/handler.ts @@ -24,7 +24,7 @@ export const handler: Handler = async (event: SQSEvent): Promise