Upgrade AWS SDK DynamoDB from v2 > v3

package.json

@@ -41,6 +41,7 @@
  "@ksmithut/prettier-standard": "^0.0.10",
  "@types/hapi__hapi": "^20.0.10",
  "@types/jest": "^27.4.0",
+ "aws-sdk-mock": "^6.2.0",
  "babel-jest": "^27.5.1",
  "clone-deep": "^4.0.1",
  "dynamics-web-api": "^1.7.3",
@@ -104,6 +105,13 @@
  ],
  "testEnvironment": "node",
  "testRunner": "jest-circus/runner",
- "silent": true
+ "silent": false,
+ "verbose": true
+ },
+ "dependencies": {
+ "@aws-sdk/client-dynamodb": "^3.665.0",
+ "@aws-sdk/lib-dynamodb": "^3.665.0",
+ "@smithy/shared-ini-file-loader": "^3.1.8",
+ "aws-sdk-client-mock": "^4.0.2"
  }
 }

packages/connectors-lib/src/__tests__/aws.spec.js

@@ -1,58 +1,144 @@
-import Config from '../config.js'
-const TEST_ENDPOINT = 'http://localhost:8080'
-jest.dontMock('aws-sdk')
-describe('aws connectors', () => {
- it('configures dynamodb with a custom endpoint if one is defined in configuration', async () => {
- Config.aws.dynamodb.endpoint = TEST_ENDPOINT
- const { ddb } = require('../aws.js').default()
- expect(ddb.config.endpoint).toEqual(TEST_ENDPOINT)
+import { DynamoDB } from '@aws-sdk/client-dynamodb'
+import AWS from 'aws-sdk'
+import { DynamoDBDocument } from '@aws-sdk/lib-dynamodb'
+import Config from '../config'
+
+jest.mock('aws-sdk', () => {
+ const SQS = jest.fn().mockImplementation(config => ({
+ config: { ...config, apiVersion: '2012-11-05', region: config.region || 'eu-west-2' }
+ }))
+ const S3 = jest.fn().mockImplementation(config => ({
+ config: { ...config, apiVersion: '2006-03-01', region: config.region || 'eu-west-2', s3ForcePathStyle: config.s3ForcePathStyle }
+ }))
+ const SecretsManager = jest.fn().mockImplementation(config => ({
+ config: { ...config, apiVersion: '2017-10-17', region: config.region || 'eu-west-2' }
+ }))
+
+ return { SQS, S3, SecretsManager }
+})
+
+jest.mock('@aws-sdk/client-dynamodb')
+jest.mock('@aws-sdk/lib-dynamodb', () => ({
+ DynamoDBDocument: {
+ from: jest.fn()
+ }
+}))
+
+describe('AWS Connectors', () => {
+ let SQS, S3, SecretsManager
+
+ beforeEach(() => {
+ DynamoDB.mockClear()
+ DynamoDBDocument.from.mockClear()
+
+ DynamoDBDocument.from.mockReturnValue({
+ send: jest.fn(),
+ queryAllPromise: jest.fn(),
+ scanAllPromise: jest.fn(),
+ batchWriteAllPromise: jest.fn(),
+ createUpdateExpression: jest.fn()
+ })
+
+ SQS = AWS.SQS
+ S3 = AWS.S3
+ SecretsManager = AWS.SecretsManager
+
+ SQS.mockClear()
+ S3.mockClear()
+ SecretsManager.mockClear()
  })
 
- it('configures sqs with a custom endpoint if one is defined in configuration', async () => {
- Config.aws.sqs.endpoint = TEST_ENDPOINT
- const { sqs } = require('../aws.js').default()
- expect(sqs.config.endpoint).toEqual(TEST_ENDPOINT)
+ it('configures dynamodb with a custom endpoint if one is defined in configuration', () => {
+ const TEST_ENDPOINT = 'http://localhost:8080'
+ Config.aws.dynamodb.endpoint = TEST_ENDPOINT
+ require('../aws.js').default()
+ expect(DynamoDB).toHaveBeenCalledWith(
+ expect.objectContaining({
+ endpoint: TEST_ENDPOINT
+ })
+ )
+ expect(DynamoDBDocument.from).toHaveBeenCalledWith(expect.any(DynamoDB))
  })
 
- it('uses the default dynamodb endpoint if it is not overridden in configuration', async () => {
+ it('uses the default dynamodb endpoint if it is not overridden in configuration', () => {
  process.env.AWS_REGION = 'eu-west-2'
  delete Config.aws.dynamodb.endpoint
- const { ddb } = require('../aws.js').default()
- expect(ddb.config.endpoint).toEqual('dynamodb.eu-west-2.amazonaws.com')
+ require('../aws.js').default()
+ expect(DynamoDB).toHaveBeenCalledWith(
+ expect.objectContaining({
+ region: 'eu-west-2'
+ })
+ )
+ expect(DynamoDBDocument.from).toHaveBeenCalledWith(expect.any(DynamoDB))
+ })
+
+ it('configures sqs with a custom endpoint if one is defined in configuration', () => {
+ const TEST_ENDPOINT = 'http://localhost:8080'
+ Config.aws.sqs.endpoint = TEST_ENDPOINT
+ require('../aws.js').default()
+ expect(SQS).toHaveBeenCalledWith(
+ expect.objectContaining({
+ apiVersion: '2012-11-05',
+ endpoint: TEST_ENDPOINT
+ })
+ )
  })
 
- it('uses the default sqs endpoint if it is not overridden in configuration', async () => {
+ it('uses the default sqs endpoint if it is not overridden in configuration', () => {
  process.env.AWS_REGION = 'eu-west-2'
  delete Config.aws.sqs.endpoint
- const { sqs } = require('../aws.js').default()
- expect(sqs.config.endpoint).toEqual('sqs.eu-west-2.amazonaws.com')
+ require('../aws.js').default()
+ expect(SQS).toHaveBeenCalledWith(
+ expect.objectContaining({
+ apiVersion: '2012-11-05'
+ })
+ )
  })
 
- it('configures s3 with a custom endpoint if one is defined in configuration', async () => {
+ it('configures s3 with a custom endpoint if one is defined in configuration', () => {
+ const TEST_ENDPOINT = 'http://localhost:8080'
  Config.aws.s3.endpoint = TEST_ENDPOINT
- const { s3 } = require('../aws.js').default()
- expect(s3.config.endpoint).toEqual(TEST_ENDPOINT)
- expect(s3.config.s3ForcePathStyle).toBeTruthy()
+ require('../aws.js').default()
+ expect(S3).toHaveBeenCalledWith(
+ expect.objectContaining({
+ apiVersion: '2006-03-01',
+ endpoint: TEST_ENDPOINT,
+ s3ForcePathStyle: true
+ })
+ )
  })
 
- it('uses default s3 settings if a custom endpoint is not defined', async () => {
+ it('uses default s3 settings if a custom endpoint is not defined', () => {
  process.env.AWS_REGION = 'eu-west-2'
  delete Config.aws.s3.endpoint
- const { s3 } = require('../aws.js').default()
- expect(s3.config.endpoint).toEqual('s3.eu-west-2.amazonaws.com')
- expect(s3.config.s3ForcePathStyle).toBeFalsy()
+ require('../aws.js').default()
+ expect(S3).toHaveBeenCalledWith(
+ expect.objectContaining({
+ apiVersion: '2006-03-01'
+ })
+ )
  })
 
- it('configures secretsmanager with a custom endpoint if one is defined in configuration', async () => {
+ it('configures secretsmanager with a custom endpoint if one is defined in configuration', () => {
+ const TEST_ENDPOINT = 'http://localhost:8080'
  Config.aws.secretsManager.endpoint = TEST_ENDPOINT
- const { secretsManager } = require('../aws.js').default()
- expect(secretsManager.config.endpoint).toEqual(TEST_ENDPOINT)
+ require('../aws.js').default()
+ expect(SecretsManager).toHaveBeenCalledWith(
+ expect.objectContaining({
+ apiVersion: '2017-10-17',
+ endpoint: TEST_ENDPOINT
+ })
+ )
  })
 
- it('uses default secretsmanager settings if a custom endpoint is not defined', async () => {
+ it('uses default secretsmanager settings if a custom endpoint is not defined', () => {
  process.env.AWS_REGION = 'eu-west-2'
  delete Config.aws.secretsManager.endpoint
- const { secretsManager } = require('../aws.js').default()
- expect(secretsManager.config.endpoint).toEqual('secretsmanager.eu-west-2.amazonaws.com')
+ require('../aws.js').default()
+ expect(SecretsManager).toHaveBeenCalledWith(
+ expect.objectContaining({
+ apiVersion: '2017-10-17'
+ })
+ )
  })
 })

packages/connectors-lib/src/__tests__/document-client-decorator.spec.js

@@ -1,85 +1,55 @@
-import AWSSdk from 'aws-sdk'
+import { DynamoDBDocumentClient, QueryCommand, ScanCommand, BatchWriteCommand } from '@aws-sdk/lib-dynamodb'
+import { mockClient } from 'aws-sdk-client-mock'
 import AWS from '../aws.js'
 const { docClient } = AWS()
 
 describe('document client decorations', () => {
+ const ddbMock = mockClient(DynamoDBDocumentClient)
+ beforeEach(() => {
+ ddbMock.reset()
+ })
+
  it('deals with pagination where DynamoDB returns a LastEvaluatedKey in a query response', async () => {
  const testLastEvaluatedKey = { id: '16324258-85-92746491' }
 
- AWSSdk.DynamoDB.DocumentClient.__setNextResponses(
- 'query',
- {
- Items: [],
- LastEvaluatedKey: testLastEvaluatedKey
- },
- {
- Items: []
- }
- )
- await docClient.queryAllPromise({
- TableName: 'TEST'
- })
- expect(AWSSdk.DynamoDB.DocumentClient.mockedMethods.query).toHaveBeenNthCalledWith(
- 1,
- expect.objectContaining({
- TableName: 'TEST'
- })
- )
- expect(AWSSdk.DynamoDB.DocumentClient.mockedMethods.query).toHaveBeenNthCalledWith(
- 2,
- expect.objectContaining({
- TableName: 'TEST',
- ExclusiveStartKey: testLastEvaluatedKey
- })
- )
+ // mock QueryCommand to return items with a LastEvaluatedKey
+ ddbMock.on(QueryCommand).resolvesOnce({ Items: [], LastEvaluatedKey: testLastEvaluatedKey }).resolvesOnce({ Items: [] })
+
+ await docClient.queryAllPromise({ TableName: 'TEST' })
+
+ // check QueryCommand was called twice & with the correct parameters
+ expect(ddbMock.send.callCount).toBe(2)
+ expect(ddbMock.send.firstCall.args[0].input.TableName).toEqual('TEST')
+ expect(ddbMock.send.secondCall.args[0].input.ExclusiveStartKey).toEqual(testLastEvaluatedKey)
  })
 
  it('deals with pagination where DynamoDB returns a LastEvaluatedKey in a scan response', async () => {
  const testLastEvaluatedKey = { id: '16324258-85-92746491' }
 
- AWSSdk.DynamoDB.DocumentClient.__setNextResponses(
- 'scan',
- {
- Items: [],
- LastEvaluatedKey: testLastEvaluatedKey
- },
- {
- Items: []
- }
- )
- await docClient.scanAllPromise({
- TableName: 'TEST'
- })
- expect(AWSSdk.DynamoDB.DocumentClient.mockedMethods.scan).toHaveBeenNthCalledWith(
- 1,
- expect.objectContaining({
- TableName: 'TEST'
- })
- )
- expect(AWSSdk.DynamoDB.DocumentClient.mockedMethods.scan).toHaveBeenNthCalledWith(
- 2,
- expect.objectContaining({
- TableName: 'TEST',
- ExclusiveStartKey: testLastEvaluatedKey
- })
- )
+ // mock ScanCommand to return items with a LastEvaluatedKey
+ ddbMock.on(ScanCommand).resolvesOnce({ Items: [], LastEvaluatedKey: testLastEvaluatedKey }).resolvesOnce({ Items: [] })
+
+ await docClient.scanAllPromise({ TableName: 'TEST' })
+
+ // check ScanCommand was called twicce & with the correct parameters
+ expect(ddbMock.send.callCount).toBe(2)
+ expect(ddbMock.send.firstCall.args[0].input.TableName).toEqual('TEST')
+ expect(ddbMock.send.secondCall.args[0].input.ExclusiveStartKey).toEqual(testLastEvaluatedKey)
  })
 
  it('deals with UnprocessedItems when making batchWrite requests to DynamoDB', async () => {
- AWSSdk.DynamoDB.DocumentClient.__setNextResponses(
- 'batchWrite',
- {
+ // mock BatchWriteCommand to return UnprocessedItems
+ ddbMock
+ .on(BatchWriteCommand)
+ .resolvesOnce({
  UnprocessedItems: {
  NameOfTableToUpdate: [
  { PutRequest: { Item: { key: '1', field: 'data1' } } },
  { PutRequest: { Item: { key: '2', field: 'data2' } } }
  ]
  }
- },
- {
- UnprocessedItems: null
- }
- )
+ })
+ .resolvesOnce({ UnprocessedItems: null })
  await docClient.batchWriteAllPromise({
  RequestItems: {
  NameOfTableToUpdate: [
@@ -89,30 +59,11 @@ describe('document client decorations', () => {
  ]
  }
  })
- expect(AWSSdk.DynamoDB.DocumentClient.mockedMethods.batchWrite).toHaveBeenCalledTimes(2)
- expect(AWSSdk.DynamoDB.DocumentClient.mockedMethods.batchWrite).toHaveBeenNthCalledWith(
- 1,
- expect.objectContaining({
- RequestItems: {
- NameOfTableToUpdate: [
- { PutRequest: { Item: { key: '1', field: 'data1' } } },
- { PutRequest: { Item: { key: '2', field: 'data2' } } },
- { PutRequest: { Item: { key: '3', field: 'data3' } } }
- ]
- }
- })
- )
- expect(AWSSdk.DynamoDB.DocumentClient.mockedMethods.batchWrite).toHaveBeenNthCalledWith(
- 2,
- expect.objectContaining({
- RequestItems: {
- NameOfTableToUpdate: [
- { PutRequest: { Item: { key: '1', field: 'data1' } } },
- { PutRequest: { Item: { key: '2', field: 'data2' } } }
- ]
- }
- })
- )
+
+ // check BatchWriteCommand was called twice & with the correct parameters
+ expect(ddbMock.send.callCount).toBe(2)
+ expect(ddbMock.send.firstCall.args[0].input.RequestItems.NameOfTableToUpdate).toHaveLength(3)
+ expect(ddbMock.send.secondCall.args[0].input.RequestItems.NameOfTableToUpdate).toHaveLength(2)
  })
 
  it('deals with UnprocessedItems when making batchWrite requests to DynamoDB up to the given retry limit', async () => {
@@ -124,7 +75,8 @@ describe('document client decorations', () => {
  ]
  }
  })
- AWSSdk.DynamoDB.DocumentClient.__setNextResponses('batchWrite', ...batchWriteResponses)
+ ddbMock.on(BatchWriteCommand).resolves(...batchWriteResponses)
+
  const request = {
  RequestItems: {
  NameOfTableToUpdate: [
@@ -137,7 +89,7 @@ describe('document client decorations', () => {
  // Don't delay on setTimeouts!
  jest.spyOn(global, 'setTimeout').mockImplementation(cb => cb())
  await expect(docClient.batchWriteAllPromise(request)).rejects.toThrow(
- 'Failed to write items to DynamoDB using batch write.  UnprocessedItems were returned and maxRetries has been reached.'
+ 'Failed to write items to DynamoDB using batch write. UnprocessedItems were returned and maxRetries has been reached.'
  )
  })
 

packages/connectors-lib/src/aws.js

@@ -1,16 +1,20 @@
 import Config from './config.js'
+import { DynamoDB } from '@aws-sdk/client-dynamodb'
 import { createDocumentClient } from './documentclient-decorator.js'
 import AWS from 'aws-sdk'
-const { DynamoDB, SQS, S3, SecretsManager } = AWS
+
+const { SQS, S3, SecretsManager } = AWS
 
 export default function () {
+ const dynamoDBInstance = new DynamoDB({
+ apiVersion: '2012-08-10',
+ ...(Config.aws.dynamodb.endpoint && {
+ endpoint: Config.aws.dynamodb.endpoint
+ })
+ })
+
  return {
- ddb: new DynamoDB({
- apiVersion: '2012-08-10',
- ...(Config.aws.dynamodb.endpoint && {
- endpoint: Config.aws.dynamodb.endpoint
- })
- }),
+ ddb: dynamoDBInstance,
  docClient: createDocumentClient({
  convertEmptyValues: true,
  apiVersion: '2012-08-10',