diff --git a/packages/amplify-e2e-core/package.json b/packages/amplify-e2e-core/package.json index 66e5c26578..ece0fcb014 100644 --- a/packages/amplify-e2e-core/package.json +++ b/packages/amplify-e2e-core/package.json @@ -27,6 +27,7 @@ "@aws-sdk/client-sts": "3.338.0", "@aws-sdk/credential-providers": "3.338.0", "amplify-headless-interface": "^1.17.3", + "axios": "^0.26.0", "chalk": "^4.1.1", "execa": "^5.1.1", "fs-extra": "^8.1.0", diff --git a/packages/amplify-e2e-core/src/utils/rds.ts b/packages/amplify-e2e-core/src/utils/rds.ts index c1e68a1104..1986a39595 100644 --- a/packages/amplify-e2e-core/src/utils/rds.ts +++ b/packages/amplify-e2e-core/src/utils/rds.ts @@ -7,11 +7,15 @@ import { } from '@aws-sdk/client-rds'; import { EC2Client, AuthorizeSecurityGroupIngressCommand, RevokeSecurityGroupIngressCommand } from '@aws-sdk/client-ec2'; import { knex } from 'knex'; +import axios from 'axios'; const DEFAULT_DB_INSTANCE_TYPE = 'db.m5.large'; const DEFAULT_DB_STORAGE = 8; const DEFAULT_SECURITY_GROUP = 'default'; +const IPIFY_URL = 'https://api.ipify.org/'; +const AWSCHECKIP_URL = 'https://checkip.amazonaws.com/'; + /** * Creates a new RDS instance using the given input configuration and returns the details of the created RDS instance. * @param config Configuration of the database instance @@ -268,3 +272,13 @@ export const getResource = (resources: Map, resourcePrefix: string, } return undefined; }; + +export const getIpRanges = async (): Promise => { + return Promise.all( + [IPIFY_URL, AWSCHECKIP_URL].map(async (url) => { + const response = await axios(url); + const ipParts = response.data.trim().split('.'); + return `${ipParts[0]}.${ipParts[1]}.0.0/16`; + }), + ); +}; diff --git a/packages/amplify-e2e-tests/src/__tests__/rds-import-vpc.test.ts b/packages/amplify-e2e-tests/src/__tests__/rds-import-vpc.test.ts index d9bf8b60f4..3a73ece277 100644 --- a/packages/amplify-e2e-tests/src/__tests__/rds-import-vpc.test.ts +++ b/packages/amplify-e2e-tests/src/__tests__/rds-import-vpc.test.ts @@ -212,8 +212,6 @@ describe('RDS Tests', () => { }, }); - // VPC will not have VPC endpoints for SSM defined and the security group's inbound rule for port 443 is not defined. - // Expect the listComponents query to fail with an error. expect(appSyncClient).toBeDefined(); const result = await listComponents(appSyncClient); diff --git a/packages/amplify-e2e-tests/src/__tests__/rds-model-v2.test.ts b/packages/amplify-e2e-tests/src/__tests__/rds-model-v2.test.ts index 02eb9c9e0e..3dcafe0d2c 100644 --- a/packages/amplify-e2e-tests/src/__tests__/rds-model-v2.test.ts +++ b/packages/amplify-e2e-tests/src/__tests__/rds-model-v2.test.ts @@ -133,6 +133,7 @@ describe('RDS Model Directive', () => { // 1. Create a RDS Instance // 2. Add the external IP address of the current machine to security group inbound rule to allow public access // 3. Connect to the database and execute DDL + // 4. Remove the inbound rules. const db = await createRDSInstance({ identifier, @@ -172,11 +173,7 @@ describe('RDS Model Directive', () => { 'CREATE TABLE Student (studentId INT NOT NULL, classId CHAR(1) NOT NULL, FirstName VARCHAR(20), LastName VARCHAR(50), PRIMARY KEY (studentId, classId))', ]); dbAdapter.cleanup(); - }; - const cleanupDatabase = async (): Promise => { - // 1. Remove the IP address from the security group - // 2. Delete the RDS instance await Promise.all( ipAddresses.map((ip) => removeRDSPortInboundRule({ region, @@ -185,6 +182,9 @@ describe('RDS Model Directive', () => { }), ), ); + }; + + const cleanupDatabase = async (): Promise => { await deleteDBInstance(identifier, region); }; @@ -204,7 +204,7 @@ describe('RDS Model Directive', () => { port, username, password, - useVpc: false, + useVpc: true, apiExists: true, }); diff --git a/packages/amplify-e2e-tests/src/__tests__/rds-refers-to.test.ts b/packages/amplify-e2e-tests/src/__tests__/rds-refers-to.test.ts index 00ad373e8f..b1af50205c 100644 --- a/packages/amplify-e2e-tests/src/__tests__/rds-refers-to.test.ts +++ b/packages/amplify-e2e-tests/src/__tests__/rds-refers-to.test.ts @@ -3,31 +3,30 @@ import { addApiWithoutSchema, addRDSPortInboundRule, amplifyPush, - apiGenerateSchema, createNewProjectDir, createRDSInstance, deleteDBInstance, deleteProject, deleteProjectDir, getAppSyncApi, + getIpRanges, getProjectMeta, importRDSDatabase, initJSProjectWithProfile, removeRDSPortInboundRule, } from 'amplify-category-api-e2e-core'; -import { existsSync, writeFileSync, mkdirSync, readFileSync } from 'fs-extra'; +import { existsSync, writeFileSync } from 'fs-extra'; import generator from 'generate-password'; import path from 'path'; import AWSAppSyncClient, { AUTH_TYPE } from 'aws-appsync'; import { GQLQueryHelper } from '../query-utils/gql-helper'; -import { gql } from 'graphql-transformer-core'; -import { ObjectTypeDefinitionNode, parse } from 'graphql'; // to deal with bug in cognito-identity-js (global as any).fetch = require('node-fetch'); describe('RDS Relational Directives', () => { const publicIpCidr = '0.0.0.0/0'; + const ipAddresses = []; const [db_user, db_password, db_identifier] = generator.generateMultiple(3); // Generate settings for RDS instance @@ -96,11 +95,16 @@ describe('RDS Relational Directives', () => { }); port = db.port; host = db.endpoint; - await addRDSPortInboundRule({ - region, - port: db.port, - cidrIp: publicIpCidr, - }); + + ipAddresses.push(...(await getIpRanges())); + await Promise.all( + ipAddresses.map((ip) => addRDSPortInboundRule({ + region, + port: db.port, + cidrIp: ip, + }), + ), + ); const dbAdapter = new RDSTestDataProvider({ host: db.endpoint, @@ -118,16 +122,18 @@ describe('RDS Relational Directives', () => { 'CREATE TABLE Task (id VARCHAR(40) PRIMARY KEY, description VARCHAR(255))', ]); dbAdapter.cleanup(); + + await Promise.all( + ipAddresses.map((ip) => removeRDSPortInboundRule({ + region, + port, + cidrIp: ip, + }), + ), + ); }; const cleanupDatabase = async (): Promise => { - // 1. Remove the IP address from the security group - // 2. Delete the RDS instance - await removeRDSPortInboundRule({ - region, - port: port, - cidrIp: publicIpCidr, - }); await deleteDBInstance(identifier, region); }; @@ -147,7 +153,7 @@ describe('RDS Relational Directives', () => { port, username, password, - useVpc: false, + useVpc: true, apiExists: true, }); diff --git a/packages/amplify-e2e-tests/src/__tests__/rds-relational-directives.test.ts b/packages/amplify-e2e-tests/src/__tests__/rds-relational-directives.test.ts index 5fadecdfbf..75e4446ded 100644 --- a/packages/amplify-e2e-tests/src/__tests__/rds-relational-directives.test.ts +++ b/packages/amplify-e2e-tests/src/__tests__/rds-relational-directives.test.ts @@ -10,6 +10,7 @@ import { deleteProject, deleteProjectDir, getAppSyncApi, + getIpRanges, getProjectMeta, importRDSDatabase, initJSProjectWithProfile, @@ -28,6 +29,7 @@ import { ObjectTypeDefinitionNode, parse } from 'graphql'; describe('RDS Relational Directives', () => { const publicIpCidr = '0.0.0.0/0'; + const ipAddresses = []; const [db_user, db_password, db_identifier] = generator.generateMultiple(3); // Generate settings for RDS instance @@ -101,11 +103,16 @@ describe('RDS Relational Directives', () => { }); port = db.port; host = db.endpoint; - await addRDSPortInboundRule({ - region, - port: db.port, - cidrIp: publicIpCidr, - }); + + ipAddresses.push(...(await getIpRanges())); + await Promise.all( + ipAddresses.map((ip) => addRDSPortInboundRule({ + region, + port: db.port, + cidrIp: ip, + }), + ), + ); const dbAdapter = new RDSTestDataProvider({ host: db.endpoint, @@ -131,16 +138,18 @@ describe('RDS Relational Directives', () => { "INSERT INTO ZipCode VALUES ('20160', 'Lincoln', 'VA', 'US')", ]); dbAdapter.cleanup(); + + await Promise.all( + ipAddresses.map((ip) => removeRDSPortInboundRule({ + region, + port, + cidrIp: ip, + }), + ), + ); }; const cleanupDatabase = async (): Promise => { - // 1. Remove the IP address from the security group - // 2. Delete the RDS instance - await removeRDSPortInboundRule({ - region, - port: port, - cidrIp: publicIpCidr, - }); await deleteDBInstance(identifier, region); }; @@ -173,7 +182,7 @@ describe('RDS Relational Directives', () => { port, username, password, - useVpc: false, + useVpc: true, apiExists: true, }); diff --git a/packages/amplify-e2e-tests/src/__tests__/rds-v2-generate-schema.test.ts b/packages/amplify-e2e-tests/src/__tests__/rds-v2-generate-schema.test.ts index e47bfaf3c5..77e910fcbc 100644 --- a/packages/amplify-e2e-tests/src/__tests__/rds-v2-generate-schema.test.ts +++ b/packages/amplify-e2e-tests/src/__tests__/rds-v2-generate-schema.test.ts @@ -12,6 +12,7 @@ import { removeRDSPortInboundRule, apiGenerateSchema, apiGenerateSchemaWithError, + getIpRanges, } from 'amplify-category-api-e2e-core'; import axios from 'axios'; import { existsSync, readFileSync, writeFileSync } from 'fs-extra'; @@ -21,6 +22,7 @@ import path from 'path'; describe('RDS Generate Schema tests', () => { let publicIpCidr = '0.0.0.0/0'; + const ipAddresses = []; const [db_user, db_password, db_identifier] = generator.generateMultiple(3); // Generate settings for RDS instance @@ -55,7 +57,7 @@ describe('RDS Generate Schema tests', () => { port, username, password, - useVpc: false, + useVpc: true, apiExists: true, }); }); @@ -92,11 +94,16 @@ describe('RDS Generate Schema tests', () => { }); port = db.port; host = db.endpoint; - await addRDSPortInboundRule({ - region, - port: db.port, - cidrIp: publicIpCidr, - }); + + ipAddresses.push(...(await getIpRanges())); + await Promise.all( + ipAddresses.map((ip) => addRDSPortInboundRule({ + region, + port: db.port, + cidrIp: ip, + }), + ), + ); const dbAdapter = new RDSTestDataProvider({ host: db.endpoint, @@ -111,16 +118,18 @@ describe('RDS Generate Schema tests', () => { 'CREATE TABLE tbl_todos (ID INT PRIMARY KEY, description VARCHAR(20))', ]); dbAdapter.cleanup(); + + await Promise.all( + ipAddresses.map((ip) => removeRDSPortInboundRule({ + region, + port, + cidrIp: ip, + }), + ), + ); }; const cleanupDatabase = async () => { - // 1. Remove the IP address from the security group - // 2. Delete the RDS instance - await removeRDSPortInboundRule({ - region, - port: port, - cidrIp: publicIpCidr, - }); await deleteDBInstance(identifier, region); }; diff --git a/packages/amplify-e2e-tests/src/__tests__/rds-v2.test.ts b/packages/amplify-e2e-tests/src/__tests__/rds-v2.test.ts index 0e56f77912..76527356d2 100644 --- a/packages/amplify-e2e-tests/src/__tests__/rds-v2.test.ts +++ b/packages/amplify-e2e-tests/src/__tests__/rds-v2.test.ts @@ -7,6 +7,7 @@ import { deleteDBInstance, deleteProject, deleteProjectDir, + getIpRanges, importRDSDatabase, initJSProjectWithProfile, removeRDSPortInboundRule, @@ -18,7 +19,8 @@ import { ObjectTypeDefinitionNode, parse } from 'graphql'; import path from 'path'; describe('RDS Tests', () => { - let publicIpCidr = '0.0.0.0/0'; + const publicIpCidr = '0.0.0.0/0'; + const ipAddresses = []; const [db_user, db_password, db_identifier] = generator.generateMultiple(3); const RDS_MAPPING_FILE = 'https://amplify-rds-layer-resources.s3.amazonaws.com/rds-layer-mapping.json'; @@ -34,10 +36,6 @@ describe('RDS Tests', () => { let projRoot; beforeAll(async () => { - // Get the public IP of the machine running the test - const url = 'http://api.ipify.org/'; - const response = await axios(url); - publicIpCidr = `${response.data.trim()}/32`; await setupDatabase(); }); @@ -73,11 +71,16 @@ describe('RDS Tests', () => { }); port = db.port; host = db.endpoint; - await addRDSPortInboundRule({ - region, - port: db.port, - cidrIp: publicIpCidr, - }); + + ipAddresses.push(...(await getIpRanges())); + await Promise.all( + ipAddresses.map((ip) => addRDSPortInboundRule({ + region, + port: db.port, + cidrIp: ip, + }), + ), + ); const dbAdapter = new RDSTestDataProvider({ host: db.endpoint, @@ -92,16 +95,18 @@ describe('RDS Tests', () => { 'CREATE TABLE Employee (ID INT PRIMARY KEY, FirstName VARCHAR(20), LastName VARCHAR(50))', ]); dbAdapter.cleanup(); + + await Promise.all( + ipAddresses.map((ip) => removeRDSPortInboundRule({ + region, + port, + cidrIp: ip, + }), + ), + ); }; const cleanupDatabase = async () => { - // 1. Remove the IP address from the security group - // 2. Delete the RDS instance - await removeRDSPortInboundRule({ - region, - port: port, - cidrIp: publicIpCidr, - }); await deleteDBInstance(identifier, region); }; @@ -120,7 +125,7 @@ describe('RDS Tests', () => { port, username, password, - useVpc: false, + useVpc: true, apiExists: true, }); diff --git a/packages/amplify-graphql-model-transformer/rds-lambda/handler.ts b/packages/amplify-graphql-model-transformer/rds-lambda/handler.ts index dea276b9c4..fef28880ba 100644 --- a/packages/amplify-graphql-model-transformer/rds-lambda/handler.ts +++ b/packages/amplify-graphql-model-transformer/rds-lambda/handler.ts @@ -1,4 +1,4 @@ -import { SSMClient, GetParameterCommand } from '@aws-sdk/client-ssm'; +import { SSMClient, GetParameterCommand, GetParameterCommandOutput } from '@aws-sdk/client-ssm'; // @ts-ignore import { DBAdapter, DBConfig, getDBAdapter } from 'rds-query-processor'; @@ -6,6 +6,7 @@ let adapter: DBAdapter; let secretsClient: SSMClient; const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); +const WAIT_COMPLETE = 'WAIT_COMPLETE'; export const run = async (event): Promise => { if (!adapter) { @@ -25,10 +26,9 @@ const createSSMClient = (): void => { }); }; -const wait10SecondsAndThrowError = async (): Promise => { +const wait10Seconds = async (): Promise => { await delay(10000); - console.log('Unable to retrieve secret for database connection from SSM. If your database is in VPC, verify that you have VPC endpoints for SSM defined and the security group\'s inbound rule for port 443 is defined.'); - throw new Error('Unable to get the database credentials. Check the logs for more details.'); + return WAIT_COMPLETE; }; const getSSMValue = async (key: string | undefined): Promise => { @@ -44,11 +44,23 @@ const getSSMValue = async (key: string | undefined): Promise => { // the security group's inbound rule for port 443 is not defined, // the SSM client waits for the entire lambda execution time and times out. // If the parameter is not retrieved within 10 seconds, throw an error. - const data = await Promise.race([secretsClient.send(parameterCommand), wait10SecondsAndThrowError()]); - if ((data?.$metadata?.httpStatusCode && data?.$metadata?.httpStatusCode >= 400) || !data?.Parameter?.Value) { + const data = await Promise.race([secretsClient.send(parameterCommand), wait10Seconds()]); + + // If string is returned, throw error. + if ( + (typeof data === 'string' || data instanceof String) && + data === WAIT_COMPLETE + ) { + console.log('Unable to retrieve secret for database connection from SSM. If your database is in VPC, verify that you have VPC endpoints for SSM defined and the security group\'s inbound rule for port 443 is defined.'); + throw new Error('Unable to get the database credentials. Check the logs for more details.'); + } + + // Read the value from the GetParameter response. + const response = data as GetParameterCommandOutput; + if ((response?.$metadata?.httpStatusCode && response?.$metadata?.httpStatusCode >= 400) || !response?.Parameter?.Value) { throw new Error('Unable to get secret for database connection'); } - return data.Parameter.Value; + return response.Parameter.Value; }; const getDBConfig = async (): DBConfig => {