Skip to content

Commit

Permalink
Fixed CDK lambda function bugs which were causing timeout and unautho…
Browse files Browse the repository at this point in the history
…rized errors when using the deployed GraphQL API app:

-VPC config now has the appropriate subnets and security group configured
-policy actions now reference the correct aws service depending on the neptune type
-environment now references the correct region when the neptune type is neptune-graph
-refactored endpoint parsing to use new util function so that the logic is contained in place and parsing is done in one function call instead of sprinkled throughout the code
-added unit tests for endpoint parsing
-added integration test verification for CDK deployment to check that the generated CDK js file contains some expected content
  • Loading branch information
andreachild committed Oct 17, 2024
1 parent 909ed51 commit 5f41c81
Show file tree
Hide file tree
Showing 9 changed files with 178 additions and 138 deletions.
4 changes: 4 additions & 0 deletions src/CDKPipelineApp.js
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,10 @@ async function createAWSpipelineCDK({
CDKFile = CDKFile.replace( "const NEPTUNE_DB_NAME = '';", `const NEPTUNE_DB_NAME = '${NEPTUNE_DB_NAME}';` );
CDKFile = CDKFile.replace( "const NEPTUNE_TYPE = '';", `const NEPTUNE_TYPE = '${NEPTUNE_TYPE}';` );
CDKFile = CDKFile.replace( "const NEPTUNE_DBSubnetGroup = null;", `const NEPTUNE_DBSubnetGroup = '${NEPTUNE_DBSubnetGroup}';` );
if (neptuneClusterInfo) {
CDKFile = CDKFile.replace("const NEPTUNE_DBSubnetIds = null;", `const NEPTUNE_DBSubnetIds = '${neptuneClusterInfo.dbSubnetIds}';`);
CDKFile = CDKFile.replace("const NEPTUNE_VpcSecurityGroupId = null;",`const NEPTUNE_VpcSecurityGroupId = '${neptuneClusterInfo.vpcSecurityGroupId}';`);
}
CDKFile = CDKFile.replace( "const NEPTUNE_IAM_AUTH = false;", `const NEPTUNE_IAM_AUTH = ${isNeptuneIAMAuth};` );
CDKFile = CDKFile.replace( "const NEPTUNE_IAM_POLICY_RESOURCE = '*';", `const NEPTUNE_IAM_POLICY_RESOURCE = '${NEPTUNE_IAM_POLICY_RESOURCE}';` );

Expand Down
19 changes: 10 additions & 9 deletions src/NeptuneSchema.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import { aws4Interceptor } from "aws4-axios";
import { fromNodeProviderChain } from "@aws-sdk/credential-providers";
import { NeptunedataClient, ExecuteOpenCypherQueryCommand } from "@aws-sdk/client-neptunedata";
import { loggerDebug, loggerError, loggerInfo, yellow } from "./logger.js";
import { parseNeptuneDomainFromHost, parseNeptuneGraphName } from "./util.js";
import { ExecuteQueryCommand, GetGraphSummaryCommand, NeptuneGraphClient } from "@aws-sdk/client-neptune-graph";

const NEPTUNE_DB = 'neptune-db';
Expand All @@ -25,7 +24,8 @@ const HTTP_LANGUAGE = 'openCypher';
const NEPTUNE_GRAPH_LANGUAGE = 'OPEN_CYPHER';
let HOST = '';
let PORT = 8182;
let REGION = ''
let REGION = '';
let DOMAIN = '';
let SAMPLE = 5000;
let NEPTUNE_TYPE = NEPTUNE_DB;
let NAME = '';
Expand Down Expand Up @@ -331,12 +331,13 @@ async function getEdgesDirectionsCardinality() {
}


function setGetNeptuneSchemaParameters(host, port, region, neptuneType) {
HOST = host;
PORT = port;
REGION = region;
NEPTUNE_TYPE = neptuneType;
NAME = parseNeptuneGraphName(host);
function setGetNeptuneSchemaParameters(neptuneInfo) {
HOST = neptuneInfo.host;
PORT = neptuneInfo.port;
REGION = neptuneInfo.region;
NEPTUNE_TYPE = neptuneInfo.neptuneType;
NAME = neptuneInfo.graphName;
DOMAIN = neptuneInfo.domain;
}

function getNeptunedataClient() {
Expand All @@ -354,7 +355,7 @@ function getNeptuneGraphClient() {
loggerInfo('Instantiating NeptuneGraphClient')
neptuneGraphClient = new NeptuneGraphClient({
port: PORT,
host: parseNeptuneDomainFromHost(HOST),
host: DOMAIN,
region: REGION,
protocol: NEPTUNE_GRAPH_PROTOCOL,
});
Expand Down
107 changes: 53 additions & 54 deletions src/main.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ let spinner = null;
// find global installation dir
import path from 'path';
import { fileURLToPath } from 'url';
import { parseNeptuneDomainFromEndpoint } from "./util.js";
import { parseNeptuneEndpoint } from "./util.js";
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

Expand Down Expand Up @@ -63,7 +63,7 @@ let createUpdatePipelineNeptuneDatabaseName = '';
let removePipelineName = '';
let inputCDKpipeline = false;
let inputCDKpipelineName = '';
let inputCDKpipelineEnpoint = '';
let inputCDKpipelineEndpoint = '';
let inputCDKpipelineFile = '';
let inputCDKpipelineRegion = '';
let inputCDKpipelineDatabaseName = '';
Expand Down Expand Up @@ -212,10 +212,14 @@ function processArgs() {
inputCDKpipeline = true;
break;
case '-ce':
// support miss-spelled option for backwards compatibility - could be removed for next major release
case '--output-aws-pipeline-cdk-neptume-endpoint':
case '--output-aws-pipeline-cdk-neptune-endpoint':
inputCDKpipelineEnpoint = array[index + 1];
inputCDKpipelineEndpoint = array[index + 1];
break;
case '-cd':
// support miss-spelled option for backwards compatibility - could be removed for next major release
case '--output-aws-pipeline-cdk-neptume-database-name':
case '--output-aws-pipeline-cdk-neptune-database-name':
inputCDKpipelineDatabaseName = array[index + 1];
break;
Expand Down Expand Up @@ -285,37 +289,31 @@ async function main() {
}
}


let neptuneInfo;
// Check if any of the Neptune endpoints are a neptune analytic endpoint and if so, set the neptuneType and IAM to required
const nonEmptyEndpoints = [inputGraphDBSchemaNeptuneEndpoint, createUpdatePipelineEndpoint, inputCDKpipelineEnpoint].filter(endpoint => endpoint !== '');
const isNeptuneAnalyticsGraph = nonEmptyEndpoints.length > 0 && parseNeptuneDomainFromEndpoint(nonEmptyEndpoints[0]).includes(NEPTUNE_GRAPH);
if (isNeptuneAnalyticsGraph) {
neptuneType = NEPTUNE_GRAPH;
// neptune analytics requires IAM
loggerInfo("Detected neptune-graph from input endpoint - setting IAM auth to true as it is required for neptune analytics")
isNeptuneIAMAuth = true;
// only one of these endpoints are expected to be non-empty at the same time
const nonEmptyEndpoints = [inputGraphDBSchemaNeptuneEndpoint, createUpdatePipelineEndpoint, inputCDKpipelineEndpoint].filter(endpoint => endpoint !== '');
if (nonEmptyEndpoints.length > 0) {
neptuneInfo = parseNeptuneEndpoint(nonEmptyEndpoints[0]);
neptuneType = neptuneInfo.neptuneType;
if (neptuneType === NEPTUNE_GRAPH) {
// neptune analytics requires IAM
loggerInfo("Detected neptune-graph from input endpoint - setting IAM auth to true as it is required for neptune analytics")
isNeptuneIAMAuth = true;
}
}

// Get Neptune schema from endpoint
if (inputGraphDBSchemaNeptuneEndpoint != '' && inputGraphDBSchema == '' && inputGraphDBSchemaFile == '') {
let endpointParts = inputGraphDBSchemaNeptuneEndpoint.split(':');
if (endpointParts.length != 2) {
loggerError('Neptune endpoint must be in the form of host:port');
process.exit(1);
if (!neptuneInfo) {
neptuneInfo = parseNeptuneEndpoint(inputGraphDBSchemaNeptuneEndpoint);
}
let neptuneHost = endpointParts[0];
let neptunePort = endpointParts[1];

let neptuneRegionParts = inputGraphDBSchemaNeptuneEndpoint.split('.');
let neptuneRegion = '';
if (neptuneType === NEPTUNE_DB)
neptuneRegion = neptuneRegionParts[2];
else
neptuneRegion = neptuneRegionParts[1];

loggerInfo('Retrieving Neptune schema');
loggerDebug('Getting Neptune schema from endpoint: ' + yellow(neptuneHost + ':' + neptunePort), {toConsole: true});
loggerDebug('Getting Neptune schema from endpoint: ' + yellow(inputGraphDBSchemaNeptuneEndpoint), {toConsole: true});

setGetNeptuneSchemaParameters(neptuneHost, neptunePort, neptuneRegion, neptuneType);
setGetNeptuneSchemaParameters(neptuneInfo);
let startTime = performance.now();
inputGraphDBSchema = await getNeptuneSchema();
let endTime = performance.now();
Expand Down Expand Up @@ -372,15 +370,11 @@ async function main() {
process.exit(1);
}
if (createUpdatePipelineEndpoint != '') {
let parts = createUpdatePipelineEndpoint.split('.');
createUpdatePipelineNeptuneDatabaseName = parts[0];

let parsedRegion;
if (neptuneType === NEPTUNE_DB) {
parsedRegion = parts[2];
} else {
parsedRegion = parts[1];
if (!neptuneInfo) {
neptuneInfo = parseNeptuneEndpoint(createUpdatePipelineEndpoint);
}
createUpdatePipelineNeptuneDatabaseName = neptuneInfo.graphName;
const parsedRegion = neptuneInfo.region;

if (createUpdatePipelineRegion !== parsedRegion) {
if (createUpdatePipelineRegion !== '') {
Expand All @@ -399,27 +393,37 @@ async function main() {
// CDK
if (inputCDKpipeline) {
if (!inputGraphDBSchemaNeptuneEndpoint == '') {
inputCDKpipelineEnpoint = inputGraphDBSchemaNeptuneEndpoint;
inputCDKpipelineEndpoint = inputGraphDBSchemaNeptuneEndpoint;
}
if (inputCDKpipelineEnpoint == '' &&
if (inputCDKpipelineEndpoint == '' &&
inputCDKpipelineRegion == '' && inputCDKpipelineDatabaseName == '') {
loggerError('AWS CDK: is required a Neptune endpoint, or a Neptune database name and region.');
process.exit(1);
}
if (inputCDKpipelineEnpoint == '' &&
if (inputCDKpipelineEndpoint == '' &&
!inputCDKpipelineRegion == '' && inputCDKpipelineDatabaseName == '') {
loggerError('AWS CDK: a Neptune database name is required.');
process.exit(1);
}
if (inputCDKpipelineEnpoint == '' &&
if (inputCDKpipelineEndpoint == '' &&
inputCDKpipelineRegion == '' && !inputCDKpipelineDatabaseName == '') {
loggerError('AWS CDK: a Neptune database region is required.');
process.exit(1);
}
if (inputCDKpipelineEnpoint != '') {
let parts = inputCDKpipelineEnpoint.split('.');
inputCDKpipelineDatabaseName = parts[0];
inputCDKpipelineRegion = parts[2];
if (inputCDKpipelineEndpoint != '') {
if (!neptuneInfo) {
neptuneInfo = parseNeptuneEndpoint(inputCDKpipelineEndpoint);
}
inputCDKpipelineDatabaseName = neptuneInfo.graphName;
const parsedRegion = neptuneInfo.region;
if (inputCDKpipelineRegion !== parsedRegion) {
if (inputCDKpipelineRegion !== '') {
loggerInfo('Switching CDK region from ' + inputCDKpipelineRegion + ' to region parsed from endpoint: ' + parsedRegion);
} else {
loggerInfo('Region parsed from CDK endpoint: ' + parsedRegion);
}
inputCDKpipelineRegion = parsedRegion;
}
}
if (inputCDKpipelineName == '') {
inputCDKpipelineName = inputCDKpipelineDatabaseName;
Expand Down Expand Up @@ -576,13 +580,11 @@ async function main() {
// Create Update AWS Pipeline
if (createUpdatePipeline) {
try {
let endpointParts = createUpdatePipelineEndpoint.split(':');
if (endpointParts.length < 2) {
loggerError('Neptune endpoint must be in the form of host:port');
process.exit(1);
if (!neptuneInfo) {
neptuneInfo = parseNeptuneEndpoint(createUpdatePipelineEndpoint);
}
let neptuneHost = endpointParts[0];
let neptunePort = endpointParts[1];
let neptuneHost = neptuneInfo.host;
let neptunePort = neptuneInfo.port;

await createUpdateAWSpipeline( createUpdatePipelineName,
createUpdatePipelineNeptuneDatabaseName,
Expand All @@ -608,14 +610,11 @@ async function main() {
try {
loggerInfo('Creating CDK File', {toConsole: true});

let endpointParts = inputCDKpipelineEnpoint.split(':');
if (endpointParts.length < 2) {
loggerError('Neptune endpoint must be in the form of host:port');
process.exit(1);
if (!neptuneInfo) {
neptuneInfo = parseNeptuneEndpoint(inputCDKpipelineEndpoint);
}
let neptuneHost = endpointParts[0];
let neptunePort = endpointParts[1];

let neptuneHost = neptuneInfo.host;
let neptunePort = neptuneInfo.port;

if (inputCDKpipelineFile == '') {
inputCDKpipelineFile = `${outputFolderPath}/${inputCDKpipelineName}-cdk.js`;
Expand Down
20 changes: 6 additions & 14 deletions src/pipelineResources.js
Original file line number Diff line number Diff line change
Expand Up @@ -330,19 +330,6 @@ async function createLambdaRole() {


if (NEPTUNE_IAM_AUTH) {

let action = [];
if (NEPTUNE_TYPE === NEPTUNE_DB) {
action = [
"neptune-db:DeleteDataViaQuery",
"neptune-db:connect",
"neptune-db:ReadDataViaQuery",
"neptune-db:WriteDataViaQuery"
];
} else {
action = ["neptune-graph:*"]
}

// Create Neptune query policy
startSpinner('Creating policy for Neptune queries', true);
let policyName = NAME+"NeptuneQueryPolicy";
Expand All @@ -352,7 +339,12 @@ async function createLambdaRole() {
Statement: [
{
Effect: "Allow",
Action: action,
Action: [
NEPTUNE_TYPE + ':connect',
NEPTUNE_TYPE + ':DeleteDataViaQuery',
NEPTUNE_TYPE + ':ReadDataViaQuery',
NEPTUNE_TYPE + ':WriteDataViaQuery'
],
Resource: NEPTUNE_IAM_POLICY_RESOURCE
},
],
Expand Down
49 changes: 29 additions & 20 deletions src/test/util.test.js
Original file line number Diff line number Diff line change
@@ -1,37 +1,46 @@
import {parseNeptuneDomainFromEndpoint, parseNeptuneDomainFromHost, parseNeptuneGraphName} from '../util.js';
import {parseNeptuneDomainFromHost, parseNeptuneEndpoint} from '../util.js';

test('parse domain from neptune cluster host', () => {
expect(parseNeptuneDomainFromHost('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com')).toBe('neptune.amazonaws.com');
expect(parseNeptuneDomainFromHost('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com'))
.toBe('neptune.amazonaws.com');
});

test('parse domain from neptune analytics host', () => {
expect(parseNeptuneDomainFromHost('g-abcdef.us-west-2.neptune-graph.amazonaws.com')).toBe('neptune-graph.amazonaws.com');
expect(parseNeptuneDomainFromHost('g-abcdef.us-west-2.neptune-graph.amazonaws.com'))
.toBe('neptune-graph.amazonaws.com');
});

test('parse domain from host without enough parts throws error', () => {
expect(() => parseNeptuneDomainFromHost('invalid.com')).toThrow('Cannot parse neptune host invalid.com because it has 2 part(s) delimited by . but expected at least 5');
expect(() => parseNeptuneDomainFromHost('invalid.com'))
.toThrow('Cannot parse neptune host invalid.com because it has 2 part(s) delimited by . but expected at least 5');
});

test('parse domain from neptune cluster endpoint', () => {
expect(parseNeptuneDomainFromEndpoint('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com:8182')).toBe('neptune.amazonaws.com');
test('parse neptune db endpoint', () => {
let neptuneInfo = parseNeptuneEndpoint('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com:8182');
expect(neptuneInfo).toHaveProperty('port', '8182');
expect(neptuneInfo).toHaveProperty('host', 'db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com');
expect(neptuneInfo).toHaveProperty('domain', 'neptune.amazonaws.com');
expect(neptuneInfo).toHaveProperty('region', 'us-west-2');
expect(neptuneInfo).toHaveProperty('graphName', 'db-neptune-abc-def');
expect(neptuneInfo).toHaveProperty('neptuneType', 'neptune-db');
});

test('parse domain from neptune analytics endpoint', () => {
expect(parseNeptuneDomainFromEndpoint('g-abcdef.us-west-2.neptune-graph.amazonaws.com:8182')).toBe('neptune-graph.amazonaws.com');
test('parse neptune analytics endpoint', () => {
let neptuneInfo = parseNeptuneEndpoint('g-abcdef.us-east-1.neptune-graph.amazonaws.com:8183');
expect(neptuneInfo).toHaveProperty('port', '8183');
expect(neptuneInfo).toHaveProperty('host', 'g-abcdef.us-east-1.neptune-graph.amazonaws.com');
expect(neptuneInfo).toHaveProperty('domain', 'neptune-graph.amazonaws.com');
expect(neptuneInfo).toHaveProperty('region', 'us-east-1');
expect(neptuneInfo).toHaveProperty('graphName', 'g-abcdef');
expect(neptuneInfo).toHaveProperty('neptuneType', 'neptune-graph');
});

test('parse domain from endpoint without enough parts throws error', () => {
expect(() => parseNeptuneDomainFromEndpoint('g-abcdef.us-west-2.neptune-graph.amazonaws.com')).toThrow('Cannot parse domain from neptune endpoint g-abcdef.us-west-2.neptune-graph.amazonaws.com because it has 1 part(s) delimited by : but expected 2');
test('parse neptune endpoint without port throws error', () => {
expect(() => parseNeptuneEndpoint('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com'))
.toThrow('Cannot parse neptune endpoint db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com because it is not in expected format of host:port');
});

test('parse name from neptune cluster host', () => {
expect(parseNeptuneGraphName('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com')).toBe('db-neptune-abc-def');
});

test('parse name from neptune analytics host', () => {
expect(parseNeptuneGraphName('g-abcdef.us-west-2.neptune-graph.amazonaws.com')).toBe('g-abcdef');
});

test('parse name from host without enough parts throws error', () => {
expect(() => parseNeptuneGraphName('invalid.com')).toThrow('Cannot parse neptune host invalid.com because it has 2 part(s) delimited by . but expected at least 5');
test('parse neptune endpoint not enough parts in domain throws error', () => {
expect(() => parseNeptuneEndpoint('invalid.com:1234'))
.toThrow('Cannot parse neptune host invalid.com because it has 2 part(s) delimited by . but expected at least 5');
});
Loading

0 comments on commit 5f41c81

Please sign in to comment.