Skip to content

Commit

Permalink
Merge pull request #34 from Bit-Quill/achild/fixCdk
Browse files Browse the repository at this point in the history
Fix CDK deployment
  • Loading branch information
Cole-Greer authored Oct 17, 2024
2 parents 909ed51 + 5f41c81 commit 8800cba
Show file tree
Hide file tree
Showing 9 changed files with 178 additions and 138 deletions.
4 changes: 4 additions & 0 deletions src/CDKPipelineApp.js
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,10 @@ async function createAWSpipelineCDK({
CDKFile = CDKFile.replace( "const NEPTUNE_DB_NAME = '';", `const NEPTUNE_DB_NAME = '${NEPTUNE_DB_NAME}';` );
CDKFile = CDKFile.replace( "const NEPTUNE_TYPE = '';", `const NEPTUNE_TYPE = '${NEPTUNE_TYPE}';` );
CDKFile = CDKFile.replace( "const NEPTUNE_DBSubnetGroup = null;", `const NEPTUNE_DBSubnetGroup = '${NEPTUNE_DBSubnetGroup}';` );
if (neptuneClusterInfo) {
CDKFile = CDKFile.replace("const NEPTUNE_DBSubnetIds = null;", `const NEPTUNE_DBSubnetIds = '${neptuneClusterInfo.dbSubnetIds}';`);
CDKFile = CDKFile.replace("const NEPTUNE_VpcSecurityGroupId = null;",`const NEPTUNE_VpcSecurityGroupId = '${neptuneClusterInfo.vpcSecurityGroupId}';`);
}
CDKFile = CDKFile.replace( "const NEPTUNE_IAM_AUTH = false;", `const NEPTUNE_IAM_AUTH = ${isNeptuneIAMAuth};` );
CDKFile = CDKFile.replace( "const NEPTUNE_IAM_POLICY_RESOURCE = '*';", `const NEPTUNE_IAM_POLICY_RESOURCE = '${NEPTUNE_IAM_POLICY_RESOURCE}';` );

Expand Down
19 changes: 10 additions & 9 deletions src/NeptuneSchema.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import { aws4Interceptor } from "aws4-axios";
import { fromNodeProviderChain } from "@aws-sdk/credential-providers";
import { NeptunedataClient, ExecuteOpenCypherQueryCommand } from "@aws-sdk/client-neptunedata";
import { loggerDebug, loggerError, loggerInfo, yellow } from "./logger.js";
import { parseNeptuneDomainFromHost, parseNeptuneGraphName } from "./util.js";
import { ExecuteQueryCommand, GetGraphSummaryCommand, NeptuneGraphClient } from "@aws-sdk/client-neptune-graph";

const NEPTUNE_DB = 'neptune-db';
Expand All @@ -25,7 +24,8 @@ const HTTP_LANGUAGE = 'openCypher';
const NEPTUNE_GRAPH_LANGUAGE = 'OPEN_CYPHER';
let HOST = '';
let PORT = 8182;
let REGION = ''
let REGION = '';
let DOMAIN = '';
let SAMPLE = 5000;
let NEPTUNE_TYPE = NEPTUNE_DB;
let NAME = '';
Expand Down Expand Up @@ -331,12 +331,13 @@ async function getEdgesDirectionsCardinality() {
}


function setGetNeptuneSchemaParameters(host, port, region, neptuneType) {
HOST = host;
PORT = port;
REGION = region;
NEPTUNE_TYPE = neptuneType;
NAME = parseNeptuneGraphName(host);
function setGetNeptuneSchemaParameters(neptuneInfo) {
HOST = neptuneInfo.host;
PORT = neptuneInfo.port;
REGION = neptuneInfo.region;
NEPTUNE_TYPE = neptuneInfo.neptuneType;
NAME = neptuneInfo.graphName;
DOMAIN = neptuneInfo.domain;
}

function getNeptunedataClient() {
Expand All @@ -354,7 +355,7 @@ function getNeptuneGraphClient() {
loggerInfo('Instantiating NeptuneGraphClient')
neptuneGraphClient = new NeptuneGraphClient({
port: PORT,
host: parseNeptuneDomainFromHost(HOST),
host: DOMAIN,
region: REGION,
protocol: NEPTUNE_GRAPH_PROTOCOL,
});
Expand Down
107 changes: 53 additions & 54 deletions src/main.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ let spinner = null;
// find global installation dir
import path from 'path';
import { fileURLToPath } from 'url';
import { parseNeptuneDomainFromEndpoint } from "./util.js";
import { parseNeptuneEndpoint } from "./util.js";
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

Expand Down Expand Up @@ -63,7 +63,7 @@ let createUpdatePipelineNeptuneDatabaseName = '';
let removePipelineName = '';
let inputCDKpipeline = false;
let inputCDKpipelineName = '';
let inputCDKpipelineEnpoint = '';
let inputCDKpipelineEndpoint = '';
let inputCDKpipelineFile = '';
let inputCDKpipelineRegion = '';
let inputCDKpipelineDatabaseName = '';
Expand Down Expand Up @@ -212,10 +212,14 @@ function processArgs() {
inputCDKpipeline = true;
break;
case '-ce':
// support miss-spelled option for backwards compatibility - could be removed for next major release
case '--output-aws-pipeline-cdk-neptume-endpoint':
case '--output-aws-pipeline-cdk-neptune-endpoint':
inputCDKpipelineEnpoint = array[index + 1];
inputCDKpipelineEndpoint = array[index + 1];
break;
case '-cd':
// support miss-spelled option for backwards compatibility - could be removed for next major release
case '--output-aws-pipeline-cdk-neptume-database-name':
case '--output-aws-pipeline-cdk-neptune-database-name':
inputCDKpipelineDatabaseName = array[index + 1];
break;
Expand Down Expand Up @@ -285,37 +289,31 @@ async function main() {
}
}


let neptuneInfo;
// Check if any of the Neptune endpoints are a neptune analytic endpoint and if so, set the neptuneType and IAM to required
const nonEmptyEndpoints = [inputGraphDBSchemaNeptuneEndpoint, createUpdatePipelineEndpoint, inputCDKpipelineEnpoint].filter(endpoint => endpoint !== '');
const isNeptuneAnalyticsGraph = nonEmptyEndpoints.length > 0 && parseNeptuneDomainFromEndpoint(nonEmptyEndpoints[0]).includes(NEPTUNE_GRAPH);
if (isNeptuneAnalyticsGraph) {
neptuneType = NEPTUNE_GRAPH;
// neptune analytics requires IAM
loggerInfo("Detected neptune-graph from input endpoint - setting IAM auth to true as it is required for neptune analytics")
isNeptuneIAMAuth = true;
// only one of these endpoints are expected to be non-empty at the same time
const nonEmptyEndpoints = [inputGraphDBSchemaNeptuneEndpoint, createUpdatePipelineEndpoint, inputCDKpipelineEndpoint].filter(endpoint => endpoint !== '');
if (nonEmptyEndpoints.length > 0) {
neptuneInfo = parseNeptuneEndpoint(nonEmptyEndpoints[0]);
neptuneType = neptuneInfo.neptuneType;
if (neptuneType === NEPTUNE_GRAPH) {
// neptune analytics requires IAM
loggerInfo("Detected neptune-graph from input endpoint - setting IAM auth to true as it is required for neptune analytics")
isNeptuneIAMAuth = true;
}
}

// Get Neptune schema from endpoint
if (inputGraphDBSchemaNeptuneEndpoint != '' && inputGraphDBSchema == '' && inputGraphDBSchemaFile == '') {
let endpointParts = inputGraphDBSchemaNeptuneEndpoint.split(':');
if (endpointParts.length != 2) {
loggerError('Neptune endpoint must be in the form of host:port');
process.exit(1);
if (!neptuneInfo) {
neptuneInfo = parseNeptuneEndpoint(inputGraphDBSchemaNeptuneEndpoint);
}
let neptuneHost = endpointParts[0];
let neptunePort = endpointParts[1];

let neptuneRegionParts = inputGraphDBSchemaNeptuneEndpoint.split('.');
let neptuneRegion = '';
if (neptuneType === NEPTUNE_DB)
neptuneRegion = neptuneRegionParts[2];
else
neptuneRegion = neptuneRegionParts[1];

loggerInfo('Retrieving Neptune schema');
loggerDebug('Getting Neptune schema from endpoint: ' + yellow(neptuneHost + ':' + neptunePort), {toConsole: true});
loggerDebug('Getting Neptune schema from endpoint: ' + yellow(inputGraphDBSchemaNeptuneEndpoint), {toConsole: true});

setGetNeptuneSchemaParameters(neptuneHost, neptunePort, neptuneRegion, neptuneType);
setGetNeptuneSchemaParameters(neptuneInfo);
let startTime = performance.now();
inputGraphDBSchema = await getNeptuneSchema();
let endTime = performance.now();
Expand Down Expand Up @@ -372,15 +370,11 @@ async function main() {
process.exit(1);
}
if (createUpdatePipelineEndpoint != '') {
let parts = createUpdatePipelineEndpoint.split('.');
createUpdatePipelineNeptuneDatabaseName = parts[0];

let parsedRegion;
if (neptuneType === NEPTUNE_DB) {
parsedRegion = parts[2];
} else {
parsedRegion = parts[1];
if (!neptuneInfo) {
neptuneInfo = parseNeptuneEndpoint(createUpdatePipelineEndpoint);
}
createUpdatePipelineNeptuneDatabaseName = neptuneInfo.graphName;
const parsedRegion = neptuneInfo.region;

if (createUpdatePipelineRegion !== parsedRegion) {
if (createUpdatePipelineRegion !== '') {
Expand All @@ -399,27 +393,37 @@ async function main() {
// CDK
if (inputCDKpipeline) {
if (!inputGraphDBSchemaNeptuneEndpoint == '') {
inputCDKpipelineEnpoint = inputGraphDBSchemaNeptuneEndpoint;
inputCDKpipelineEndpoint = inputGraphDBSchemaNeptuneEndpoint;
}
if (inputCDKpipelineEnpoint == '' &&
if (inputCDKpipelineEndpoint == '' &&
inputCDKpipelineRegion == '' && inputCDKpipelineDatabaseName == '') {
loggerError('AWS CDK: is required a Neptune endpoint, or a Neptune database name and region.');
process.exit(1);
}
if (inputCDKpipelineEnpoint == '' &&
if (inputCDKpipelineEndpoint == '' &&
!inputCDKpipelineRegion == '' && inputCDKpipelineDatabaseName == '') {
loggerError('AWS CDK: a Neptune database name is required.');
process.exit(1);
}
if (inputCDKpipelineEnpoint == '' &&
if (inputCDKpipelineEndpoint == '' &&
inputCDKpipelineRegion == '' && !inputCDKpipelineDatabaseName == '') {
loggerError('AWS CDK: a Neptune database region is required.');
process.exit(1);
}
if (inputCDKpipelineEnpoint != '') {
let parts = inputCDKpipelineEnpoint.split('.');
inputCDKpipelineDatabaseName = parts[0];
inputCDKpipelineRegion = parts[2];
if (inputCDKpipelineEndpoint != '') {
if (!neptuneInfo) {
neptuneInfo = parseNeptuneEndpoint(inputCDKpipelineEndpoint);
}
inputCDKpipelineDatabaseName = neptuneInfo.graphName;
const parsedRegion = neptuneInfo.region;
if (inputCDKpipelineRegion !== parsedRegion) {
if (inputCDKpipelineRegion !== '') {
loggerInfo('Switching CDK region from ' + inputCDKpipelineRegion + ' to region parsed from endpoint: ' + parsedRegion);
} else {
loggerInfo('Region parsed from CDK endpoint: ' + parsedRegion);
}
inputCDKpipelineRegion = parsedRegion;
}
}
if (inputCDKpipelineName == '') {
inputCDKpipelineName = inputCDKpipelineDatabaseName;
Expand Down Expand Up @@ -576,13 +580,11 @@ async function main() {
// Create Update AWS Pipeline
if (createUpdatePipeline) {
try {
let endpointParts = createUpdatePipelineEndpoint.split(':');
if (endpointParts.length < 2) {
loggerError('Neptune endpoint must be in the form of host:port');
process.exit(1);
if (!neptuneInfo) {
neptuneInfo = parseNeptuneEndpoint(createUpdatePipelineEndpoint);
}
let neptuneHost = endpointParts[0];
let neptunePort = endpointParts[1];
let neptuneHost = neptuneInfo.host;
let neptunePort = neptuneInfo.port;

await createUpdateAWSpipeline( createUpdatePipelineName,
createUpdatePipelineNeptuneDatabaseName,
Expand All @@ -608,14 +610,11 @@ async function main() {
try {
loggerInfo('Creating CDK File', {toConsole: true});

let endpointParts = inputCDKpipelineEnpoint.split(':');
if (endpointParts.length < 2) {
loggerError('Neptune endpoint must be in the form of host:port');
process.exit(1);
if (!neptuneInfo) {
neptuneInfo = parseNeptuneEndpoint(inputCDKpipelineEndpoint);
}
let neptuneHost = endpointParts[0];
let neptunePort = endpointParts[1];

let neptuneHost = neptuneInfo.host;
let neptunePort = neptuneInfo.port;

if (inputCDKpipelineFile == '') {
inputCDKpipelineFile = `${outputFolderPath}/${inputCDKpipelineName}-cdk.js`;
Expand Down
20 changes: 6 additions & 14 deletions src/pipelineResources.js
Original file line number Diff line number Diff line change
Expand Up @@ -330,19 +330,6 @@ async function createLambdaRole() {


if (NEPTUNE_IAM_AUTH) {

let action = [];
if (NEPTUNE_TYPE === NEPTUNE_DB) {
action = [
"neptune-db:DeleteDataViaQuery",
"neptune-db:connect",
"neptune-db:ReadDataViaQuery",
"neptune-db:WriteDataViaQuery"
];
} else {
action = ["neptune-graph:*"]
}

// Create Neptune query policy
startSpinner('Creating policy for Neptune queries', true);
let policyName = NAME+"NeptuneQueryPolicy";
Expand All @@ -352,7 +339,12 @@ async function createLambdaRole() {
Statement: [
{
Effect: "Allow",
Action: action,
Action: [
NEPTUNE_TYPE + ':connect',
NEPTUNE_TYPE + ':DeleteDataViaQuery',
NEPTUNE_TYPE + ':ReadDataViaQuery',
NEPTUNE_TYPE + ':WriteDataViaQuery'
],
Resource: NEPTUNE_IAM_POLICY_RESOURCE
},
],
Expand Down
49 changes: 29 additions & 20 deletions src/test/util.test.js
Original file line number Diff line number Diff line change
@@ -1,37 +1,46 @@
import {parseNeptuneDomainFromEndpoint, parseNeptuneDomainFromHost, parseNeptuneGraphName} from '../util.js';
import {parseNeptuneDomainFromHost, parseNeptuneEndpoint} from '../util.js';

test('parse domain from neptune cluster host', () => {
expect(parseNeptuneDomainFromHost('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com')).toBe('neptune.amazonaws.com');
expect(parseNeptuneDomainFromHost('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com'))
.toBe('neptune.amazonaws.com');
});

test('parse domain from neptune analytics host', () => {
expect(parseNeptuneDomainFromHost('g-abcdef.us-west-2.neptune-graph.amazonaws.com')).toBe('neptune-graph.amazonaws.com');
expect(parseNeptuneDomainFromHost('g-abcdef.us-west-2.neptune-graph.amazonaws.com'))
.toBe('neptune-graph.amazonaws.com');
});

test('parse domain from host without enough parts throws error', () => {
expect(() => parseNeptuneDomainFromHost('invalid.com')).toThrow('Cannot parse neptune host invalid.com because it has 2 part(s) delimited by . but expected at least 5');
expect(() => parseNeptuneDomainFromHost('invalid.com'))
.toThrow('Cannot parse neptune host invalid.com because it has 2 part(s) delimited by . but expected at least 5');
});

test('parse domain from neptune cluster endpoint', () => {
expect(parseNeptuneDomainFromEndpoint('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com:8182')).toBe('neptune.amazonaws.com');
test('parse neptune db endpoint', () => {
let neptuneInfo = parseNeptuneEndpoint('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com:8182');
expect(neptuneInfo).toHaveProperty('port', '8182');
expect(neptuneInfo).toHaveProperty('host', 'db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com');
expect(neptuneInfo).toHaveProperty('domain', 'neptune.amazonaws.com');
expect(neptuneInfo).toHaveProperty('region', 'us-west-2');
expect(neptuneInfo).toHaveProperty('graphName', 'db-neptune-abc-def');
expect(neptuneInfo).toHaveProperty('neptuneType', 'neptune-db');
});

test('parse domain from neptune analytics endpoint', () => {
expect(parseNeptuneDomainFromEndpoint('g-abcdef.us-west-2.neptune-graph.amazonaws.com:8182')).toBe('neptune-graph.amazonaws.com');
test('parse neptune analytics endpoint', () => {
let neptuneInfo = parseNeptuneEndpoint('g-abcdef.us-east-1.neptune-graph.amazonaws.com:8183');
expect(neptuneInfo).toHaveProperty('port', '8183');
expect(neptuneInfo).toHaveProperty('host', 'g-abcdef.us-east-1.neptune-graph.amazonaws.com');
expect(neptuneInfo).toHaveProperty('domain', 'neptune-graph.amazonaws.com');
expect(neptuneInfo).toHaveProperty('region', 'us-east-1');
expect(neptuneInfo).toHaveProperty('graphName', 'g-abcdef');
expect(neptuneInfo).toHaveProperty('neptuneType', 'neptune-graph');
});

test('parse domain from endpoint without enough parts throws error', () => {
expect(() => parseNeptuneDomainFromEndpoint('g-abcdef.us-west-2.neptune-graph.amazonaws.com')).toThrow('Cannot parse domain from neptune endpoint g-abcdef.us-west-2.neptune-graph.amazonaws.com because it has 1 part(s) delimited by : but expected 2');
test('parse neptune endpoint without port throws error', () => {
expect(() => parseNeptuneEndpoint('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com'))
.toThrow('Cannot parse neptune endpoint db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com because it is not in expected format of host:port');
});

test('parse name from neptune cluster host', () => {
expect(parseNeptuneGraphName('db-neptune-abc-def.cluster-xyz.us-west-2.neptune.amazonaws.com')).toBe('db-neptune-abc-def');
});

test('parse name from neptune analytics host', () => {
expect(parseNeptuneGraphName('g-abcdef.us-west-2.neptune-graph.amazonaws.com')).toBe('g-abcdef');
});

test('parse name from host without enough parts throws error', () => {
expect(() => parseNeptuneGraphName('invalid.com')).toThrow('Cannot parse neptune host invalid.com because it has 2 part(s) delimited by . but expected at least 5');
test('parse neptune endpoint not enough parts in domain throws error', () => {
expect(() => parseNeptuneEndpoint('invalid.com:1234'))
.toThrow('Cannot parse neptune host invalid.com because it has 2 part(s) delimited by . but expected at least 5');
});
Loading

0 comments on commit 8800cba

Please sign in to comment.