From 35789e1f3d2289d39868782c90e55315d7fce50d Mon Sep 17 00:00:00 2001 From: chulanovskyi Date: Tue, 22 Oct 2024 16:02:07 +0300 Subject: [PATCH] feat: use another way of fetching the existing labels --- reverse_engineering/api.js | 37 +++++++++++++++--------------- reverse_engineering/neo4jHelper.js | 34 ++++++++++++++++++--------- 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js index f078fe5..bbcca90 100644 --- a/reverse_engineering/api.js +++ b/reverse_engineering/api.js @@ -146,7 +146,7 @@ module.exports = { const isMultiDb = await neo4j.supportsMultiDb(); const dbVersion = await neo4j.getDbVersion(); - logger.log('info', `Database version: ${dbVersion}`); + logger.log('info', `Version: ${dbVersion}`, 'Database info'); const modelProps = { dbVersion, }; @@ -166,7 +166,7 @@ module.exports = { metaData.indexes = modelProps.dbVersion === '3.x' ? prepareIndexes3x(indexes) : prepareIndexes4x(indexes); - const countIndexes = (indexes && indexes.length) || 0; + const countIndexes = indexes?.length || 0; logger.progress({ message: 'Indexes retrieved successfully. Found ' + countIndexes + ' index(es)', containerName: dbName, @@ -184,7 +184,7 @@ module.exports = { .then(constraints => { metaData.constraints = prepareConstraints(constraints, modelProps.dbVersion); - const countConstraints = (constraints && constraints.length) || 0; + const countConstraints = constraints?.length || 0; logger.progress({ message: 'Constraints retrieved successfully ' + countConstraints + ' constraint(s)', containerName: dbName, @@ -276,7 +276,7 @@ module.exports = { logger.log('info', '', 'Reverse-engineering completed'); setTimeout(() => { - cb(err, packages.labels, modelProps, [].concat.apply([], packages.relationships)); + cb(err, packages.labels, modelProps, [].concat(...packages.relationships)); }, 1000); }, ); @@ -337,8 +337,7 @@ const getNodesData = (dbName, labels, isMultiDb, data, logger) => { .then(quantity => { const count = getSampleDocSize(quantity, data.recordSamplingSettings); logger(labelName, 'Found ' + count + ' nodes'); - - return neo4j.getNodes(labelName, count, dbName, isMultiDb); + return neo4j.getNodes({ label: labelName, limit: count, dbName, isMultiDb }); }) .then(documents => { logger(labelName, 'Data retrieved successfully'); @@ -388,14 +387,14 @@ const getRelationshipData = ( .getCountRelationshipsData(chain.start, chain.relationship, chain.end, dbName, isMultiDb) .then(quantity => { const count = getSampleDocSize(quantity, recordSamplingSettings); - return neo4j.getRelationshipData( - chain.start, - chain.relationship, - chain.end, + return neo4j.getRelationshipData({ + start: chain.start, + relationship: chain.relationship, + end: chain.end, count, dbName, isMultiDb, - ); + }); }) .then(rawDocuments => { const documents = deserializeData(rawDocuments); @@ -481,10 +480,10 @@ const getLabelPackage = ( }; const prepareIndexes3x = indexes => { - const hasProperties = /INDEX\s+ON\s+\:(.*)\((.*)\)/i; + const hasProperties = /INDEX\s+ON\s+:(.*)\((.*)\)/i; const map = {}; - indexes.forEach((index, i) => { + indexes.forEach(index => { if (!index.properties) { if (hasProperties.test(index.description)) { const parsedDescription = index.description.match(hasProperties); @@ -527,7 +526,7 @@ const prepareIndexes4x = indexes => { provider: index.provider || index.indexProvider, }; - index.labelsOrTypes.forEach((label, i) => { + index.labelsOrTypes.forEach(label => { if (!map[label]) { map[label] = [index_obj]; } else { @@ -588,10 +587,10 @@ const prepareConstraints5x = constraints => { }; const prepareConstraints4x = constraints => { - const isUnique = /^constraint\s+on\s+\([\s\S]+\:([\S\s]+)\s*\)\s+assert\s+[\s\S]+\.([\s\S]+)\s*\)\s+IS\s+UNIQUE/i; + const isUnique = /^constraint\s+on\s+\([\s\S]+:([\S\s]+)\s*\)\s+assert\s+[\s\S]+\.([\s\S]+)\s*\)\s+IS\s+UNIQUE/i; const isNodeKey = - /^constraint\s+on\s+\([\s\S]+\:\s*([\S\s]+)\s*\)\s+assert\s+(?:\(\s*([\s\S]+)\s*\)|[\s\S]+\.\s*([\S\s]+)\s*)\s+IS\s+NODE\s+KEY/i; - const isExists = /^constraint\s+on\s+\([\s\S]+\:([\s\S]+)\s*\)\s+assert\s+exists\([\s\S]+\.([\s\S]+)\s*\)/i; + /^constraint\s+on\s+\([\s\S]+:\s*([\S\s]+)\s*\)\s+assert\s+(?:\(\s*([\s\S]+)\s*\)|[\s\S]+\.\s*([\S\s]+)\s*)\s+IS\s+NODE\s+KEY/i; + const isExists = /^constraint\s+on\s+\([\s\S]+:([\s\S]+)\s*\)\s+assert\s+exists\([\s\S]+\.([\s\S]+)\s*\)/i; let result = { nodeAndRelationship: {} }; const addToResult = (result, name, label, key, type, keyName = 'key') => { const labelName = label.trim(); @@ -819,7 +818,7 @@ const getSnippetPropertiesByName = name => { const properties = {}; snippet.properties.forEach(fieldSchema => { - properties[fieldSchema.name] = Object.assign({}, fieldSchema); + properties[fieldSchema.name] = { ...fieldSchema }; delete properties[fieldSchema.name].name; }); @@ -829,7 +828,7 @@ const getSnippetPropertiesByName = name => { const separateConstraintsByType = (constraints = []) => { return constraints.reduce( (result, constraint) => { - constraint = Object.assign({}, constraint); + constraint = { ...constraint }; const type = constraint.type; delete constraint.type; result[type].push(constraint); diff --git a/reverse_engineering/neo4jHelper.js b/reverse_engineering/neo4jHelper.js index b4db0b2..a1d586d 100644 --- a/reverse_engineering/neo4jHelper.js +++ b/reverse_engineering/neo4jHelper.js @@ -6,9 +6,10 @@ let driver; let isSshTunnel = false; const EXECUTE_TIME_OUT_CODE = 'EXECUTE_TIME_OUT'; -let timeout; +const defaultTimeout = 300000; +let timeout = defaultTimeout; -const setTimeOut = data => (timeout = data?.queryRequestTimeout || 300000); +const setTimeOut = data => (timeout = data?.queryRequestTimeout || defaultTimeout); const connectToInstance = (info, checkConnection) => { return checkConnection(info.host, info.port).then( @@ -84,7 +85,7 @@ const execute = async (command, database = undefined, isMultiDb = false) => { database = undefined; } const executeTimeout = new Promise((_, reject) => - setTimeout(() => reject(getExecuteTimeoutError(timeout)), timeout), + setTimeout(() => reject(getExecuteTimeoutError({ timeout, command })), timeout), ); let result = []; let db; @@ -116,8 +117,8 @@ const execute = async (command, database = undefined, isMultiDb = false) => { } }; -const getExecuteTimeoutError = timeout => { - const error = new Error(`execute timeout: ${timeout}ms exceeded`); +const getExecuteTimeoutError = ({ timeout, command }) => { + const error = new Error(`Execute timeout: ${timeout}ms exceeded. Query:\n${command}`); error.code = EXECUTE_TIME_OUT_CODE; return error; }; @@ -137,16 +138,27 @@ const castInteger = properties => { }; const getLabels = async ({ database, isMultiDb, logger }) => { + const step = 'Retrieving labels information'; try { + const labelsMeta = new Map(); + + const allLabels = await execute('CALL db.labels()', database, isMultiDb); + allLabels.forEach(({ label }) => labelsMeta.set(label, 0)); + const recordsCounter = await execute( - 'MATCH (n) RETURN DISTINCT COUNT(labels(n)) as labelsCount', + 'MATCH (n) RETURN DISTINCT labels(n) as label, COUNT(*) as total ORDER BY total DESC', database, isMultiDb, ); - logger.log('info', `Found ${_.head(recordsCounter).labelsCount} labels`, 'Retrieving labels information'); + recordsCounter.forEach(record => { + const label = _.head(record.label); + labelsMeta.set(label, record.total); + }); + + const labelsToLog = [...labelsMeta.entries()].map(([label, total]) => `${label} (${total})`).join('\n'); + logger.log('info', `Found ${labelsMeta.size} unique labels:\n${labelsToLog}`, step); - const records = await execute('MATCH (n) RETURN DISTINCT labels(n) as label', database, isMultiDb); - return _.flatMap(records, record => record.label); + return [...labelsMeta.keys()]; } catch (error) { const errorStep = error.step || 'Error of retrieving labels'; logger.log('error', error, errorStep); @@ -223,13 +235,13 @@ const getActiveMultiDbsNames = async () => { .map(({ name }) => name); }; -const getNodes = (label, limit = 100, dbName, isMultiDb) => { +const getNodes = ({ label, limit = 100, dbName, isMultiDb }) => { return execute(`MATCH (row:\`${label}\`) RETURN row LIMIT ${limit}`, dbName, isMultiDb).then(result => { return result.map(record => castInteger(record.row.properties)); }); }; -const getRelationshipData = (start, relationship, end, limit = 100, dbName, isMultiDb) => { +const getRelationshipData = ({ start, relationship, end, limit = 100, dbName, isMultiDb }) => { return execute( `MATCH (:\`${start}\`)-[row:\`${relationship}\`]-(:\`${end}\`) RETURN row LIMIT ${limit}`, dbName,