Skip to content

Commit

Permalink
feat: use another way of fetching the existing labels
Browse files Browse the repository at this point in the history
  • Loading branch information
@chulanovskyi-bs
chulanovskyi-bs committed Oct 22, 2024
1 parent a97cac7 commit 35789e1
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 30 deletions.
37 changes: 18 additions & 19 deletions reverse_engineering/api.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ module.exports = {

const isMultiDb = await neo4j.supportsMultiDb();
const dbVersion = await neo4j.getDbVersion();
logger.log('info', `Database version: ${dbVersion}`);
logger.log('info', `Version: ${dbVersion}`, 'Database info');
const modelProps = {
dbVersion,
};
Expand All @@ -166,7 +166,7 @@ module.exports = {
metaData.indexes =
modelProps.dbVersion === '3.x' ? prepareIndexes3x(indexes) : prepareIndexes4x(indexes);

const countIndexes = (indexes && indexes.length) || 0;
const countIndexes = indexes?.length || 0;
logger.progress({
message: 'Indexes retrieved successfully. Found ' + countIndexes + ' index(es)',
containerName: dbName,
Expand All @@ -184,7 +184,7 @@ module.exports = {
.then(constraints => {
metaData.constraints = prepareConstraints(constraints, modelProps.dbVersion);

const countConstraints = (constraints && constraints.length) || 0;
const countConstraints = constraints?.length || 0;
logger.progress({
message: 'Constraints retrieved successfully ' + countConstraints + ' constraint(s)',
containerName: dbName,
Expand Down Expand Up @@ -276,7 +276,7 @@ module.exports = {
logger.log('info', '', 'Reverse-engineering completed');

setTimeout(() => {
cb(err, packages.labels, modelProps, [].concat.apply([], packages.relationships));
cb(err, packages.labels, modelProps, [].concat(...packages.relationships));
}, 1000);
},
);
Expand Down Expand Up @@ -337,8 +337,7 @@ const getNodesData = (dbName, labels, isMultiDb, data, logger) => {
.then(quantity => {
const count = getSampleDocSize(quantity, data.recordSamplingSettings);
logger(labelName, 'Found ' + count + ' nodes');

return neo4j.getNodes(labelName, count, dbName, isMultiDb);
return neo4j.getNodes({ label: labelName, limit: count, dbName, isMultiDb });
})
.then(documents => {
logger(labelName, 'Data retrieved successfully');
Expand Down Expand Up @@ -388,14 +387,14 @@ const getRelationshipData = (
.getCountRelationshipsData(chain.start, chain.relationship, chain.end, dbName, isMultiDb)
.then(quantity => {
const count = getSampleDocSize(quantity, recordSamplingSettings);
return neo4j.getRelationshipData(
chain.start,
chain.relationship,
chain.end,
return neo4j.getRelationshipData({
start: chain.start,
relationship: chain.relationship,
end: chain.end,
count,
dbName,
isMultiDb,
);
});
})
.then(rawDocuments => {
const documents = deserializeData(rawDocuments);
Expand Down Expand Up @@ -481,10 +480,10 @@ const getLabelPackage = (
};

const prepareIndexes3x = indexes => {
const hasProperties = /INDEX\s+ON\s+\:(.*)\((.*)\)/i;
const hasProperties = /INDEX\s+ON\s+:(.*)\((.*)\)/i;
const map = {};

indexes.forEach((index, i) => {
indexes.forEach(index => {
if (!index.properties) {
if (hasProperties.test(index.description)) {
const parsedDescription = index.description.match(hasProperties);
Expand Down Expand Up @@ -527,7 +526,7 @@ const prepareIndexes4x = indexes => {
provider: index.provider || index.indexProvider,
};

index.labelsOrTypes.forEach((label, i) => {
index.labelsOrTypes.forEach(label => {
if (!map[label]) {
map[label] = [index_obj];
} else {
Expand Down Expand Up @@ -588,10 +587,10 @@ const prepareConstraints5x = constraints => {
};

const prepareConstraints4x = constraints => {
const isUnique = /^constraint\s+on\s+\([\s\S]+\:([\S\s]+)\s*\)\s+assert\s+[\s\S]+\.([\s\S]+)\s*\)\s+IS\s+UNIQUE/i;
const isUnique = /^constraint\s+on\s+\([\s\S]+:([\S\s]+)\s*\)\s+assert\s+[\s\S]+\.([\s\S]+)\s*\)\s+IS\s+UNIQUE/i;
const isNodeKey =
/^constraint\s+on\s+\([\s\S]+\:\s*([\S\s]+)\s*\)\s+assert\s+(?:\(\s*([\s\S]+)\s*\)|[\s\S]+\.\s*([\S\s]+)\s*)\s+IS\s+NODE\s+KEY/i;
const isExists = /^constraint\s+on\s+\([\s\S]+\:([\s\S]+)\s*\)\s+assert\s+exists\([\s\S]+\.([\s\S]+)\s*\)/i;
/^constraint\s+on\s+\([\s\S]+:\s*([\S\s]+)\s*\)\s+assert\s+(?:\(\s*([\s\S]+)\s*\)|[\s\S]+\.\s*([\S\s]+)\s*)\s+IS\s+NODE\s+KEY/i;
const isExists = /^constraint\s+on\s+\([\s\S]+:([\s\S]+)\s*\)\s+assert\s+exists\([\s\S]+\.([\s\S]+)\s*\)/i;
let result = { nodeAndRelationship: {} };
const addToResult = (result, name, label, key, type, keyName = 'key') => {
const labelName = label.trim();
Expand Down Expand Up @@ -819,7 +818,7 @@ const getSnippetPropertiesByName = name => {
const properties = {};

snippet.properties.forEach(fieldSchema => {
properties[fieldSchema.name] = Object.assign({}, fieldSchema);
properties[fieldSchema.name] = { ...fieldSchema };
delete properties[fieldSchema.name].name;
});

Expand All @@ -829,7 +828,7 @@ const getSnippetPropertiesByName = name => {
const separateConstraintsByType = (constraints = []) => {
return constraints.reduce(
(result, constraint) => {
constraint = Object.assign({}, constraint);
constraint = { ...constraint };
const type = constraint.type;
delete constraint.type;
result[type].push(constraint);
Expand Down
34 changes: 23 additions & 11 deletions reverse_engineering/neo4jHelper.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@ let driver;
let isSshTunnel = false;

const EXECUTE_TIME_OUT_CODE = 'EXECUTE_TIME_OUT';
let timeout;
const defaultTimeout = 300000;
let timeout = defaultTimeout;

const setTimeOut = data => (timeout = data?.queryRequestTimeout || 300000);
const setTimeOut = data => (timeout = data?.queryRequestTimeout || defaultTimeout);

const connectToInstance = (info, checkConnection) => {
return checkConnection(info.host, info.port).then(
Expand Down Expand Up @@ -84,7 +85,7 @@ const execute = async (command, database = undefined, isMultiDb = false) => {
database = undefined;
}
const executeTimeout = new Promise((_, reject) =>
setTimeout(() => reject(getExecuteTimeoutError(timeout)), timeout),
setTimeout(() => reject(getExecuteTimeoutError({ timeout, command })), timeout),
);
let result = [];
let db;
Expand Down Expand Up @@ -116,8 +117,8 @@ const execute = async (command, database = undefined, isMultiDb = false) => {
}
};

const getExecuteTimeoutError = timeout => {
const error = new Error(`execute timeout: ${timeout}ms exceeded`);
const getExecuteTimeoutError = ({ timeout, command }) => {
const error = new Error(`Execute timeout: ${timeout}ms exceeded. Query:\n${command}`);
error.code = EXECUTE_TIME_OUT_CODE;
return error;
};
Expand All @@ -137,16 +138,27 @@ const castInteger = properties => {
};

const getLabels = async ({ database, isMultiDb, logger }) => {
const step = 'Retrieving labels information';
try {
const labelsMeta = new Map();

const allLabels = await execute('CALL db.labels()', database, isMultiDb);
allLabels.forEach(({ label }) => labelsMeta.set(label, 0));

const recordsCounter = await execute(
'MATCH (n) RETURN DISTINCT COUNT(labels(n)) as labelsCount',
'MATCH (n) RETURN DISTINCT labels(n) as label, COUNT(*) as total ORDER BY total DESC',
database,
isMultiDb,
);
logger.log('info', `Found ${_.head(recordsCounter).labelsCount} labels`, 'Retrieving labels information');
recordsCounter.forEach(record => {
const label = _.head(record.label);
labelsMeta.set(label, record.total);
});

const labelsToLog = [...labelsMeta.entries()].map(([label, total]) => `${label} (${total})`).join('\n');
logger.log('info', `Found ${labelsMeta.size} unique labels:\n${labelsToLog}`, step);

const records = await execute('MATCH (n) RETURN DISTINCT labels(n) as label', database, isMultiDb);
return _.flatMap(records, record => record.label);
return [...labelsMeta.keys()];
} catch (error) {
const errorStep = error.step || 'Error of retrieving labels';
logger.log('error', error, errorStep);
Expand Down Expand Up @@ -223,13 +235,13 @@ const getActiveMultiDbsNames = async () => {
.map(({ name }) => name);
};

const getNodes = (label, limit = 100, dbName, isMultiDb) => {
const getNodes = ({ label, limit = 100, dbName, isMultiDb }) => {
return execute(`MATCH (row:\`${label}\`) RETURN row LIMIT ${limit}`, dbName, isMultiDb).then(result => {
return result.map(record => castInteger(record.row.properties));
});
};

const getRelationshipData = (start, relationship, end, limit = 100, dbName, isMultiDb) => {
const getRelationshipData = ({ start, relationship, end, limit = 100, dbName, isMultiDb }) => {
return execute(
`MATCH (:\`${start}\`)-[row:\`${relationship}\`]-(:\`${end}\`) RETURN row LIMIT ${limit}`,
dbName,
Expand Down

0 comments on commit 35789e1

Please sign in to comment.