diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js index 62c4b91..ba41c4d 100644 --- a/reverse_engineering/api.js +++ b/reverse_engineering/api.js @@ -126,7 +126,7 @@ module.exports = { const tables = data.collectionData.collections; const keyspacesNames = data.collectionData.dataBaseNames; const includeEmptyCollection = data.includeEmptyCollection; - const recordSamplingSettings = { ...data.recordSamplingSettings, isTerminal: data.isTerminal }; + const recordSamplingSettings = data.recordSamplingSettings; async.map(keyspacesNames, (keyspaceName, keyspaceCallback) => { const entityNames = cassandra.splitEntityNames(tables[keyspaceName]); diff --git a/reverse_engineering/cassandraHelper.js b/reverse_engineering/cassandraHelper.js index 19ca1e8..1e651b4 100644 --- a/reverse_engineering/cassandraHelper.js +++ b/reverse_engineering/cassandraHelper.js @@ -528,54 +528,49 @@ module.exports = (_) => { }; const scanRecords = (keyspace, table, recordSamplingSettings, logger) => { - return getSizeOfRows(keyspace, table, recordSamplingSettings).then( - (size) => - new Promise((resolve, reject) => { + return getSizeOfRows(keyspace, table, recordSamplingSettings).then( + size => + new Promise((resolve, reject) => { let rows = []; - + logger.log('info', { table: `${keyspace}.${table}`, limit: size }, 'Scan records'); - if (!size) { - return resolve(rows); + if (!size) { + return resolve(rows); } - const options = { prepare: true, autoPage: true, retry: new CassandraRetryPolicy(logger) }; - const selQuery = `SELECT * FROM "${keyspace}"."${table}" LIMIT ${size}`; - - state.client.eachRow( - selQuery, - [], - options, - function (n, row) { - rows.push(row); - }, - (err, rs) => { - return err ? reject(err) : resolve(rows); - } - ); - }) - ); - }; + const options = { prepare: true, autoPage: true, retry: new CassandraRetryPolicy(logger) }; + const selQuery = `SELECT * FROM "${keyspace}"."${table}" LIMIT ${size}`; + + state.client.eachRow( + selQuery, + [], + options, + function (n, row) { + rows.push(row); + }, + (err, rs) => { + return err ? reject(err) : resolve(rows); + }, + ); + }), + ); + }; const getSizeOfRows = (keyspace, table, recordSamplingSettings) => { - if(recordSamplingSettings.active === 'absolute') { - return Promise.resolve(recordSamplingSettings.absolute.value) + if (recordSamplingSettings.active === 'absolute') { + return Promise.resolve(Number(recordSamplingSettings.absolute.value)); } - const defaultCount = 1000; const countQueryLimit = getCountLimit(recordSamplingSettings); const query = `SELECT COUNT(*) FROM "${keyspace}"."${table}" LIMIT ${countQueryLimit}`; return execute(query).then(count => { - const rowsCount = _.get(count, 'rows[0].count.low', defaultCount); - - if (!rowsCount) { - return 0; - } + const rowsCount = _.get(count, 'rows[0].count.low', recordSamplingSettings.maxValue); - return getSampleDocSize(rowsCount, recordSamplingSettings) - }) - } + return getSampleDocSize(rowsCount, recordSamplingSettings); + }); + }; const getEntityLevelData = (table, tableName, searchIndex) => { @@ -1061,10 +1056,9 @@ module.exports = (_) => { const getSampleDocSize = (count, recordSamplingSettings) => { - const per = recordSamplingSettings.relative.value; - return (recordSamplingSettings.active === 'absolute') - ? recordSamplingSettings.absolute.value - : Math.round( count/100 * per); + const limit = Math.ceil((count * recordSamplingSettings.relative.value) / 100); + + return Math.min(limit, recordSamplingSettings.maxValue); }; const cleanOutComments = (script) => { @@ -1154,7 +1148,7 @@ module.exports = (_) => { const getCountLimit = (recordSamplingSettings) => { const per = recordSamplingSettings.relative.value; - const max = recordSamplingSettings.isTerminal ? 100000 : 10000; + const max = recordSamplingSettings.maxValue; return Math.round((max / per) * 100); };