Merge pull request #115 from dannleed/fix/HCK-3553-standardization-of…

…-sampling-restrictions HCK-3553: standardize record sampling limitation
hackolade · Jul 18, 2023 · 7ee1ca9 · 7ee1ca9
2 parents 543ff96 + 1db40ae
commit 7ee1ca9
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 40 deletions.
diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js
@@ -126,7 +126,7 @@ module.exports = {
 		const tables = data.collectionData.collections;
 		const keyspacesNames = data.collectionData.dataBaseNames;
 		const includeEmptyCollection = data.includeEmptyCollection;
-		const recordSamplingSettings = { ...data.recordSamplingSettings, isTerminal: data.isTerminal };
+		const recordSamplingSettings = data.recordSamplingSettings;
 
 		async.map(keyspacesNames, (keyspaceName, keyspaceCallback) => {
 			const entityNames = cassandra.splitEntityNames(tables[keyspaceName]);

diff --git a/reverse_engineering/cassandraHelper.js b/reverse_engineering/cassandraHelper.js
@@ -528,54 +528,49 @@ module.exports = (_) => {
 	};
 
 	const scanRecords = (keyspace, table, recordSamplingSettings, logger) => {
-        return getSizeOfRows(keyspace, table, recordSamplingSettings).then(
-            (size) =>
-                new Promise((resolve, reject) => {
+		return getSizeOfRows(keyspace, table, recordSamplingSettings).then(
+			size =>
+				new Promise((resolve, reject) => {
 					let rows = [];
-					
+
 					logger.log('info', { table: `${keyspace}.${table}`, limit: size }, 'Scan records');
 
-                    if (!size) {
-                        return resolve(rows);
+					if (!size) {
+						return resolve(rows);
 					}
 
-                    const options = { prepare: true, autoPage: true, retry: new CassandraRetryPolicy(logger) };
-                    const selQuery = `SELECT * FROM "${keyspace}"."${table}" LIMIT ${size}`;
-
-                    state.client.eachRow(
-                        selQuery,
-                        [],
-                        options,
-                        function (n, row) {
-                            rows.push(row);
-                        },
-                        (err, rs) => {
-                            return err ? reject(err) : resolve(rows);
-                        }
-                    );
-                })
-        );
-    };
+					const options = { prepare: true, autoPage: true, retry: new CassandraRetryPolicy(logger) };
+					const selQuery = `SELECT * FROM "${keyspace}"."${table}" LIMIT ${size}`;
+
+					state.client.eachRow(
+						selQuery,
+						[],
+						options,
+						function (n, row) {
+							rows.push(row);
+						},
+						(err, rs) => {
+							return err ? reject(err) : resolve(rows);
+						},
+					);
+				}),
+		);
+	};
 
 	const getSizeOfRows = (keyspace, table, recordSamplingSettings) => {
-		if(recordSamplingSettings.active === 'absolute') {
-			return Promise.resolve(recordSamplingSettings.absolute.value)
+		if (recordSamplingSettings.active === 'absolute') {
+			return Promise.resolve(Number(recordSamplingSettings.absolute.value));
 		}
 
-		const defaultCount = 1000;
 		const countQueryLimit = getCountLimit(recordSamplingSettings);
 		const query = `SELECT COUNT(*) FROM "${keyspace}"."${table}" LIMIT ${countQueryLimit}`;
 
 		return execute(query).then(count => {
-			const rowsCount = _.get(count, 'rows[0].count.low', defaultCount);
-
-			if (!rowsCount) {
-				return 0;
-			}
+			const rowsCount = _.get(count, 'rows[0].count.low', recordSamplingSettings.maxValue);
 
-			return getSampleDocSize(rowsCount, recordSamplingSettings)
-		})
-	}
+			return getSampleDocSize(rowsCount, recordSamplingSettings);
+		});
+	};
 
 
 	const getEntityLevelData = (table, tableName, searchIndex) => {
@@ -1061,10 +1056,9 @@ module.exports = (_) => {
 
 
 	const getSampleDocSize = (count, recordSamplingSettings) => {
-		const per = recordSamplingSettings.relative.value;
-		return (recordSamplingSettings.active === 'absolute')
-			? recordSamplingSettings.absolute.value
-				: Math.round( count/100 * per);
+		const limit = Math.ceil((count * recordSamplingSettings.relative.value) / 100);
+
+		return Math.min(limit, recordSamplingSettings.maxValue);
 	};
 
 	const cleanOutComments = (script) => {
@@ -1154,7 +1148,7 @@ module.exports = (_) => {
 
 	const getCountLimit = (recordSamplingSettings) => {
         const per = recordSamplingSettings.relative.value;
-        const max = recordSamplingSettings.isTerminal ? 100000 : 10000;
+        const max = recordSamplingSettings.maxValue;
 
         return Math.round((max / per) * 100);
     };