Merge pull request #121 from symbol/fix/gh_120-node-discovery-fix

fix: node discovery extended, empty host info bug fixed, logging enhanced
symbol · Jan 6, 2022 · 214b8a8 · 214b8a8
2 parents d78a50c + 1fb1330
commit 214b8a8
Show file tree

Hide file tree

Showing 11 changed files with 208 additions and 75 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -21,6 +21,7 @@
     },
     "devDependencies": {
         "@openapitools/openapi-generator-cli": "^2.4.15",
+        "@types/humanize-duration": "^3.27.0",
         "@types/mongoose": "^5.7.14",
         "@types/node": "^14.14.10",
         "@types/winston": "^2.4.4",
@@ -49,6 +50,7 @@
         "cors": "^2.8.5",
         "dotenv": "^8.2.0",
         "express": "^4.17.1",
+        "humanize-duration": "^3.27.1",
         "module-alias": "^2.2.2",
         "mongoose": "^5.9.16",
         "symbol-sdk": "^1.0.3",

diff --git a/src/config/config.json b/src/config/config.json
@@ -17,6 +17,8 @@
 	"PEER_NODE_PORT": 7900,
 	"REQUEST_TIMEOUT": 5000,
 	"NUMBER_OF_NODE_REQUEST_CHUNK": 10,
+	"NODE_PEERS_REQUEST_CHUNK_SIZE": 50,
+	"CHAIN_HEIGHT_REQUEST_CHUNK_SIZE": 10,
 	"PREFERRED_NODES": ["*.symboldev.network"],
 	"MIN_PARTNER_NODE_VERSION": 16777728
 }
diff --git a/src/config/index.ts b/src/config/index.ts
@@ -18,6 +18,8 @@ interface Symbol {
 interface Monitor {
 	NODE_MONITOR_SCHEDULE_INTERVAL: number;
 	NUMBER_OF_NODE_REQUEST_CHUNK: number;
+	NODE_PEERS_REQUEST_CHUNK_SIZE: number;
+	CHAIN_HEIGHT_REQUEST_CHUNK_SIZE: number;
 	CHAIN_HEIGHT_MONITOR_SCHEDULE_INTERVAL: number;
 	GEOLOCATION_MONITOR_SCHEDULE_INTERVAL: number;
 	API_NODE_PORT: number;
@@ -49,6 +51,8 @@ export const symbol: Symbol = {
 export const monitor: Monitor = {
 	NODE_MONITOR_SCHEDULE_INTERVAL: Number(process.env.NODE_MONITOR_SCHEDULE_INTERVAL) || config.NODE_MONITOR_SCHEDULE_INTERVAL,
 	NUMBER_OF_NODE_REQUEST_CHUNK: Number(process.env.NUMBER_OF_NODE_REQUEST_CHUNK) || config.NUMBER_OF_NODE_REQUEST_CHUNK,
+	NODE_PEERS_REQUEST_CHUNK_SIZE: Number(process.env.NODE_PEERS_REQUEST_CHUNK_SIZE) || config.NODE_PEERS_REQUEST_CHUNK_SIZE,
+	CHAIN_HEIGHT_REQUEST_CHUNK_SIZE: Number(process.env.CHAIN_HEIGHT_REQUEST_CHUNK_SIZE) || config.CHAIN_HEIGHT_REQUEST_CHUNK_SIZE,
 	CHAIN_HEIGHT_MONITOR_SCHEDULE_INTERVAL:
 		Number(process.env.CHAIN_HEIGHT_MONITOR_SCHEDULE_INTERVAL) || config.CHAIN_HEIGHT_MONITOR_SCHEDULE_INTERVAL,
 	GEOLOCATION_MONITOR_SCHEDULE_INTERVAL:
@@ -85,6 +89,12 @@ export const verifyConfig = (cfg: Config): boolean => {
 	if (isNaN(cfg.monitor.NUMBER_OF_NODE_REQUEST_CHUNK) || cfg.monitor.NUMBER_OF_NODE_REQUEST_CHUNK < 0)
 		error = 'Invalid "NUMBER_OF_NODE_REQUEST_CHUNK"';
 
+	if (isNaN(cfg.monitor.NODE_PEERS_REQUEST_CHUNK_SIZE) || cfg.monitor.NODE_PEERS_REQUEST_CHUNK_SIZE < 0)
+		error = 'Invalid "NODE_PEERS_REQUEST_CHUNK_SIZE"';
+
+	if (isNaN(cfg.monitor.CHAIN_HEIGHT_REQUEST_CHUNK_SIZE) || cfg.monitor.CHAIN_HEIGHT_REQUEST_CHUNK_SIZE < 0)
+		error = 'Invalid "CHAIN_HEIGHT_REQUEST_CHUNK_SIZE"';
+
 	if (isNaN(cfg.monitor.API_NODE_PORT) || cfg.monitor.API_NODE_PORT <= 0 || cfg.monitor.API_NODE_PORT >= 10000)
 		error = 'Invalid "API_NODE_PORT"';
 

diff --git a/src/models/Node.ts b/src/models/Node.ts
@@ -50,6 +50,7 @@ const NodeSchema: Schema = new Schema({
 			type: Number,
 			required: false,
 		},
+		required: false,
 	},
 	apiStatus: {
 		webSocket: {
@@ -236,7 +237,9 @@ NodeSchema.set('toObject', {
 export const Node = mongoose.model<NodeDocument>('Node', NodeSchema);
 
 export const validateNodeModel = (node: any): boolean => {
-	if (!node || typeof node !== 'object') return false;
+	if (!node || typeof node !== 'object') {
+		return false;
+	}
 
 	return !new Node(node).validateSync();
 };
diff --git a/src/services/ApiNodeService.ts b/src/services/ApiNodeService.ts
@@ -163,7 +163,7 @@ export class ApiNodeService {
 		try {
 			return (await HTTP.get(`${hostUrl}/node/info`)).data;
 		} catch (e) {
-			logger.error(`Fail to request /node/info: ${hostUrl}`, e);
+			logger.error(`[getNodeInfo] Fail to request /node/info: ${hostUrl}`, e);
 			return null;
 		}
 	};
@@ -172,7 +172,7 @@ export class ApiNodeService {
 		try {
 			return (await HTTP.get(`${hostUrl}/chain/info`)).data;
 		} catch (e) {
-			logger.error(`Fail to request /chain/info: ${hostUrl}`, e);
+			logger.error(`[getNodeChainInfo] Fail to request /chain/info: ${hostUrl}`, e);
 			return null;
 		}
 	};
@@ -183,7 +183,7 @@ export class ApiNodeService {
 
 			return nodeServerInfo.serverInfo;
 		} catch (e) {
-			logger.error(`Fail to request /node/server: ${hostUrl}`, e);
+			logger.error(`[getNodeServer] Fail to request /node/server: ${hostUrl}`, e);
 			return null;
 		}
 	};
@@ -194,7 +194,7 @@ export class ApiNodeService {
 
 			return health.status;
 		} catch (e) {
-			logger.error(`Fail to request /node/health: ${hostUrl}`, e);
+			logger.error(`[getNodeHealth] Fail to request /node/health: ${hostUrl}`, e);
 			return null;
 		}
 	};

diff --git a/src/services/ChainHeightMonitor.ts b/src/services/ChainHeightMonitor.ts
@@ -5,8 +5,8 @@ import { Logger } from '@src/infrastructure';
 
 import { INode } from '@src/models/Node';
 import { INodeHeightStats } from '@src/models/NodeHeightStats';
-import { symbol } from '@src/config';
-import { isAPIRole, basename, sleep } from '@src/utils';
+import { symbol, monitor } from '@src/config';
+import { isAPIRole, basename, sleep, runTaskInChunks } from '@src/utils';
 
 const logger: winston.Logger = Logger.getLogger(basename(__filename));
 
@@ -61,7 +61,7 @@ export class ChainHeightMonitor {
 		try {
 			this.nodeList = (await DataBase.getNodeList()).filter((node) => isAPIRole(node.roles));
 		} catch (e) {
-			logger.error('Failed to get node list. Use nodes from config');
+			logger.error('[getNodeList] Failed to get node list. Use nodes from config');
 			for (const node of symbol.NODES) {
 				const url = new URL(node);
 				const hostUrl = await ApiNodeService.buildHostUrl(url.hostname);
@@ -80,32 +80,32 @@ export class ChainHeightMonitor {
 
 	private getNodeChainHeight = async () => {
 		logger.info(`Getting height stats for ${this.nodeList.length} nodes`);
-		const nodes: INode[] = this.nodeList;
-		const nodeChainInfoPromises = nodes.map((node) => {
-			const isHttps = node.apiStatus?.isHttpsEnabled;
-			const protocol = isHttps ? 'https:' : 'http:';
-			const port = isHttps ? 3001 : 3000;
 
-			const hostUrl = `${protocol}//${node.host}:${port}`;
-
-			return ApiNodeService.getNodeChainInfo(hostUrl);
-		});
-		const nodeChainInfoList = await Promise.all(nodeChainInfoPromises);
-
-		for (let chainInfo of nodeChainInfoList) {
-			try {
-				if (chainInfo) {
-					if (this.heights[chainInfo.height]) this.heights[chainInfo.height]++;
-					else this.heights[chainInfo.height] = 1;
-
-					if (this.finalizedHeights[chainInfo.latestFinalizedBlock.height])
-						this.finalizedHeights[chainInfo.latestFinalizedBlock.height]++;
-					else this.finalizedHeights[chainInfo.latestFinalizedBlock.height] = 1;
+		await runTaskInChunks(this.nodeList, monitor.CHAIN_HEIGHT_REQUEST_CHUNK_SIZE, logger, 'getNodeChainHeight', async (nodes) => {
+			const nodeChainInfoPromises = nodes.map((node) => {
+				const isHttps = node.apiStatus?.isHttpsEnabled;
+				const protocol = isHttps ? 'https:' : 'http:';
+				const port = isHttps ? 3001 : 3000;
+
+				const hostUrl = `${protocol}//${node.host}:${port}`;
+
+				return ApiNodeService.getNodeChainInfo(hostUrl);
+			});
+			const nodeChainInfoList = await Promise.all(nodeChainInfoPromises);
+
+			for (const chainInfo of nodeChainInfoList) {
+				try {
+					if (chainInfo) {
+						this.heights[chainInfo.height] = (this.heights[chainInfo.height] || 0) + 1;
+						this.finalizedHeights[chainInfo.latestFinalizedBlock.height] =
+							(this.finalizedHeights[chainInfo.latestFinalizedBlock.height] || 0) + 1;
+					}
+				} catch (e) {
+					logger.error(`Node chain height monitor failed. ${e.message}`);
 				}
-			} catch (e) {
-				logger.error(`Node chain height monitor failed. ${e.message}`);
 			}
-		}
+			return nodeChainInfoList;
+		});
 	};
 
 	private clear = () => {

diff --git a/src/services/DataBase.ts b/src/services/DataBase.ts
@@ -109,28 +109,30 @@ export class DataBase {
 		collectionName: string,
 	) => {
 		const prevState = await model.find().exec();
-		let error = Error();
 
 		try {
 			await model.deleteMany();
 		} catch (e) {
-			logger.error(`Update collection "${collectionName}" failed. Error during "model.deleteMany()". ${error.message}`);
-			throw error;
+			const msg = `Update collection "${collectionName}" failed. Error during "model.deleteMany()". ${e.message}`;
+
+			logger.error(msg);
+			throw new Error(msg);
 		}
 
 		try {
 			await model.insertMany(documents);
 		} catch (e) {
-			logger.error(`Update collection "${collectionName}" failed. Error during "model.insertMany()". ${error.message}`);
-			await model.insertMany(prevState);
-			throw error;
+			const msg = `Update collection "${collectionName}" failed. Error during "model.insertMany()". ${e.message}`;
+
+			logger.error(msg);
+			throw new Error(msg);
 		}
 
 		const currentState = await model.find().exec();
 
 		if (documents.length !== currentState.length) {
 			logger.error(
-				`Update collection "${collectionName}" failed. Collectin.length(${currentState.length}) !== documentsToInsert.length(${documents.length})`,
+				`Update collection "${collectionName}" failed. Collection.length(${currentState.length}) !== documentsToInsert.length(${documents.length})`,
 			);
 			await model.insertMany(prevState);
 			throw new Error(`Failed to update collection "${collectionName}. Length verification failed`);

diff --git a/src/services/HostInfo.ts b/src/services/HostInfo.ts
@@ -43,7 +43,7 @@ export class HostInfo {
 				zip: data.zip,
 			};
 		} catch (e) {
-			logger.error(`Failed to get host ${host} info ${e.message}`);
+			logger.error(`[getHostDetail] Failed to get host ${host} info ${e.message}`);
 			return null;
 		}
 	};