diff --git a/bin/control.sh b/bin/control.sh index 13056fbb..5c928926 100755 --- a/bin/control.sh +++ b/bin/control.sh @@ -26,7 +26,7 @@ HOMEDIR="$(dirname "$(cd -- "$(dirname "$0")" && (pwd -P 2>/dev/null || pwd))")" cd $HOMEDIR # # the path to your binary, including options if necessary -BINARY="node --expose_gc --always_compact $HOMEDIR/lib/main.js" +BINARY="node $HOMEDIR/lib/main.js" # # the path to your PID file PIDFILE=$HOMEDIR/logs/cronicled.pid diff --git a/bin/debug.sh b/bin/debug.sh index f927d87b..98795191 100755 --- a/bin/debug.sh +++ b/bin/debug.sh @@ -7,4 +7,4 @@ HOMEDIR="$(dirname "$(cd -- "$(dirname "$0")" && (pwd -P 2>/dev/null || pwd))")" cd $HOMEDIR -node --expose_gc --always_compact --trace-warnings $HOMEDIR/lib/main.js --debug --echo "$@" +node --trace-warnings $HOMEDIR/lib/main.js --debug --echo "$@" diff --git a/lib/api.js b/lib/api.js index 589fda0e..32a17d9f 100644 --- a/lib/api.js +++ b/lib/api.js @@ -57,6 +57,41 @@ module.exports = Class.create({ } ); }, + api_status: function(args, callback) { + // simple status, used by monitoring tools + var tick_age = 0; + var now = Tools.timeNow(); + if (this.lastTick) tick_age = now - this.lastTick; + + // do not cache this API response + this.forceNoCacheResponse(args); + + var data = { + code: 0, + version: this.server.__version, + node: process.version, + hostname: this.server.hostname, + ip: this.server.ip, + pid: process.pid, + now: now, + uptime: Math.floor( now - (this.server.started || now) ), + last_tick: this.lastTick || now, + tick_age: tick_age, + cpu: process.cpuUsage(), + mem: process.memoryUsage() + }; + + callback(data); + + // self-check: if tick_age is over 60 seconds, force a self restart + if (tick_age > 60) { + var msg = "EMERGENCY SHUTDOWN: Tick age is over 60 seconds (" + Math.floor(tick_age) + "s)"; + this.logger.set('sync', true); + this.logDebug(1, msg, data); + this.restartLocalServer({ reason: msg }); + } + }, + forceNoCacheResponse: function(args) { // make sure this response isn't cached, ever args.response.setHeader( 'Cache-Control', 'no-cache, no-store, must-revalidate, proxy-revalidate' ); diff --git a/lib/discovery.js b/lib/discovery.js index f39e237a..a9daa6cb 100644 --- a/lib/discovery.js +++ b/lib/discovery.js @@ -22,6 +22,12 @@ module.exports = Class.create({ this.nearbyServers = {}; this.lastDiscoveryBroadcast = 0; + // disable if port is unset + if (!this.server.config.get('udp_broadcast_port')) { + if (callback) callback(); + return; + } + // guess best broadcast IP this.broadcastIP = this.server.config.get('broadcast_ip') || this.calcBroadcastIP(); this.logDebug(4, "Using broadcast IP: " + this.broadcastIP ); diff --git a/lib/engine.js b/lib/engine.js index 4624807c..ca66f607 100644 --- a/lib/engine.js +++ b/lib/engine.js @@ -226,7 +226,8 @@ module.exports = Class.create({ tick: function() { // called every second var self = this; - var now = Tools.timeNow(true); + this.lastTick = Tools.timeNow(); + var now = Math.floor(this.lastTick); if (this.numSocketClients) { var status = { @@ -585,7 +586,7 @@ module.exports = Class.create({ function(hostname, callback) { var server = candidates[hostname]; - var api_url = self.getServerBaseAPIURL( hostname, server.ip ) + '/app/ping'; + var api_url = self.getServerBaseAPIURL( hostname, server.ip ) + '/app/status'; self.logDebug(10, "Sending API request to remote server: " + hostname + ": " + api_url); // send request @@ -602,6 +603,10 @@ module.exports = Class.create({ self.logDebug(10, "Failed to ping server: " + hostname + ": " + data.description); return callback(); } + if (data.tick_age > 60) { + self.logDebug(1, "WARNING: Failed to ping server: " + hostname + ": Tick age is " + Math.floor(data.tick_age) + "s", data); + return callback(); + } // success, at least one superior ponged our ping // relinquish command to them @@ -1040,6 +1045,7 @@ module.exports = Class.create({ // perform emergency shutdown due to uncaught exception this.logger.set('sync', true); this.logError('crash', "Emergency Shutdown: " + err); + this.logDebug(1, "Emergency Shutdown: " + err); this.abortAllLocalJobs(); } diff --git a/package.json b/package.json index 8c212267..18adc751 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "Cronicle", - "version": "0.8.24", + "version": "0.8.25", "description": "A simple, distributed task scheduler and runner with a web based UI.", "author": "Joseph Huckaby ", "homepage": "https://github.com/jhuckaby/Cronicle", @@ -51,7 +51,7 @@ "pixl-request": "^1.0.21", "pixl-mail": "^1.0.8", "pixl-perf": "^1.0.5", - "pixl-server": "^1.0.14", + "pixl-server": "^1.0.15", "pixl-server-storage": "^2.0.9", "pixl-server-web": "^1.1.7", "pixl-server-api": "^1.0.1", diff --git a/sample_conf/config.json b/sample_conf/config.json index 0054e712..09312f69 100644 --- a/sample_conf/config.json +++ b/sample_conf/config.json @@ -7,7 +7,7 @@ "log_dir": "logs", "log_filename": "[component].log", - "log_columns": ["hires_epoch", "date", "hostname", "component", "category", "code", "msg", "data"], + "log_columns": ["hires_epoch", "date", "hostname", "pid", "component", "category", "code", "msg", "data"], "log_archive_path": "logs/archives/[yyyy]/[mm]/[dd]/[filename]-[yyyy]-[mm]-[dd].log.gz", "log_crashes": true, "copy_job_logs_to": "",