Skip to content

Commit

Permalink
Reporting:
Browse files Browse the repository at this point in the history
- Replace jmx.js with external.js. Add graphProcess() which replaces spawnAndMonitor()
- graphJmx() checks for existence of jmxstat/jmxstat.jar
- Add report.getReport, which returns the first report by the same name or creates one

Util:
- Add LineReader class

Console:
- Change keyboard shortcuts on console UI
- Update console dygraph version
- Change console x-axis to use timestamps
  • Loading branch information
jonjlee committed Jul 29, 2011
1 parent 2cafa3b commit d4893f8
Show file tree
Hide file tree
Showing 15 changed files with 182 additions and 144 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
.PHONY: clean templates compile
PROCESS_TPL = scripts/process_tpl.js
SOURCES = lib/header.js lib/config.js lib/util.js lib/stats.js lib/loop/loop.js lib/loop/multiloop.js lib/monitoring/collectors.js lib/monitoring/statslogger.js lib/monitoring/monitor.js lib/monitoring/monitorgroup.js lib/http.js lib/reporting/*.tpl.js lib/reporting/template.js lib/reporting/report.js lib/reporting/reportmanager.js lib/reporting/jmx.js lib/loadtesting.js lib/remote/endpoint.js lib/remote/endpointclient.js lib/remote/slave.js lib/remote/slaves.js lib/remote/slavenode.js lib/remote/cluster.js lib/remote/httphandler.js lib/remote/remotetesting.js
SOURCES = lib/header.js lib/config.js lib/util.js lib/stats.js lib/loop/loop.js lib/loop/multiloop.js lib/monitoring/collectors.js lib/monitoring/statslogger.js lib/monitoring/monitor.js lib/monitoring/monitorgroup.js lib/http.js lib/reporting/*.tpl.js lib/reporting/template.js lib/reporting/report.js lib/reporting/reportmanager.js lib/reporting/external.js lib/loadtesting.js lib/remote/endpoint.js lib/remote/endpointclient.js lib/remote/slave.js lib/remote/slaves.js lib/remote/slavenode.js lib/remote/cluster.js lib/remote/httphandler.js lib/remote/remotetesting.js

all: compile

Expand Down
7 changes: 4 additions & 3 deletions RELEASE-NOTES.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ Compatible with node v0.4.x

Features:

* rps is now a separate stat from result codes
* summary graph time does not auto-update on open
* graph x-axis now use times rather than minutes since test start
* Add graphJmx() and graphProcess(); deprecate spawnAndMonitor(). These provide an easy way to graph JMX attributes as well as output from external processes, such as iostat.
* rps is a separate stat from result codes
* Test Results page timestamp does not auto-update on open
* X-axes now use real timestamps rather than minutes since test start

## v0.3.0 (2011/06/16) ##

Expand Down
1 change: 0 additions & 1 deletion TODO
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
- Add zipf number generator
- Download/copy data in CSV
- Allow output directory to be customized
- Add JMX monitoring support
- Add support for bar graphs
- Methods for graphing histograms
- Allow graphs to be overlayed
Expand Down
4 changes: 2 additions & 2 deletions console/console.html
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
<input id="txtNewHost" type="text"></input>
<button id="cmdAdd">Add</button>
</div>
<div style="float: right; color: gray;">&lt; k &nbsp;&nbsp; j &gt;</div>
<div style="float: right; color: gray;">&lt; K &nbsp;&nbsp; J &gt;</div>
</td>
</tr></table>
</div>
Expand All @@ -49,7 +49,7 @@
<div id="pnlRightColumn">
<div>
<h3><a href="#">Overall Statistics</a></h3>
<div id="pnlSummary" class="ui-widget-content ui-corner-all"/>
<div id="pnlSummary" class="ui-widget-content ui-corner-all"></div>
</div>
<div>
<h3><a href="#">Test Details</a></h3>
Expand Down
21 changes: 12 additions & 9 deletions console/js/console.ui.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ var Dygraph;
// ---------------
// UI creation
// ---------------
var CHART_LEGEND_WIDTH = 70;
var CHART_LEGEND_WIDTH = 120;

var doc, optNodes, frmAddNode, cmdAddNode, cmdAdd, txtNewHost,
pnlCharts, pnlRightColumn, pnlSummary;
Expand Down Expand Up @@ -109,7 +109,7 @@ function addNodeButton(node) {
}
function addNodeTabs(node) {
var tabs = $(['<div id="tab-charts-', node.id, '">',
' <div class="clsShortcutKeys">&lt; h &nbsp;&nbsp; l &gt;</div>',
' <div class="clsShortcutKeys">&lt; k &nbsp;&nbsp; j &gt;</div>',
' <ul></ul>',
'</div>'
].join(''));
Expand Down Expand Up @@ -151,7 +151,9 @@ function refreshReportGraphs(node) {
for (var j in charts) {
var chartId = 'chart-' + node.id + '-' + reportId + '-' + getIdFromString(j),
chartContainerId = chartId + '-container',
chartLegendId = chartId + '-legend';
chartLegendId = chartId + '-legend',
rows = charts[j].rows.map(function(x) { return [new Date(x[0])].concat(x.slice(1)); });

if (!node.graphs[chartId]) {
tab.append([
'<h2 class="clsChartTitle">', j, '</h2><div id="', chartContainerId, '" class="clsChartContainer"> ',
Expand All @@ -161,15 +163,16 @@ function refreshReportGraphs(node) {
].join(''));
node.graphs[chartId] = new Dygraph(
document.getElementById(chartId),
charts[j].rows,
rows,
{labelsDiv: $('#' + chartLegendId)[0],
labelsSeparateLines: true,
labels: charts[j].columns,
strokeWidth: 1.5,
xAxisLabelWidth: 80
});
node.graphs[chartId].container = $('#' + chartContainerId);
} else {
node.graphs[chartId].updateOptions({"file": charts[j].rows });
node.graphs[chartId].updateOptions({"file": rows, labels: charts[j].columns});
}
}

Expand Down Expand Up @@ -197,26 +200,26 @@ function initShortcuts() {
return false;
}
});
doc.bind('keydown', 'k', function() {
doc.bind('keydown', 'shift+k', function() {
var prev = optNodes.find('label.ui-state-active').parent().prev();
if (!prev) { return; }

prev.find('input').button().click();
optNodes.buttonset('refresh');
});
doc.bind('keydown', 'j', function() {
doc.bind('keydown', 'shift+j', function() {
var next = optNodes.find('label.ui-state-active').parent().next();
if (!next) { return; }

next.find('input').button().click();
optNodes.buttonset('refresh');
});
doc.bind('keydown', 'h', function() {
doc.bind('keydown', 'k', function() {
if (!selectedNode) { return; }
var selected = selectedNode.tabs.tabs('option', 'selected');
selectedNode.tabs.tabs('select', selected-1);
});
doc.bind('keydown', 'l', function() {
doc.bind('keydown', 'j', function() {
if (!selectedNode) { return; }
var selected = selectedNode.tabs.tabs('option', 'selected');
selectedNode.tabs.tabs('select', selected+1);
Expand Down
2 changes: 1 addition & 1 deletion console/js/dygraph-combined.min.js

Large diffs are not rendered by default.

59 changes: 30 additions & 29 deletions examples/graphjmx.ex.js
100644 → 100755
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
#!/usr/bin/env node

/*jslint forin:true */

var assert = require('assert'),
Expand All @@ -10,38 +12,37 @@ REPORT_MANAGER.setLogFile('.reporting.test-output.html');
var hostAndPort = 'localhost:9999',
refreshInterval = 2;

var report = REPORT_MANAGER.addReport('JMX'),
memory = report.getChart('Memory'),
cpu = report.getChart('CPU');

var jmx = reporting.spawnAndMonitor(
/HeapMemoryUsage.max=(.*) HeapMemoryUsage.committed=(.*) HeapMemoryUsage.used=(.*) SystemLoadAverage=([0-9\.]*)/,
['max', 'committed', 'used', 'loadavg'],
'java', [
'-jar', 'jmxstat/jmxstat.jar',
hostAndPort,
'java.lang:type=Memory[HeapMemoryUsage.max,HeapMemoryUsage.committed,HeapMemoryUsage.used]',
'java.lang:type=OperatingSystem[SystemLoadAverage]',
refreshInterval
]
),
iostat = reporting.spawnAndMonitor(
/ +[^ ]+ +[^ ]+ +[^ ]+ +([^ ]+) +([^ ]+) +[^ ]+ +[^ ]+ +[^ ]+ +[^ ]+/,
['user', 'system'],
'iostat', [refreshInterval]
);

jmx.stderr.on('data', function (data) {
console.log(data.toString());
var jmx = reporting.graphJmx({
host: 'localhost:9999',
reportName: 'Monitors',
chartName: 'Heap',
mbeans: {
'Used': 'java.lang:type=Memory[HeapMemoryUsage.used]',
'Committed': 'java.lang:type=Memory[HeapMemoryUsage.committed]'
},
dataFormatter: function(data) {
return {
Used: data.Used / 1024,
Committed: data.Committed /= 1024
};
},
interval: refreshInterval
});

jmx.on('exit', function (code) {
if (code !== 0) { console.log('JMX monitor died with code ' + code); }
process.exit(code);
reporting.graphProcess({
reportName: 'Monitors',
chartName: 'CPU (iostat)',
command: 'iostat -C ' + refreshInterval,
columns: [null, null, null, 'tps', 'MB/s'],
});

jmx.on('data', function(data) {
memory.put({max: data.max/1024, committed: data.committed/1024, used: data.used/1024});
jmx.stderr.on('data', function(data) {
console.log(data.toString());
});

cpu.updateFromEventEmitter(iostat, ['user', 'system']);
jmx.on('exit', function(code) {
if (code !== 0) {
console.log('JMX monitor died with code ' + code);
}
process.exit(code);
});
1 change: 1 addition & 0 deletions lib/header.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ var util = require('util'),
http = require('http'),
url = require('url'),
fs = require('fs'),
path = require('path'),
events = require('events'),
querystring = require('querystring'),
child_process = require('child_process');
Expand Down
82 changes: 82 additions & 0 deletions lib/reporting/external.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
/*jslint forin:true */

var BUILD_AS_SINGLE_FILE;
if (!BUILD_AS_SINGLE_FILE) {
var child_process = require('child_process');
var REPORT_MANAGER = require('./reportmanager').REPORT_MANAGER;
var util = require('../util');
var path = require('path');
}

var graphProcess;

var graphJmx = exports.graphJmx = function(options) {
// Verify that java & jmxstat jar can be found. Search for jmxstat/jmxstat.jar located next to the
// current module or a parent module that included it.
var m = module;
var jmxstat, found = false;
while (m && !found) {
jmxstat = path.join(path.dirname(m.filename), 'jmxstat/jmxstat.jar');
found = path.existsSync(jmxstat);
m = m.parent;
}
if (!found) {
throw new Error('jmxstat/jmxstat.jar not found.');
}

// Build command line args, output regex, and field labels
var regex = '\\d{2}:\\d{2}:\\d{2}', columns = [], mbeans = [];
for (var mbean in options.mbeans) {
regex += '\\t([^\\t]*)';
columns.push(mbean);
mbeans.push(options.mbeans[mbean]);
}

// Start jmxstat
var interval = options.interval || '';
return graphProcess({
reportName: options.reportName || options.host || 'Monitor',
chartName: options.chartName || 'JMX',
command: 'java -jar ' + jmxstat + ' ' + options.host + ' ' + mbeans.join(' ') + ' ' + interval,
columns: columns,
regex: regex,
dataFormatter: options.dataFormatter
});
};


/** Spawn a child process, extract data using a regex, and graph the results on the summary report.
Returns a standard ChildProcess object.
*/
var graphProcess = exports.graphProcess = function(options) {
var delimiter = options.delimiter || ' +',
columns = options.columns || [],
fieldRegex = columns.map(function() { return '(.*?)'; }).join(delimiter), // e.g. (.*?) +(.*?) +...
regex = options.regex || ('^ *' + fieldRegex + ' *$'),
splitIdx = columns.indexOf(options.splitBy) + 1;

var report = REPORT_MANAGER.getReport(options.reportName || 'Monitor'),
graph = report.getChart(options.chartName || options.command),
format = options.dataFormatter || function(x) { return x; };

var proc = child_process.spawn('/bin/sh', ['-c', options.command], options.spawnOptions),
lr = new util.LineReader(proc.stdout);

lr.on('data', function (line) {
var vals = line.match(regex);
if (vals) {
var obj = {}, prefix = '';
if (splitIdx > 0 && vals[splitIdx]) {
prefix = vals[splitIdx] + ' ';
}
for (var i = 1; i < vals.length; i++) {
if (columns[i-1]) {
obj[prefix + columns[i-1]] = vals[i];
}
}
graph.put(format(obj));
}
});

return proc;
};
3 changes: 2 additions & 1 deletion lib/reporting/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,5 @@ exports.Report = report.Report;
exports.Chart = report.Chart;
exports.ReportGroup = report.ReportGroup;
exports.REPORT_MANAGER= require('./reportmanager').REPORT_MANAGER;
exports.graphJmx = require('./jmx').graphJmx;
exports.graphProcess = require('./external').graphProcess;
exports.graphJmx = require('./external').graphJmx;
88 changes: 0 additions & 88 deletions lib/reporting/jmx.js

This file was deleted.

1 change: 1 addition & 0 deletions lib/reporting/jmxstat
Loading

0 comments on commit d4893f8

Please sign in to comment.