diff --git a/core/lib/engine_http.js b/core/lib/engine_http.js index 8d5e64432f..fe38189279 100644 --- a/core/lib/engine_http.js +++ b/core/lib/engine_http.js @@ -407,10 +407,13 @@ HttpEngine.prototype.step = function step(requestSpec, ee, opts) { functionNames, function iteratee(functionName, next) { let processFunc = config.processor[functionName]; - processFunc(requestParams, res, context, ee, function(err) { + processFunc(requestParams, res, context, ee, function(err, assertionResult) { if (err) { return next(err); } + if (assertionResult) { + ee.emit('assertions', assertionResult) + } return next(null); }); }, function(err) { diff --git a/core/lib/runner.js b/core/lib/runner.js index 863afebcb3..65ee8fa779 100644 --- a/core/lib/runner.js +++ b/core/lib/runner.js @@ -51,11 +51,11 @@ function validate(script) { async function runner(script, payload, options, callback) { let opts = _.assign({ - periodicStats: script.config.statsInterval || 10, - mode: script.config.mode || "uniform", - isAggregateReport: true - }, - options); + periodicStats: script.config.statsInterval || 10, + mode: script.config.mode || "uniform", + isAggregateReport: true + }, + options); if (opts.isAggregateReport === false) { isAggregateReport = false; @@ -109,29 +109,29 @@ async function runner(script, payload, options, callback) { // load engines: // let runnerEngines = _.map( - Object.assign({}, Engines, runnableScript.config.engines), - function loadEngine(engineConfig, engineName) { - let moduleName = "artillery-engine-" + engineName; - try { - if (Engines[engineName]) { - moduleName = "./engine_" + engineName; + Object.assign({}, Engines, runnableScript.config.engines), + function loadEngine(engineConfig, engineName) { + let moduleName = "artillery-engine-" + engineName; + try { + if (Engines[engineName]) { + moduleName = "./engine_" + engineName; + } + let Engine = require(moduleName); + let engine = new Engine(runnableScript, ee, engineUtil); + engine.__name = engineName; + return engine; + } catch (err) { + console.log( + "WARNING: engine %s specified but module %s could not be loaded", + engineName, + moduleName); + console.log(err.stack); + warnings.engines[engineName] = { + message: "Could not load", + error: err + }; } - let Engine = require(moduleName); - let engine = new Engine(runnableScript, ee, engineUtil); - engine.__name = engineName; - return engine; - } catch (err) { - console.log( - "WARNING: engine %s specified but module %s could not be loaded", - engineName, - moduleName); - console.log(err.stack); - warnings.engines[engineName] = { - message: "Could not load", - error: err - }; } - } ); // @@ -174,8 +174,8 @@ async function runner(script, payload, options, callback) { debug(ignoreErr); } runnableScript.config.plugins = Object.assign( - runnableScript.config.plugins, - additionalPlugins); + runnableScript.config.plugins, + additionalPlugins); } _.each(runnableScript.config.plugins, function tryToLoadPlugin(pluginConfig, pluginName) { @@ -203,9 +203,9 @@ async function runner(script, payload, options, callback) { if (!Plugin || !plugin) { console.log( - "WARNING: plugin %s specified but module %s could not be loaded", - pluginName, - requireString); + "WARNING: plugin %s specified but module %s could not be loaded", + pluginName, + requireString); warnings.plugins[pluginName] = { message: "Could not load" }; @@ -233,22 +233,22 @@ async function runner(script, payload, options, callback) { ee.stop = function(done) { // allow plugins to cleanup A.eachSeries( - runnerPlugins, - function(plugin, next) { - if (plugin.cleanup) { - plugin.cleanup(function(err) { - if (err) { - debug(err); - } + runnerPlugins, + function(plugin, next) { + if (plugin.cleanup) { + plugin.cleanup(function(err) { + if (err) { + debug(err); + } + return next(); + }); + } else { return next(); - }); - } else { - return next(); - } - }, - function(err) { - return done(err); - }); + } + }, + function(err) { + return done(err); + }); }; // FIXME: Warnings should be returned from this function instead along with @@ -360,6 +360,9 @@ function runScenario(script, intermediate, runState, contextVars) { runState.scenarioEvents.on("error", function(errCode) { intermediate.addError(errCode); }); + runState.scenarioEvents.on("assertions", function(assertionsResults) { + intermediate.addAssertion(assertionsResults); + }); runState.scenarioEvents.on("request", function() { intermediate.newRequest(); @@ -381,21 +384,21 @@ function runScenario(script, intermediate, runState, contextVars) { }); runState.compiledScenarios = _.map( - script.scenarios, - function(scenarioSpec) { - const name = scenarioSpec.engine || script.config.engine || "http"; - const engine = runState.engines.find((e) => e.__name === name); - return engine.createScenario(scenarioSpec, runState.scenarioEvents); - } + script.scenarios, + function(scenarioSpec) { + const name = scenarioSpec.engine || script.config.engine || "http"; + const engine = runState.engines.find((e) => e.__name === name); + return engine.createScenario(scenarioSpec, runState.scenarioEvents); + } ); } let i = runState.picker()[0]; debug("picking scenario %s (%s) weight = %s", - i, - script.scenarios[i].name, - script.scenarios[i].weight); + i, + script.scenarios[i].name, + script.scenarios[i].weight); intermediate.newScenario(script.scenarios[i].name || i); diff --git a/core/lib/stats2.js b/core/lib/stats2.js index 570ebcf378..254408d081 100644 --- a/core/lib/stats2.js +++ b/core/lib/stats2.js @@ -25,13 +25,13 @@ function create() { */ function combine(statsObjects) { let result = create(); - L.each(statsObjects, function(stats) { - L.each(stats._latencies, function(latency) { + L.each(statsObjects, function (stats) { + L.each(stats._latencies, function (latency) { result._latencies.push(latency); }); result._generatedScenarios += stats._generatedScenarios; - L.each(stats._scenarioCounter, function(count, name) { - if(result._scenarioCounter[name]) { + L.each(stats._scenarioCounter, function (count, name) { + if (result._scenarioCounter[name]) { result._scenarioCounter[name] += count; } else { result._scenarioCounter[name] = count; @@ -39,41 +39,41 @@ function combine(statsObjects) { }); result._completedScenarios += stats._completedScenarios; result._scenariosAvoided += stats._scenariosAvoided; - L.each(stats._codes, function(count, code) { - if(result._codes[code]) { + L.each(stats._codes, function (count, code) { + if (result._codes[code]) { result._codes[code] += count; } else { result._codes[code] = count; } }); - L.each(stats._errors, function(count, error) { - if(result._errors[error]) { + L.each(stats._errors, function (count, error) { + if (result._errors[error]) { result._errors[error] += count; } else { result._errors[error] = count; } }); - L.each(stats._requestTimestamps, function(timestamp) { + L.each(stats._requestTimestamps, function (timestamp) { result._requestTimestamps.push(timestamp); }); result._completedRequests += stats._completedRequests; - L.each(stats._scenarioLatencies, function(latency) { + L.each(stats._scenarioLatencies, function (latency) { result._scenarioLatencies.push(latency); }); result._matches += stats._matches; - L.each(stats._counters, function(value, name) { + L.each(stats._counters, function (value, name) { if (!result._counters[name]) { result._counters[name] = 0; } result._counters[name] += value; }); - L.each(stats._customStats, function(values, name) { + L.each(stats._customStats, function (values, name) { if (!result._customStats[name]) { result._customStats[name] = []; } - L.each(values, function(v) { + L.each(values, function (v) { result._customStats[name].push(v); }); }); @@ -89,16 +89,16 @@ function Stats() { return this.reset(); } -Stats.prototype.addEntry = function(entry) { +Stats.prototype.addEntry = function (entry) { this._entries.push(entry); return this; }; -Stats.prototype.getEntries = function() { +Stats.prototype.getEntries = function () { return this._entries; }; -Stats.prototype.newScenario = function(name) { +Stats.prototype.newScenario = function (name) { if (this._scenarioCounter[name]) { this._scenarioCounter[name]++; } else { @@ -109,17 +109,17 @@ Stats.prototype.newScenario = function(name) { return this; }; -Stats.prototype.completedScenario = function() { +Stats.prototype.completedScenario = function () { this._completedScenarios++; return this; }; -Stats.prototype.avoidedScenario = function() { +Stats.prototype.avoidedScenario = function () { this._scenariosAvoided++; return this; }; -Stats.prototype.addCode = function(code) { +Stats.prototype.addCode = function (code) { if (!this._codes[code]) { this._codes[code] = 0; } @@ -127,7 +127,7 @@ Stats.prototype.addCode = function(code) { return this; }; -Stats.prototype.addError = function(errCode) { +Stats.prototype.addError = function (errCode) { if (!this._errors[errCode]) { this._errors[errCode] = 0; } @@ -135,36 +135,60 @@ Stats.prototype.addError = function(errCode) { return this; }; -Stats.prototype.newRequest = function() { +Stats.prototype.addAssertion = function (assertionResult) { + + if (!this._assertions[assertionResult.name]) { + this._assertions[assertionResult.name] = {} + } + + assertionResult.results.forEach(ar => { + const assertionName = `${ar.type} ${ar.expected}`; + if (!this._assertions[assertionResult.name][assertionName]) { + this._assertions[assertionResult.name][assertionName] = {success: 0, fail: 0, failureResponses: {}} + } + if (ar.ok) { + this._assertions[assertionResult.name][assertionName].success++; + } else { + this._assertions[assertionResult.name][assertionName].fail++; + if (!this._assertions[assertionResult.name][assertionName].failureResponses[ar.got]) { + this._assertions[assertionResult.name][assertionName].failureResponses[ar.got] = 0 + } + this._assertions[assertionResult.name][assertionName].failureResponses[ar.got]++; + } + }); + return this; +}; + +Stats.prototype.newRequest = function () { this._requestTimestamps.push(Date.now()); return this; }; -Stats.prototype.completedRequest = function() { +Stats.prototype.completedRequest = function () { this._completedRequests++; return this; }; -Stats.prototype.addLatency = function(delta) { +Stats.prototype.addLatency = function (delta) { this._latencies.push(delta); return this; }; -Stats.prototype.addScenarioLatency = function(delta) { +Stats.prototype.addScenarioLatency = function (delta) { this._scenarioLatencies.push(delta); return this; }; -Stats.prototype.addMatch = function() { +Stats.prototype.addMatch = function () { this._matches++; return this; }; -Stats.prototype.clone = function() { +Stats.prototype.clone = function () { return L.cloneDeep(this); }; -Stats.prototype.report = function() { +Stats.prototype.report = function () { let result = {}; result.timestamp = new Date().toISOString(); @@ -186,7 +210,7 @@ Stats.prototype.report = function() { let now = Date.now(); let count = L.size(this._requestTimestamps); let mean = Math.round( - (count / (Math.round((now - startedAt) / 10) / 100)) * 100) / 100; + (count / (Math.round((now - startedAt) / 10) / 100)) * 100) / 100; result.rps = { count: count, @@ -204,13 +228,14 @@ Stats.prototype.report = function() { result.scenarioCounts = this._scenarioCounter; result.errors = this._errors; + result.assertions = this._assertions; result.codes = this._codes; result.matches = this._matches; result.latencies = latencies; result.customStats = {}; - L.each(this._customStats, function(ns, name) { + L.each(this._customStats, function (ns, name) { result.customStats[name] = { min: round(L.min(ns), 1), max: round(L.max(ns), 1), @@ -230,7 +255,7 @@ Stats.prototype.report = function() { return result; }; -Stats.prototype.addCustomStat = function(name, n) { +Stats.prototype.addCustomStat = function (name, n) { if (!this._customStats[name]) { this._customStats[name] = []; } @@ -239,7 +264,7 @@ Stats.prototype.addCustomStat = function(name, n) { return this; }; -Stats.prototype.counter = function(name, value) { +Stats.prototype.counter = function (name, value) { if (!this._counters[name]) { this._counters[name] = 0; } @@ -247,13 +272,14 @@ Stats.prototype.counter = function(name, value) { return this; }; -Stats.prototype.reset = function() { +Stats.prototype.reset = function () { this._entries = []; this._latencies = []; this._generatedScenarios = 0; this._completedScenarios = 0; this._codes = {}; this._errors = {}; + this._assertions = {}; this._requestTimestamps = []; this._completedRequests = 0; this._scenarioLatencies = []; @@ -267,7 +293,7 @@ Stats.prototype.reset = function() { return this; }; -Stats.prototype.free = function() { +Stats.prototype.free = function () { return this; };