From 3110d0ee90995dffba0a8a4e5d88ecfcc8627bf9 Mon Sep 17 00:00:00 2001 From: Aaron Quint Date: Sun, 9 Feb 2014 23:30:23 -0500 Subject: [PATCH 1/3] Allow for aliasing the different timer metrics These can be set as needed in the config file and use the existing settings as the defaults --- exampleConfig.js | 14 ++++++++++++ lib/process_metrics.js | 41 +++++++++++++++++++++++++---------- stats.js | 4 ++-- test/process_metrics_tests.js | 30 ++++++++++++------------- 4 files changed, 60 insertions(+), 29 deletions(-) diff --git a/exampleConfig.js b/exampleConfig.js index 527cc5af..7319491d 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -53,6 +53,20 @@ Optional Variables: prefixStats: prefix to use for the statsd statistics data for this running instance of statsd [default: statsd] applies to both legacy and new namespacing + timerAliases: Individual aliases for the different percentile and timer metrics + meanPrefix: [default 'mean_'] + maxPrefix: [default 'upper_'] + minPrefix: [default 'lower_'] + sumPrefix: [default 'sum_'] + stddev: [default 'std'] + max: [default 'upper'] + min: [default 'lower'] + count: [default 'count'] + count_ps: [default 'count_ps'] + sum: [default 'sum'] + mean: [default 'mean'] + median: [default 'median'] + console: prettyprint: whether to prettyprint the console backend output [true or false, default: true] diff --git a/lib/process_metrics.js b/lib/process_metrics.js index bfaa7b2c..f3f54be8 100644 --- a/lib/process_metrics.js +++ b/lib/process_metrics.js @@ -1,6 +1,6 @@ /*jshint node:true, laxcomma:true */ -var process_metrics = function (metrics, flushInterval, ts, flushCallback) { +var process_metrics = function (metrics, conf, flushCallback) { var starttime = Date.now(); var key; var counter_rates = {}; @@ -11,6 +11,23 @@ var process_metrics = function (metrics, flushInterval, ts, flushCallback) { var timer_counters = metrics.timer_counters; var pctThreshold = metrics.pctThreshold; var histogram = metrics.histogram; + var flushInterval = conf.flushInterval; + var configAliases = conf.timerAliases || {}; + + var metricAliases = { + meanPrefix: configAliases.meanPrefix || 'mean_', + maxPrefix: configAliases.maxPrefix || 'upper_', + minPrefix: configAliases.maxPrefix || 'lower_', + sumPrefix: configAliases.maxPrefix || 'sum_', + stddev: configAliases.stddev || 'std', + max: configAliases.max || 'upper', + min: configAliases.min || 'lower', + count: configAliases.count || 'count', + count_ps: configAliases.count || 'count_ps', + sum: configAliases.sum || 'sum', + mean: configAliases.mean || 'mean', + median: configAliases.median || 'median' + }; for (key in counters) { var value = counters[key]; @@ -60,9 +77,9 @@ var process_metrics = function (metrics, flushInterval, ts, flushCallback) { var clean_pct = '' + pct; clean_pct = clean_pct.replace('.', '_').replace('-', 'top'); - current_timer_data["mean_" + clean_pct] = mean; - current_timer_data[(pct > 0 ? "upper_" : "lower_") + clean_pct] = thresholdBoundary; - current_timer_data["sum_" + clean_pct] = sum; + current_timer_data[metricAliases.meanPrefix + clean_pct] = mean; + current_timer_data[(pct > 0 ? metricAliases.maxPrefix : metricAliases.minPrefix) + clean_pct] = thresholdBoundary; + current_timer_data[metricAliases.sumPrefix + clean_pct] = sum; } @@ -78,14 +95,14 @@ var process_metrics = function (metrics, flushInterval, ts, flushCallback) { var median = (count % 2) ? values[mid] : (values[mid-1] + values[mid])/2; var stddev = Math.sqrt(sumOfDiffs / count); - current_timer_data["std"] = stddev; - current_timer_data["upper"] = max; - current_timer_data["lower"] = min; - current_timer_data["count"] = timer_counters[key]; - current_timer_data["count_ps"] = timer_counters[key] / (flushInterval / 1000); - current_timer_data["sum"] = sum; - current_timer_data["mean"] = mean; - current_timer_data["median"] = median; + current_timer_data[metricAliases.stddev] = stddev; + current_timer_data[metricAliases.max] = max; + current_timer_data[metricAliases.min] = min; + current_timer_data[metricAliases.count] = timer_counters[key]; + current_timer_data[metricAliases.count_ps] = timer_counters[key] / (flushInterval / 1000); + current_timer_data[metricAliases.sum] = sum; + current_timer_data[metricAliases.mean] = mean; + current_timer_data[metricAliases.median] = median; // note: values bigger than the upper limit of the last bin are ignored, by design conf = histogram || []; diff --git a/stats.js b/stats.js index ccfe6768..d26f233c 100644 --- a/stats.js +++ b/stats.js @@ -118,7 +118,7 @@ function flushMetrics() { } } - // normally gauges are not reset. so if we don't delete them, continue to persist previous value + // normally gauges are not reset. so if we don't delete them, continue to persist previous value conf.deleteGauges = conf.deleteGauges || false; if (conf.deleteGauges) { for (var gauge_key in metrics.gauges) { @@ -127,7 +127,7 @@ function flushMetrics() { } }); - pm.process_metrics(metrics_hash, flushInterval, time_stamp, function emitFlush(metrics) { + pm.process_metrics(metrics_hash, conf, function emitFlush(metrics) { backendEvents.emit('flush', time_stamp, metrics); }); diff --git a/test/process_metrics_tests.js b/test/process_metrics_tests.js index f6904467..05c6c5d0 100644 --- a/test/process_metrics_tests.js +++ b/test/process_metrics_tests.js @@ -25,14 +25,14 @@ module.exports = { counters_has_stats_count: function(test) { test.expect(1); this.metrics.counters['a'] = 2; - pm.process_metrics(this.metrics, 1000, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, {flushInterval: 1000}, function(){}); test.equal(2, this.metrics.counters['a']); test.done(); }, counters_has_correct_rate: function(test) { test.expect(1); this.metrics.counters['a'] = 2; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, {flushInterval: 100}, function(){}); test.equal(20, this.metrics.counter_rates['a']); test.done(); }, @@ -40,7 +40,7 @@ module.exports = { test.expect(1); this.metrics.timers['a'] = []; this.metrics.timer_counters['a'] = 0; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, {flushInterval: 1000}, function(){}); //potentially a cleaner way to check this test.equal(undefined, this.metrics.counter_rates['a']); test.done(); @@ -49,7 +49,7 @@ module.exports = { test.expect(8); this.metrics.timers['a'] = [100]; this.metrics.timer_counters['a'] = 1; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, {flushInterval: 100}, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(0, timer_data.std); test.equal(100, timer_data.upper); @@ -65,7 +65,7 @@ module.exports = { test.expect(8); this.metrics.timers['a'] = [100, 200, 300]; this.metrics.timer_counters['a'] = 3; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, {flushInterval: 100}, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(81.64965809277261, timer_data.std); test.equal(300, timer_data.upper); @@ -82,7 +82,7 @@ module.exports = { this.metrics.timers['a'] = [100]; this.metrics.timer_counters['a'] = 1; this.metrics.pctThreshold = [90]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, {flushInterval: 100}, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(100, timer_data.mean_90); test.equal(100, timer_data.upper_90); @@ -94,7 +94,7 @@ module.exports = { this.metrics.timers['a'] = [100]; this.metrics.timer_counters['a'] = 1; this.metrics.pctThreshold = [90, 80]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, {flushInterval: 100}, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(100, timer_data.mean_90); test.equal(100, timer_data.upper_90); @@ -109,7 +109,7 @@ module.exports = { this.metrics.timers['a'] = [100, 200, 300]; this.metrics.timer_counters['a'] = 3; this.metrics.pctThreshold = [90]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, {flushInterval: 100}, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(200, timer_data.mean_90); test.equal(300, timer_data.upper_90); @@ -121,7 +121,7 @@ module.exports = { this.metrics.timers['a'] = [100, 200, 300]; this.metrics.timer_counters['a'] = 3; this.metrics.pctThreshold = [90, 80]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, {flushInterval: 100}, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(200, timer_data.mean_90); test.equal(300, timer_data.upper_90); @@ -136,7 +136,7 @@ module.exports = { this.metrics.timers['a'] = [100, 200, 300]; this.metrics.timer_counters['a'] = 50; this.metrics.pctThreshold = [90, 80]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, {flushInterval: 100}, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(50, timer_data.count); test.equal(500, timer_data.count_ps); @@ -160,7 +160,7 @@ module.exports = { { metric: 'abcd', bins: [ 1, 5, 'inf'] }, { metric: 'abc', bins: [ 1, 2.21, 'inf'] }, { metric: 'a', bins: [ 1, 2] } ]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, {flushInterval: 100}, function(){}); timer_data = this.metrics.timer_data; // nothing matches the 'abcd' config, so nothing has bin_5 test.equal(undefined, timer_data['a']['histogram']['bin_5']); @@ -190,7 +190,7 @@ module.exports = { test.expect(3); this.metrics.timers['a'] = [100]; this.metrics.pctThreshold = [-10]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, {flushInterval: 100}, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(100, timer_data.mean_top10); test.equal(100, timer_data.lower_top10); @@ -201,7 +201,7 @@ module.exports = { test.expect(3); this.metrics.timers['a'] = [10, 10, 10, 10, 10, 10, 10, 10, 100, 200]; this.metrics.pctThreshold = [-20]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, {flushInterval: 100}, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(150, timer_data.mean_top20); test.equal(100, timer_data.lower_top20); @@ -210,7 +210,7 @@ module.exports = { }, statsd_metrics_exist: function(test) { test.expect(1); - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, {flushInterval: 100}, function(){}); statsd_metrics = this.metrics.statsd_metrics; test.notEqual(undefined, statsd_metrics["processing_time"]); test.done(); @@ -218,7 +218,7 @@ module.exports = { timers_multiple_times_even: function(test) { test.expect(1); this.metrics.timers['a'] = [300, 200, 400, 100]; - pm.process_metrics(this.metrics, 100, this.time_stamp, function(){}); + pm.process_metrics(this.metrics, {flushInterval: 100}, function(){}); timer_data = this.metrics.timer_data['a']; test.equal(250, timer_data.median); test.done(); From c42fe706759d1f6fbe8e3b2867e113bca4f2d6d0 Mon Sep 17 00:00:00 2001 From: Aaron Quint Date: Mon, 10 Feb 2014 16:29:52 -0500 Subject: [PATCH 2/3] Also allow aliasing count and rate for counters --- backends/graphite.js | 10 ++++++++-- exampleConfig.js | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/backends/graphite.js b/backends/graphite.js index f988702a..c3913064 100644 --- a/backends/graphite.js +++ b/backends/graphite.js @@ -33,6 +33,8 @@ var prefixTimer; var prefixGauge; var prefixSet; var globalSuffix; +var aliasCount; +var aliasRate; // set up namespaces var legacyNamespace = true; @@ -108,9 +110,9 @@ var flush_stats = function graphite_flush(ts, metrics) { statString += 'stats_counts.' + key + globalSuffix + value + ts_suffix; } } else { - statString += namespace.concat('rate').join(".") + globalSuffix + valuePerSecond + ts_suffix; + statString += namespace.concat(aliasRate).join(".") + globalSuffix + valuePerSecond + ts_suffix; if (flush_counts) { - statString += namespace.concat('count').join(".") + globalSuffix + value + ts_suffix; + statString += namespace.concat(aliasCount).join(".") + globalSuffix + value + ts_suffix; } } @@ -189,6 +191,8 @@ exports.init = function graphite_init(startup_time, config, events, logger) { prefixSet = config.graphite.prefixSet; globalSuffix = config.graphite.globalSuffix; legacyNamespace = config.graphite.legacyNamespace; + aliasCount = config.graphite.aliasCount; + aliasRate = config.graphite.aliasRate; // set defaults for prefixes & suffix globalPrefix = globalPrefix !== undefined ? globalPrefix : "stats"; @@ -196,6 +200,8 @@ exports.init = function graphite_init(startup_time, config, events, logger) { prefixTimer = prefixTimer !== undefined ? prefixTimer : "timers"; prefixGauge = prefixGauge !== undefined ? prefixGauge : "gauges"; prefixSet = prefixSet !== undefined ? prefixSet : "sets"; + aliasCount = aliasCount !== undefined ? aliasCount : "count"; + aliasRate = aliasRate !== undefined ? aliasRate : "rate"; legacyNamespace = legacyNamespace !== undefined ? legacyNamespace : true; // In order to unconditionally add this string, it either needs to be diff --git a/exampleConfig.js b/exampleConfig.js index 7319491d..923c3f96 100644 --- a/exampleConfig.js +++ b/exampleConfig.js @@ -67,6 +67,7 @@ Optional Variables: mean: [default 'mean'] median: [default 'median'] + console: prettyprint: whether to prettyprint the console backend output [true or false, default: true] @@ -86,6 +87,8 @@ Optional Variables: globalSuffix: global suffix to use for sending stats to graphite [default: ""] This is particularly useful for sending per host stats by settings this value to: require('os').hostname().split('.')[0] + aliasCount: Alias the metric extension for value [default 'count'] + aliasRate: Alias the metric for value per second [default 'rate'] repeater: an array of hashes of the for host: and port: that details other statsd servers to which the received From 0948223228722b6d5f7348aae2c0d015437ba7f6 Mon Sep 17 00:00:00 2001 From: Aaron Quint Date: Tue, 18 Feb 2014 12:55:46 -0500 Subject: [PATCH 3/3] Fix aliases from config --- lib/process_metrics.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/process_metrics.js b/lib/process_metrics.js index f3f54be8..5b90fcd8 100644 --- a/lib/process_metrics.js +++ b/lib/process_metrics.js @@ -17,13 +17,13 @@ var process_metrics = function (metrics, conf, flushCallback) { var metricAliases = { meanPrefix: configAliases.meanPrefix || 'mean_', maxPrefix: configAliases.maxPrefix || 'upper_', - minPrefix: configAliases.maxPrefix || 'lower_', - sumPrefix: configAliases.maxPrefix || 'sum_', + minPrefix: configAliases.minPrefix || 'lower_', + sumPrefix: configAliases.sumPrefix || 'sum_', stddev: configAliases.stddev || 'std', max: configAliases.max || 'upper', min: configAliases.min || 'lower', count: configAliases.count || 'count', - count_ps: configAliases.count || 'count_ps', + count_ps: configAliases.count_ps || 'count_ps', sum: configAliases.sum || 'sum', mean: configAliases.mean || 'mean', median: configAliases.median || 'median'