From 608ab631745b8ea8d57a4fba6bbaf4a2b11c4046 Mon Sep 17 00:00:00 2001 From: David Losada Date: Wed, 19 Dec 2012 18:57:42 +0400 Subject: [PATCH 1/6] Cloudwatch dimensions support. Dimensions are stat naming components of the form __key_value --- lib/aws-cloudwatch-statsd-backend.js | 136 +++++++++++++++------------ 1 file changed, 74 insertions(+), 62 deletions(-) diff --git a/lib/aws-cloudwatch-statsd-backend.js b/lib/aws-cloudwatch-statsd-backend.js index 310695d..a986f44 100644 --- a/lib/aws-cloudwatch-statsd-backend.js +++ b/lib/aws-cloudwatch-statsd-backend.js @@ -15,12 +15,43 @@ function CloudwatchBackend(startupTime, config, emitter){ emitter.on('flush', function(timestamp, metrics) { self.flush(timestamp, metrics); }); }; -CloudwatchBackend.prototype.processKey = function(key) { - var parts = key.split(/[\.\/-]/); - return { - metricName: parts[parts.length-1], - namespace: parts.length > 1 ? parts.splice(0, parts.length-1).join("/") : null - }; +CloudwatchBackend.prototype.processKey = function(key, sep_re) { + if (sep_re.constructor === Boolean) + sep_re = "[\.\/-]"; + var parts = key.split(new RegExp(sep_re)); + var metricName = parts[parts.length-1]; + if (parts.slice(0,parts.length-1).length == 0) { + var namespaceParts = null; + var dimensionsParts = null; + } else { + var namespaceParts = parts.slice(0,parts.length-1).filter(function(p){ return /^__/.exec(p) == null; }); + // dimension naming parts start with "__", key and value are separated by "_" + var dimensionParts = parts.slice(0,parts.length-1).filter(function(p){ return /^__/.exec(p) != null; }); + dimensionParts = dimensionParts.map(function(p){ return p.slice(2); }); + } + return { + metricName: metricName, + namespace: namespaceParts ? namespaceParts.join("/") : null, + dimensionMap: dimensionParts ? dimensionParts.map(function(x) { pp=x.split(/_/); return { Name:pp[0], Value:pp[1] } }) : null + }; +} + +CloudwatchBackend.prototype.prepareCWData = function(key, data) { + + var names = this.config.processKeyForNamespace ? this.processKey(key, this.config.processKeyForNamespace) : {}; + var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; + var metricName = this.config.metricName || names.metricName || key; + + data.Namespace = namespace; + data.MetricData.forEach(function(md) { + md.MetricName = metricName + if (names.dimensionMap) { + data.Dimension = names.dimensionMap; + } + }); + + return data; + } CloudwatchBackend.prototype.flush = function(timestamp, metrics) { @@ -29,7 +60,6 @@ CloudwatchBackend.prototype.flush = function(timestamp, metrics) { console.log(new Date(timestamp*1000).toISOString()); - var counters = metrics.counters; var gauges = metrics.gauges; var timers = metrics.timers; @@ -38,24 +68,20 @@ console.log(new Date(timestamp*1000).toISOString()); for (key in counters) { if (key.indexOf('statsd.') == 0) continue; - - names = this.config.processKeyForNamespace ? this.processKey(key) : {}; - var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; - var metricName = this.config.metricName || names.metricName || key; - - cloudwatch.PutMetricData({ - MetricData : [{ - MetricName : metricName, - Unit : 'Count', - Timestamp: new Date(timestamp*1000).toISOString(), - Value : counters[key] - }], - Namespace : namespace - }, - function(err, data) { + + cloudwatch.PutMetricData(this.prepareCWData(key, + { + MetricData : [{ + Unit : 'Count', + Timestamp: new Date(timestamp*1000).toISOString(), + Value : counters[key] + }] + }), + function(err, data) { fmt.dump(err, 'Err'); fmt.dump(data, 'Data'); - }); + } + ); } for (key in timers) { @@ -81,24 +107,19 @@ console.log(new Date(timestamp*1000).toISOString()); sum = cumulativeValues[count-1]; mean = sum / count; - names = this.config.processKeyForNamespace ? this.processKey(key) : {}; - var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; - var metricName = this.config.metricName || names.metricName || key; - - cloudwatch.PutMetricData({ - MetricData : [{ - MetricName : metricName, - Unit : 'Milliseconds', - Timestamp: new Date(timestamp*1000).toISOString(), - StatisticValues: { + cloudwatch.PutMetricData(this.prepareCWData(key, + { + MetricData : [{ + Unit : 'Milliseconds', + Timestamp: new Date(timestamp*1000).toISOString(), + StatisticValues: { Minimum: min, Maximum: max, Sum: sum, SampleCount: count } - }], - Namespace : namespace - }, + }] + }), function(err, data) { fmt.dump(err, 'Err'); fmt.dump(data, 'Data'); @@ -108,19 +129,14 @@ console.log(new Date(timestamp*1000).toISOString()); } for (key in gauges) { - names = this.config.processKeyForNamespace ? this.processKey(key) : {}; - var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; - var metricName = this.config.metricName || names.metricName || key; - - cloudwatch.PutMetricData({ - MetricData : [{ - MetricName : metricName, - Unit : 'None', - Timestamp: new Date(timestamp*1000).toISOString(), - Value : gauges[key] - }], - Namespace : namespace - }, + cloudwatch.PutMetricData(this.prepareCWData(key, + { + MetricData : [{ + Unit : 'None', + Timestamp: new Date(timestamp*1000).toISOString(), + Value : gauges[key] + }] + }), function(err, data) { fmt.dump(err, 'Err'); fmt.dump(data, 'Data'); @@ -128,19 +144,14 @@ console.log(new Date(timestamp*1000).toISOString()); } for (key in sets) { - names = this.config.processKeyForNamespace ? this.processKey(key) : {}; - var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; - var metricName = this.config.metricName || names.metricName || key; - - cloudwatch.PutMetricData({ - MetricData : [{ - MetricName : metricName, - Unit : 'None', - Timestamp: new Date(timestamp*1000).toISOString(), - Value : sets[key].values().length - }], - Namespace : namespace - }, + cloudwatch.PutMetricData(this.prepareCWData(key, + { + MetricData : [{ + Unit : 'None', + Timestamp: new Date(timestamp*1000).toISOString(), + Value : sets[key].values().length + }] + }), function(err, data) { fmt.dump(err, 'Err'); fmt.dump(data, 'Data'); @@ -159,3 +170,4 @@ exports.init = function(startupTime, config, events) { var instance = new CloudwatchBackend(startupTime, config, events); return true; }; + From 966227cbea2e70bcbd95198f209061bab2e574d4 Mon Sep 17 00:00:00 2001 From: David Losada Date: Wed, 19 Dec 2012 20:16:27 +0400 Subject: [PATCH 2/6] Correct mistakes and add doco --- README.md | 18 +++++++++++++++--- lib/aws-cloudwatch-statsd-backend.js | 15 +++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 3576786..e6dfbe5 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,11 @@ The following overrides the default and any provided namespace or metric name wi secretAccessKey: 'YOUR_SECRET_ACCESS_KEY', region: 'YOUR_REGION', namespace: 'App/Controller/Action', - metricName: 'Request' + metricName: 'Request', + dimensions: [ + { Name: "dim1", Value: "val1" }, + { Name: "dim2", Value: "val2" } + ] } } @@ -81,7 +85,7 @@ Using the option *processKeyForNamespace* (default is false) you can parse the b accessKeyId: 'YOUR_ACCESS_KEY_ID', secretAccessKey: 'YOUR_SECRET_ACCESS_KEY', region: 'YOUR_REGION', - processKeyForNames:true + processKeyForNamespace:true } } @@ -91,6 +95,14 @@ For example, sending StatsD the following is will produce the equivalent to the former configuration example. Note that both will be suppressed if overriden as in the former configuration example. +Dimensions can be specified in the statistic name, using a special __name_value syntaxis. For instance: + + App.Controller.Action.__HostName_server1:1|c + +Will set the "HostName" dimension to the value "server1". + +If a string is provided to the processKeyForNamespace option (instead of the boolean true), that string will be used as a regular expression for splitting the stat name in namespace/dimension components (instead of the default '[\.\-/]'). + ## Tutorial This project was launched with a following [blog post/tutorial](http://blog.simpletask.se/post/aggregating-monitoring-statistics-for-aws-cloudwatch) describing the implementation chain from log4net to Cloudwatch on a Windows system. @@ -101,4 +113,4 @@ Also in the series: [A CloudWatch Appender for log4net](http://blog.simpletask.se/post/awscloudwatch-log4net-appender) -[![endorse](http://api.coderwall.com/camitz/endorsecount.png)](http://coderwall.com/camitz) \ No newline at end of file +[![endorse](http://api.coderwall.com/camitz/endorsecount.png)](http://coderwall.com/camitz) diff --git a/lib/aws-cloudwatch-statsd-backend.js b/lib/aws-cloudwatch-statsd-backend.js index a986f44..4e428a5 100644 --- a/lib/aws-cloudwatch-statsd-backend.js +++ b/lib/aws-cloudwatch-statsd-backend.js @@ -19,7 +19,7 @@ CloudwatchBackend.prototype.processKey = function(key, sep_re) { if (sep_re.constructor === Boolean) sep_re = "[\.\/-]"; var parts = key.split(new RegExp(sep_re)); - var metricName = parts[parts.length-1]; + var metricName = parts[parts.length-1]; if (parts.slice(0,parts.length-1).length == 0) { var namespaceParts = null; var dimensionsParts = null; @@ -42,14 +42,21 @@ CloudwatchBackend.prototype.prepareCWData = function(key, data) { var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; var metricName = this.config.metricName || names.metricName || key; + var dimensionMap = this.config.dimensions || null; + if (names.dimensionMap != null) { + dimensionMap = names.dimensionMap.concat(dimensionMap != null ? dimensionMap : []); + } + data.Namespace = namespace; data.MetricData.forEach(function(md) { - md.MetricName = metricName - if (names.dimensionMap) { - data.Dimension = names.dimensionMap; + md.MetricName = metricName; + if (dimensionMap != null) { + md.Dimensions = dimensionMap; } }); + console.log("this goes out: %j", data); + return data; } From b4fc69953f5ccf43a0db626e8ffaecd1e51f14bd Mon Sep 17 00:00:00 2001 From: David Losada Date: Sun, 20 Jan 2013 20:00:06 +0100 Subject: [PATCH 3/6] Dimension slicing support --- lib/aws-cloudwatch-statsd-backend.js | 16 +++++++++++++--- package.json | 4 +++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/aws-cloudwatch-statsd-backend.js b/lib/aws-cloudwatch-statsd-backend.js index 4e428a5..8687bc5 100644 --- a/lib/aws-cloudwatch-statsd-backend.js +++ b/lib/aws-cloudwatch-statsd-backend.js @@ -4,6 +4,8 @@ var awssum = require('awssum'); var amazon = awssum.load('amazon/amazon'); var CloudWatch = awssum.load('amazon/cloudwatch').CloudWatch; var fmt = require('fmt'); +var combinations = require('combinations') +var _ = require('underscore') function CloudwatchBackend(startupTime, config, emitter){ var self = this; @@ -47,15 +49,23 @@ CloudwatchBackend.prototype.prepareCWData = function(key, data) { dimensionMap = names.dimensionMap.concat(dimensionMap != null ? dimensionMap : []); } + newMetricData = [] data.Namespace = namespace; data.MetricData.forEach(function(md) { + /* Push with no dimensions */ md.MetricName = metricName; + newMetricData.push(md); if (dimensionMap != null) { - md.Dimensions = dimensionMap; + /* Reproduce the metric for each possible dimension combination */ + combinations(Object.keys(dimensionMap)).forEach(function(combo) { + console.log("Combo: %s", combo); + newmd = _.clone(md); + newmd.Dimensions = combo.map(function(i) { return dimensionMap[i]; }) + newMetricData.push(newmd); + }); } }); - - console.log("this goes out: %j", data); + data.MetricData = newMetricData; return data; diff --git a/package.json b/package.json index 1626408..0c485c1 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,9 @@ "main": "lib/aws-cloudwatch-statsd-backend.js", "dependencies": { "fmt": "~0.4", - "awssum": "~0.10" + "awssum": "~0.10", + "underscore": "1.4.3", + "combinations": "0.1.0" }, "devDependencies": {}, "optionalDependencies": {}, From 52fd17d9d3185a46363c182caef6df2695284571 Mon Sep 17 00:00:00 2001 From: David Losada Date: Sat, 23 Feb 2013 21:12:02 +0100 Subject: [PATCH 4/6] Disable dimension slicing --- lib/aws-cloudwatch-statsd-backend.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/aws-cloudwatch-statsd-backend.js b/lib/aws-cloudwatch-statsd-backend.js index 8687bc5..42c67bb 100644 --- a/lib/aws-cloudwatch-statsd-backend.js +++ b/lib/aws-cloudwatch-statsd-backend.js @@ -49,14 +49,23 @@ CloudwatchBackend.prototype.prepareCWData = function(key, data) { dimensionMap = names.dimensionMap.concat(dimensionMap != null ? dimensionMap : []); } + data.Namespace = namespace; + data.MetricData.forEach(function(md) { + md.MetricName = metricName; + if (dimensionMap != null) { + md.Dimensions = dimensionMap; + } + }); + + /* ---- dimension slicing support disabled newMetricData = [] data.Namespace = namespace; data.MetricData.forEach(function(md) { - /* Push with no dimensions */ + /* Push with no dimensions * md.MetricName = metricName; newMetricData.push(md); if (dimensionMap != null) { - /* Reproduce the metric for each possible dimension combination */ + /* Reproduce the metric for each possible dimension combination * combinations(Object.keys(dimensionMap)).forEach(function(combo) { console.log("Combo: %s", combo); newmd = _.clone(md); @@ -66,6 +75,7 @@ CloudwatchBackend.prototype.prepareCWData = function(key, data) { } }); data.MetricData = newMetricData; + */ return data; From bb9fd5b03c11f62b635035bd49d1b74fd9071cd0 Mon Sep 17 00:00:00 2001 From: David Losada Date: Thu, 28 Feb 2013 13:14:08 +0100 Subject: [PATCH 5/6] Batch CW metrics for cost savings --- lib/aws-cloudwatch-statsd-backend.js | 215 ++++++++++++++------------- 1 file changed, 111 insertions(+), 104 deletions(-) diff --git a/lib/aws-cloudwatch-statsd-backend.js b/lib/aws-cloudwatch-statsd-backend.js index 42c67bb..dd02ad6 100644 --- a/lib/aws-cloudwatch-statsd-backend.js +++ b/lib/aws-cloudwatch-statsd-backend.js @@ -7,14 +7,14 @@ var fmt = require('fmt'); var combinations = require('combinations') var _ = require('underscore') -function CloudwatchBackend(startupTime, config, emitter){ - var self = this; +function CloudwatchBackend(startupTime, config, emitter) { + var self = this; - config.cloudwatch.region = config.cloudwatch.region ? amazon[config.cloudwatch.region] : null; - this.config = config.cloudwatch || {}; + config.cloudwatch.region = config.cloudwatch.region ? amazon[config.cloudwatch.region] : null; + this.config = config.cloudwatch || {}; - // attach - emitter.on('flush', function(timestamp, metrics) { self.flush(timestamp, metrics); }); + // attach + emitter.on('flush', function(timestamp, metrics) { self.flush(timestamp, metrics); }); }; CloudwatchBackend.prototype.processKey = function(key, sep_re) { @@ -81,120 +81,127 @@ CloudwatchBackend.prototype.prepareCWData = function(key, data) { } +CloudwatchBackend.prototype._addMetricData = function(packets, data) { + + if (!data.Namespace in packets) { + packets[data.Namespace] = [{ "Namespace": data.Namespace, "MetricData": [] }]; + } + last_packet = packets[data.Namespace].slice(-1)[0]; + for (var i = 0; i < data.MetricData.length; i++) { + /* Keep packets from getting too big, awssum is not doing any smart handling of limits */ + if (last_packet["MetricData"].length >= 8) { + last_packet = { "Namespace": data.Namespace, "MetricData": [] }; + packets[data.Namespace].push(last_packet); + } + last_packet.MetricData.push(data.MetricData[i]); + } + +} + CloudwatchBackend.prototype.flush = function(timestamp, metrics) { - var cloudwatch = new CloudWatch(this.config); - -console.log(new Date(timestamp*1000).toISOString()); - - var counters = metrics.counters; - var gauges = metrics.gauges; - var timers = metrics.timers; - var sets = metrics.sets; - - for (key in counters) { - if (key.indexOf('statsd.') == 0) - continue; - - cloudwatch.PutMetricData(this.prepareCWData(key, - { - MetricData : [{ + var cloudwatch = new CloudWatch(this.config); + + var counters = metrics.counters; + var gauges = metrics.gauges; + var timers = metrics.timers; + var sets = metrics.sets; + + var cwPackets = {} + + for (key in counters) { + if (key.indexOf('statsd.') == 0) + continue; + md = this.prepareCWData(key, { + MetricData : [{ Unit : 'Count', - Timestamp: new Date(timestamp*1000).toISOString(), + Timestamp: new Date(timestamp*1000).toISOString(), Value : counters[key] }] - }), - function(err, data) { - fmt.dump(err, 'Err'); - fmt.dump(data, 'Data'); - } - ); - } - - for (key in timers) { - if (timers[key].length > 0) { - var values = timers[key].sort(function (a,b) { return a-b; }); - var count = values.length; - var min = values[0]; - var max = values[count - 1]; - - var cumulativeValues = [min]; - for (var i = 1; i < count; i++) { - cumulativeValues.push(values[i] + cumulativeValues[i-1]); - } - - var sum = min; - var mean = min; - var maxAtThreshold = max; - - var message = ""; - - var key2; - - sum = cumulativeValues[count-1]; - mean = sum / count; - - cloudwatch.PutMetricData(this.prepareCWData(key, - { - MetricData : [{ - Unit : 'Milliseconds', - Timestamp: new Date(timestamp*1000).toISOString(), - StatisticValues: { - Minimum: min, - Maximum: max, - Sum: sum, - SampleCount: count - } - }] - }), - function(err, data) { - fmt.dump(err, 'Err'); - fmt.dump(data, 'Data'); - }); + }); + this._addMetricData(cwPackets, md); + } - } - } + for (key in timers) { + if (timers[key].length > 0) { + var values = timers[key].sort(function (a,b) { return a-b; }); + var count = values.length; + var min = values[0]; + var max = values[count - 1]; + + var cumulativeValues = [min]; + for (var i = 1; i < count; i++) { + cumulativeValues.push(values[i] + cumulativeValues[i-1]); + } + + var sum = min; + var mean = min; + var maxAtThreshold = max; + + var message = ""; + + var key2; + + sum = cumulativeValues[count-1]; + mean = sum / count; + + md = this.prepareCWData(key, { + MetricData : [{ + Unit : 'Milliseconds', + Timestamp: new Date(timestamp*1000).toISOString(), + StatisticValues: { + Minimum: min, + Maximum: max, + Sum: sum, + SampleCount: count + } + }] + }); + this._addMetricData(cwPackets, md); + } + } - for (key in gauges) { - cloudwatch.PutMetricData(this.prepareCWData(key, - { - MetricData : [{ + for (key in gauges) { + md = this.prepareCWData(key, { + MetricData : [{ Unit : 'None', - Timestamp: new Date(timestamp*1000).toISOString(), + Timestamp: new Date(timestamp*1000).toISOString(), Value : gauges[key] }] - }), - function(err, data) { - fmt.dump(err, 'Err'); - fmt.dump(data, 'Data'); - }); - } - - for (key in sets) { - cloudwatch.PutMetricData(this.prepareCWData(key, - { - MetricData : [{ + }); + this._addMetricData(cwPackets, md); + } + + for (key in sets) { + md = this.prepareCWData(key, { + MetricData : [{ Unit : 'None', - Timestamp: new Date(timestamp*1000).toISOString(), + Timestamp: new Date(timestamp*1000).toISOString(), Value : sets[key].values().length }] - }), - function(err, data) { - fmt.dump(err, 'Err'); - fmt.dump(data, 'Data'); - }); - - statString += 'stats.sets.' + key + '.count ' + sets[key].values().length + ' ' + ts + "\n"; - numStats += 1; - } - + }); + this._addMetricData(cwPackets, md); + statString += 'stats.sets.' + key + '.count ' + sets[key].values().length + ' ' + ts + "\n"; + numStats += 1; + } + /* Do the actual sending of data */ + for (ns in cwPackets) { + for (var i = 0; i < cwpackets[ns].length; i++) { + cloudwatch.PutMetricData( + cwPackets[ns][i], + function(err, data) { + fmt.dump(err, 'Err'); + fmt.dump(data, 'Data'); + } + ); + } + } + }; - - exports.init = function(startupTime, config, events) { - var instance = new CloudwatchBackend(startupTime, config, events); - return true; + var instance = new CloudwatchBackend(startupTime, config, events); + return true; }; From dd99ffa9e258a2b89c7a9da9d912b9edbdb5215e Mon Sep 17 00:00:00 2001 From: David Losada Date: Thu, 28 Feb 2013 16:09:37 +0100 Subject: [PATCH 6/6] Bug fixes --- lib/aws-cloudwatch-statsd-backend.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/aws-cloudwatch-statsd-backend.js b/lib/aws-cloudwatch-statsd-backend.js index dd02ad6..535b0ca 100644 --- a/lib/aws-cloudwatch-statsd-backend.js +++ b/lib/aws-cloudwatch-statsd-backend.js @@ -40,14 +40,14 @@ CloudwatchBackend.prototype.processKey = function(key, sep_re) { CloudwatchBackend.prototype.prepareCWData = function(key, data) { - var names = this.config.processKeyForNamespace ? this.processKey(key, this.config.processKeyForNamespace) : {}; - var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; - var metricName = this.config.metricName || names.metricName || key; + var names = this.config.processKeyForNamespace ? this.processKey(key, this.config.processKeyForNamespace) : {}; + var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; + var metricName = this.config.metricName || names.metricName || key; - var dimensionMap = this.config.dimensions || null; - if (names.dimensionMap != null) { - dimensionMap = names.dimensionMap.concat(dimensionMap != null ? dimensionMap : []); - } + var dimensionMap = this.config.dimensions || null; + if (names.dimensionMap != null) { + dimensionMap = names.dimensionMap.concat(dimensionMap != null ? dimensionMap : []); + } data.Namespace = namespace; data.MetricData.forEach(function(md) { @@ -83,7 +83,7 @@ CloudwatchBackend.prototype.prepareCWData = function(key, data) { CloudwatchBackend.prototype._addMetricData = function(packets, data) { - if (!data.Namespace in packets) { + if (!(data.Namespace in packets)) { packets[data.Namespace] = [{ "Namespace": data.Namespace, "MetricData": [] }]; } last_packet = packets[data.Namespace].slice(-1)[0]; @@ -187,7 +187,7 @@ CloudwatchBackend.prototype.flush = function(timestamp, metrics) { /* Do the actual sending of data */ for (ns in cwPackets) { - for (var i = 0; i < cwpackets[ns].length; i++) { + for (var i = 0; i < cwPackets[ns].length; i++) { cloudwatch.PutMetricData( cwPackets[ns][i], function(err, data) {