diff --git a/README.md b/README.md index 3576786..e6dfbe5 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,11 @@ The following overrides the default and any provided namespace or metric name wi secretAccessKey: 'YOUR_SECRET_ACCESS_KEY', region: 'YOUR_REGION', namespace: 'App/Controller/Action', - metricName: 'Request' + metricName: 'Request', + dimensions: [ + { Name: "dim1", Value: "val1" }, + { Name: "dim2", Value: "val2" } + ] } } @@ -81,7 +85,7 @@ Using the option *processKeyForNamespace* (default is false) you can parse the b accessKeyId: 'YOUR_ACCESS_KEY_ID', secretAccessKey: 'YOUR_SECRET_ACCESS_KEY', region: 'YOUR_REGION', - processKeyForNames:true + processKeyForNamespace:true } } @@ -91,6 +95,14 @@ For example, sending StatsD the following is will produce the equivalent to the former configuration example. Note that both will be suppressed if overriden as in the former configuration example. +Dimensions can be specified in the statistic name, using a special __name_value syntaxis. For instance: + + App.Controller.Action.__HostName_server1:1|c + +Will set the "HostName" dimension to the value "server1". + +If a string is provided to the processKeyForNamespace option (instead of the boolean true), that string will be used as a regular expression for splitting the stat name in namespace/dimension components (instead of the default '[\.\-/]'). + ## Tutorial This project was launched with a following [blog post/tutorial](http://blog.simpletask.se/post/aggregating-monitoring-statistics-for-aws-cloudwatch) describing the implementation chain from log4net to Cloudwatch on a Windows system. @@ -101,4 +113,4 @@ Also in the series: [A CloudWatch Appender for log4net](http://blog.simpletask.se/post/awscloudwatch-log4net-appender) -[![endorse](http://api.coderwall.com/camitz/endorsecount.png)](http://coderwall.com/camitz) \ No newline at end of file +[![endorse](http://api.coderwall.com/camitz/endorsecount.png)](http://coderwall.com/camitz) diff --git a/lib/aws-cloudwatch-statsd-backend.js b/lib/aws-cloudwatch-statsd-backend.js index 310695d..535b0ca 100644 --- a/lib/aws-cloudwatch-statsd-backend.js +++ b/lib/aws-cloudwatch-statsd-backend.js @@ -4,158 +4,204 @@ var awssum = require('awssum'); var amazon = awssum.load('amazon/amazon'); var CloudWatch = awssum.load('amazon/cloudwatch').CloudWatch; var fmt = require('fmt'); +var combinations = require('combinations') +var _ = require('underscore') -function CloudwatchBackend(startupTime, config, emitter){ - var self = this; +function CloudwatchBackend(startupTime, config, emitter) { + var self = this; - config.cloudwatch.region = config.cloudwatch.region ? amazon[config.cloudwatch.region] : null; - this.config = config.cloudwatch || {}; + config.cloudwatch.region = config.cloudwatch.region ? amazon[config.cloudwatch.region] : null; + this.config = config.cloudwatch || {}; - // attach - emitter.on('flush', function(timestamp, metrics) { self.flush(timestamp, metrics); }); + // attach + emitter.on('flush', function(timestamp, metrics) { self.flush(timestamp, metrics); }); }; -CloudwatchBackend.prototype.processKey = function(key) { - var parts = key.split(/[\.\/-]/); - return { - metricName: parts[parts.length-1], - namespace: parts.length > 1 ? parts.splice(0, parts.length-1).join("/") : null - }; +CloudwatchBackend.prototype.processKey = function(key, sep_re) { + if (sep_re.constructor === Boolean) + sep_re = "[\.\/-]"; + var parts = key.split(new RegExp(sep_re)); + var metricName = parts[parts.length-1]; + if (parts.slice(0,parts.length-1).length == 0) { + var namespaceParts = null; + var dimensionsParts = null; + } else { + var namespaceParts = parts.slice(0,parts.length-1).filter(function(p){ return /^__/.exec(p) == null; }); + // dimension naming parts start with "__", key and value are separated by "_" + var dimensionParts = parts.slice(0,parts.length-1).filter(function(p){ return /^__/.exec(p) != null; }); + dimensionParts = dimensionParts.map(function(p){ return p.slice(2); }); + } + return { + metricName: metricName, + namespace: namespaceParts ? namespaceParts.join("/") : null, + dimensionMap: dimensionParts ? dimensionParts.map(function(x) { pp=x.split(/_/); return { Name:pp[0], Value:pp[1] } }) : null + }; } -CloudwatchBackend.prototype.flush = function(timestamp, metrics) { +CloudwatchBackend.prototype.prepareCWData = function(key, data) { + + var names = this.config.processKeyForNamespace ? this.processKey(key, this.config.processKeyForNamespace) : {}; + var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; + var metricName = this.config.metricName || names.metricName || key; + + var dimensionMap = this.config.dimensions || null; + if (names.dimensionMap != null) { + dimensionMap = names.dimensionMap.concat(dimensionMap != null ? dimensionMap : []); + } + + data.Namespace = namespace; + data.MetricData.forEach(function(md) { + md.MetricName = metricName; + if (dimensionMap != null) { + md.Dimensions = dimensionMap; + } + }); + + /* ---- dimension slicing support disabled + newMetricData = [] + data.Namespace = namespace; + data.MetricData.forEach(function(md) { + /* Push with no dimensions * + md.MetricName = metricName; + newMetricData.push(md); + if (dimensionMap != null) { + /* Reproduce the metric for each possible dimension combination * + combinations(Object.keys(dimensionMap)).forEach(function(combo) { + console.log("Combo: %s", combo); + newmd = _.clone(md); + newmd.Dimensions = combo.map(function(i) { return dimensionMap[i]; }) + newMetricData.push(newmd); + }); + } + }); + data.MetricData = newMetricData; + */ + + return data; - var cloudwatch = new CloudWatch(this.config); - -console.log(new Date(timestamp*1000).toISOString()); - - - var counters = metrics.counters; - var gauges = metrics.gauges; - var timers = metrics.timers; - var sets = metrics.sets; - - for (key in counters) { - if (key.indexOf('statsd.') == 0) - continue; - - names = this.config.processKeyForNamespace ? this.processKey(key) : {}; - var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; - var metricName = this.config.metricName || names.metricName || key; - - cloudwatch.PutMetricData({ - MetricData : [{ - MetricName : metricName, - Unit : 'Count', - Timestamp: new Date(timestamp*1000).toISOString(), - Value : counters[key] - }], - Namespace : namespace - }, - function(err, data) { - fmt.dump(err, 'Err'); - fmt.dump(data, 'Data'); - }); - } - - for (key in timers) { - if (timers[key].length > 0) { - var values = timers[key].sort(function (a,b) { return a-b; }); - var count = values.length; - var min = values[0]; - var max = values[count - 1]; - - var cumulativeValues = [min]; - for (var i = 1; i < count; i++) { - cumulativeValues.push(values[i] + cumulativeValues[i-1]); - } - - var sum = min; - var mean = min; - var maxAtThreshold = max; - - var message = ""; - - var key2; - - sum = cumulativeValues[count-1]; - mean = sum / count; - - names = this.config.processKeyForNamespace ? this.processKey(key) : {}; - var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; - var metricName = this.config.metricName || names.metricName || key; - - cloudwatch.PutMetricData({ - MetricData : [{ - MetricName : metricName, - Unit : 'Milliseconds', - Timestamp: new Date(timestamp*1000).toISOString(), - StatisticValues: { - Minimum: min, - Maximum: max, - Sum: sum, - SampleCount: count - } - }], - Namespace : namespace - }, - function(err, data) { - fmt.dump(err, 'Err'); - fmt.dump(data, 'Data'); - }); - - } - } - - for (key in gauges) { - names = this.config.processKeyForNamespace ? this.processKey(key) : {}; - var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; - var metricName = this.config.metricName || names.metricName || key; - - cloudwatch.PutMetricData({ - MetricData : [{ - MetricName : metricName, - Unit : 'None', - Timestamp: new Date(timestamp*1000).toISOString(), - Value : gauges[key] - }], - Namespace : namespace - }, - function(err, data) { - fmt.dump(err, 'Err'); - fmt.dump(data, 'Data'); - }); - } - - for (key in sets) { - names = this.config.processKeyForNamespace ? this.processKey(key) : {}; - var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; - var metricName = this.config.metricName || names.metricName || key; - - cloudwatch.PutMetricData({ - MetricData : [{ - MetricName : metricName, - Unit : 'None', - Timestamp: new Date(timestamp*1000).toISOString(), - Value : sets[key].values().length - }], - Namespace : namespace - }, - function(err, data) { - fmt.dump(err, 'Err'); - fmt.dump(data, 'Data'); - }); - - statString += 'stats.sets.' + key + '.count ' + sets[key].values().length + ' ' + ts + "\n"; - numStats += 1; - } - +} -}; +CloudwatchBackend.prototype._addMetricData = function(packets, data) { + + if (!(data.Namespace in packets)) { + packets[data.Namespace] = [{ "Namespace": data.Namespace, "MetricData": [] }]; + } + last_packet = packets[data.Namespace].slice(-1)[0]; + for (var i = 0; i < data.MetricData.length; i++) { + /* Keep packets from getting too big, awssum is not doing any smart handling of limits */ + if (last_packet["MetricData"].length >= 8) { + last_packet = { "Namespace": data.Namespace, "MetricData": [] }; + packets[data.Namespace].push(last_packet); + } + last_packet.MetricData.push(data.MetricData[i]); + } + +} +CloudwatchBackend.prototype.flush = function(timestamp, metrics) { + var cloudwatch = new CloudWatch(this.config); + + var counters = metrics.counters; + var gauges = metrics.gauges; + var timers = metrics.timers; + var sets = metrics.sets; + + var cwPackets = {} + + for (key in counters) { + if (key.indexOf('statsd.') == 0) + continue; + md = this.prepareCWData(key, { + MetricData : [{ + Unit : 'Count', + Timestamp: new Date(timestamp*1000).toISOString(), + Value : counters[key] + }] + }); + this._addMetricData(cwPackets, md); + } + + for (key in timers) { + if (timers[key].length > 0) { + var values = timers[key].sort(function (a,b) { return a-b; }); + var count = values.length; + var min = values[0]; + var max = values[count - 1]; + + var cumulativeValues = [min]; + for (var i = 1; i < count; i++) { + cumulativeValues.push(values[i] + cumulativeValues[i-1]); + } + + var sum = min; + var mean = min; + var maxAtThreshold = max; + + var message = ""; + + var key2; + + sum = cumulativeValues[count-1]; + mean = sum / count; + + md = this.prepareCWData(key, { + MetricData : [{ + Unit : 'Milliseconds', + Timestamp: new Date(timestamp*1000).toISOString(), + StatisticValues: { + Minimum: min, + Maximum: max, + Sum: sum, + SampleCount: count + } + }] + }); + this._addMetricData(cwPackets, md); + } + } + + for (key in gauges) { + md = this.prepareCWData(key, { + MetricData : [{ + Unit : 'None', + Timestamp: new Date(timestamp*1000).toISOString(), + Value : gauges[key] + }] + }); + this._addMetricData(cwPackets, md); + } + + for (key in sets) { + md = this.prepareCWData(key, { + MetricData : [{ + Unit : 'None', + Timestamp: new Date(timestamp*1000).toISOString(), + Value : sets[key].values().length + }] + }); + this._addMetricData(cwPackets, md); + statString += 'stats.sets.' + key + '.count ' + sets[key].values().length + ' ' + ts + "\n"; + numStats += 1; + } + + /* Do the actual sending of data */ + for (ns in cwPackets) { + for (var i = 0; i < cwPackets[ns].length; i++) { + cloudwatch.PutMetricData( + cwPackets[ns][i], + function(err, data) { + fmt.dump(err, 'Err'); + fmt.dump(data, 'Data'); + } + ); + } + } + +}; exports.init = function(startupTime, config, events) { - var instance = new CloudwatchBackend(startupTime, config, events); - return true; + var instance = new CloudwatchBackend(startupTime, config, events); + return true; }; + diff --git a/package.json b/package.json index 1626408..0c485c1 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,9 @@ "main": "lib/aws-cloudwatch-statsd-backend.js", "dependencies": { "fmt": "~0.4", - "awssum": "~0.10" + "awssum": "~0.10", + "underscore": "1.4.3", + "combinations": "0.1.0" }, "devDependencies": {}, "optionalDependencies": {},