Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support for Cloudwatch dimensions #1

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Support for Cloudwatch dimensions #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
608ab63
Cloudwatch dimensions support. Dimensions are stat naming components …
tuxpiper Dec 19, 2012
966227c
Correct mistakes and add doco
tuxpiper Dec 19, 2012
b4fc699
Dimension slicing support
tuxpiper Jan 20, 2013
52fd17d
Disable dimension slicing
tuxpiper Feb 23, 2013
bb9fd5b
Batch CW metrics for cost savings
tuxpiper Feb 28, 2013
dd99ffa
Bug fixes
tuxpiper Feb 28, 2013
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,11 @@ The following overrides the default and any provided namespace or metric name wi
secretAccessKey: 'YOUR_SECRET_ACCESS_KEY',
region: 'YOUR_REGION',
namespace: 'App/Controller/Action',
metricName: 'Request'
metricName: 'Request',
dimensions: [
{ Name: "dim1", Value: "val1" },
{ Name: "dim2", Value: "val2" }
]
}
}

Expand All @@ -81,7 +85,7 @@ Using the option *processKeyForNamespace* (default is false) you can parse the b
accessKeyId: 'YOUR_ACCESS_KEY_ID',
secretAccessKey: 'YOUR_SECRET_ACCESS_KEY',
region: 'YOUR_REGION',
processKeyForNames:true
processKeyForNamespace:true
}
}

Expand All @@ -91,6 +95,14 @@ For example, sending StatsD the following

is will produce the equivalent to the former configuration example. Note that both will be suppressed if overriden as in the former configuration example.

Dimensions can be specified in the statistic name, using a special __name_value syntaxis. For instance:

App.Controller.Action.__HostName_server1:1|c

Will set the "HostName" dimension to the value "server1".

If a string is provided to the processKeyForNamespace option (instead of the boolean true), that string will be used as a regular expression for splitting the stat name in namespace/dimension components (instead of the default '[\.\-/]').

## Tutorial

This project was launched with a following [blog post/tutorial](http://blog.simpletask.se/post/aggregating-monitoring-statistics-for-aws-cloudwatch) describing the implementation chain from log4net to Cloudwatch on a Windows system.
Expand All @@ -101,4 +113,4 @@ Also in the series:

[A CloudWatch Appender for log4net](http://blog.simpletask.se/post/awscloudwatch-log4net-appender)

[![endorse](http://api.coderwall.com/camitz/endorsecount.png)](http://coderwall.com/camitz)
[![endorse](http://api.coderwall.com/camitz/endorsecount.png)](http://coderwall.com/camitz)
328 changes: 187 additions & 141 deletions lib/aws-cloudwatch-statsd-backend.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,158 +4,204 @@ var awssum = require('awssum');
var amazon = awssum.load('amazon/amazon');
var CloudWatch = awssum.load('amazon/cloudwatch').CloudWatch;
var fmt = require('fmt');
var combinations = require('combinations')
var _ = require('underscore')

function CloudwatchBackend(startupTime, config, emitter){
var self = this;
function CloudwatchBackend(startupTime, config, emitter) {
var self = this;

config.cloudwatch.region = config.cloudwatch.region ? amazon[config.cloudwatch.region] : null;
this.config = config.cloudwatch || {};
config.cloudwatch.region = config.cloudwatch.region ? amazon[config.cloudwatch.region] : null;
this.config = config.cloudwatch || {};

// attach
emitter.on('flush', function(timestamp, metrics) { self.flush(timestamp, metrics); });
// attach
emitter.on('flush', function(timestamp, metrics) { self.flush(timestamp, metrics); });
};

CloudwatchBackend.prototype.processKey = function(key) {
var parts = key.split(/[\.\/-]/);
return {
metricName: parts[parts.length-1],
namespace: parts.length > 1 ? parts.splice(0, parts.length-1).join("/") : null
};
CloudwatchBackend.prototype.processKey = function(key, sep_re) {
if (sep_re.constructor === Boolean)
sep_re = "[\.\/-]";
var parts = key.split(new RegExp(sep_re));
var metricName = parts[parts.length-1];
if (parts.slice(0,parts.length-1).length == 0) {
var namespaceParts = null;
var dimensionsParts = null;
} else {
var namespaceParts = parts.slice(0,parts.length-1).filter(function(p){ return /^__/.exec(p) == null; });
// dimension naming parts start with "__", key and value are separated by "_"
var dimensionParts = parts.slice(0,parts.length-1).filter(function(p){ return /^__/.exec(p) != null; });
dimensionParts = dimensionParts.map(function(p){ return p.slice(2); });
}
return {
metricName: metricName,
namespace: namespaceParts ? namespaceParts.join("/") : null,
dimensionMap: dimensionParts ? dimensionParts.map(function(x) { pp=x.split(/_/); return { Name:pp[0], Value:pp[1] } }) : null
};
}

CloudwatchBackend.prototype.flush = function(timestamp, metrics) {
CloudwatchBackend.prototype.prepareCWData = function(key, data) {

var names = this.config.processKeyForNamespace ? this.processKey(key, this.config.processKeyForNamespace) : {};
var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend";
var metricName = this.config.metricName || names.metricName || key;

var dimensionMap = this.config.dimensions || null;
if (names.dimensionMap != null) {
dimensionMap = names.dimensionMap.concat(dimensionMap != null ? dimensionMap : []);
}

data.Namespace = namespace;
data.MetricData.forEach(function(md) {
md.MetricName = metricName;
if (dimensionMap != null) {
md.Dimensions = dimensionMap;
}
});

/* ---- dimension slicing support disabled
newMetricData = []
data.Namespace = namespace;
data.MetricData.forEach(function(md) {
/* Push with no dimensions *
md.MetricName = metricName;
newMetricData.push(md);
if (dimensionMap != null) {
/* Reproduce the metric for each possible dimension combination *
combinations(Object.keys(dimensionMap)).forEach(function(combo) {
console.log("Combo: %s", combo);
newmd = _.clone(md);
newmd.Dimensions = combo.map(function(i) { return dimensionMap[i]; })
newMetricData.push(newmd);
});
}
});
data.MetricData = newMetricData;
*/

return data;

var cloudwatch = new CloudWatch(this.config);

console.log(new Date(timestamp*1000).toISOString());


var counters = metrics.counters;
var gauges = metrics.gauges;
var timers = metrics.timers;
var sets = metrics.sets;

for (key in counters) {
if (key.indexOf('statsd.') == 0)
continue;

names = this.config.processKeyForNamespace ? this.processKey(key) : {};
var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend";
var metricName = this.config.metricName || names.metricName || key;

cloudwatch.PutMetricData({
MetricData : [{
MetricName : metricName,
Unit : 'Count',
Timestamp: new Date(timestamp*1000).toISOString(),
Value : counters[key]
}],
Namespace : namespace
},
function(err, data) {
fmt.dump(err, 'Err');
fmt.dump(data, 'Data');
});
}

for (key in timers) {
if (timers[key].length > 0) {
var values = timers[key].sort(function (a,b) { return a-b; });
var count = values.length;
var min = values[0];
var max = values[count - 1];

var cumulativeValues = [min];
for (var i = 1; i < count; i++) {
cumulativeValues.push(values[i] + cumulativeValues[i-1]);
}

var sum = min;
var mean = min;
var maxAtThreshold = max;

var message = "";

var key2;

sum = cumulativeValues[count-1];
mean = sum / count;

names = this.config.processKeyForNamespace ? this.processKey(key) : {};
var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend";
var metricName = this.config.metricName || names.metricName || key;

cloudwatch.PutMetricData({
MetricData : [{
MetricName : metricName,
Unit : 'Milliseconds',
Timestamp: new Date(timestamp*1000).toISOString(),
StatisticValues: {
Minimum: min,
Maximum: max,
Sum: sum,
SampleCount: count
}
}],
Namespace : namespace
},
function(err, data) {
fmt.dump(err, 'Err');
fmt.dump(data, 'Data');
});

}
}

for (key in gauges) {
names = this.config.processKeyForNamespace ? this.processKey(key) : {};
var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend";
var metricName = this.config.metricName || names.metricName || key;

cloudwatch.PutMetricData({
MetricData : [{
MetricName : metricName,
Unit : 'None',
Timestamp: new Date(timestamp*1000).toISOString(),
Value : gauges[key]
}],
Namespace : namespace
},
function(err, data) {
fmt.dump(err, 'Err');
fmt.dump(data, 'Data');
});
}

for (key in sets) {
names = this.config.processKeyForNamespace ? this.processKey(key) : {};
var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend";
var metricName = this.config.metricName || names.metricName || key;

cloudwatch.PutMetricData({
MetricData : [{
MetricName : metricName,
Unit : 'None',
Timestamp: new Date(timestamp*1000).toISOString(),
Value : sets[key].values().length
}],
Namespace : namespace
},
function(err, data) {
fmt.dump(err, 'Err');
fmt.dump(data, 'Data');
});

statString += 'stats.sets.' + key + '.count ' + sets[key].values().length + ' ' + ts + "\n";
numStats += 1;
}

}

};
CloudwatchBackend.prototype._addMetricData = function(packets, data) {

if (!(data.Namespace in packets)) {
packets[data.Namespace] = [{ "Namespace": data.Namespace, "MetricData": [] }];
}
last_packet = packets[data.Namespace].slice(-1)[0];
for (var i = 0; i < data.MetricData.length; i++) {
/* Keep packets from getting too big, awssum is not doing any smart handling of limits */
if (last_packet["MetricData"].length >= 8) {
last_packet = { "Namespace": data.Namespace, "MetricData": [] };
packets[data.Namespace].push(last_packet);
}
last_packet.MetricData.push(data.MetricData[i]);
}

}

CloudwatchBackend.prototype.flush = function(timestamp, metrics) {

var cloudwatch = new CloudWatch(this.config);

var counters = metrics.counters;
var gauges = metrics.gauges;
var timers = metrics.timers;
var sets = metrics.sets;

var cwPackets = {}

for (key in counters) {
if (key.indexOf('statsd.') == 0)
continue;
md = this.prepareCWData(key, {
MetricData : [{
Unit : 'Count',
Timestamp: new Date(timestamp*1000).toISOString(),
Value : counters[key]
}]
});
this._addMetricData(cwPackets, md);
}

for (key in timers) {
if (timers[key].length > 0) {
var values = timers[key].sort(function (a,b) { return a-b; });
var count = values.length;
var min = values[0];
var max = values[count - 1];

var cumulativeValues = [min];
for (var i = 1; i < count; i++) {
cumulativeValues.push(values[i] + cumulativeValues[i-1]);
}

var sum = min;
var mean = min;
var maxAtThreshold = max;

var message = "";

var key2;

sum = cumulativeValues[count-1];
mean = sum / count;

md = this.prepareCWData(key, {
MetricData : [{
Unit : 'Milliseconds',
Timestamp: new Date(timestamp*1000).toISOString(),
StatisticValues: {
Minimum: min,
Maximum: max,
Sum: sum,
SampleCount: count
}
}]
});
this._addMetricData(cwPackets, md);
}
}

for (key in gauges) {
md = this.prepareCWData(key, {
MetricData : [{
Unit : 'None',
Timestamp: new Date(timestamp*1000).toISOString(),
Value : gauges[key]
}]
});
this._addMetricData(cwPackets, md);
}

for (key in sets) {
md = this.prepareCWData(key, {
MetricData : [{
Unit : 'None',
Timestamp: new Date(timestamp*1000).toISOString(),
Value : sets[key].values().length
}]
});
this._addMetricData(cwPackets, md);
statString += 'stats.sets.' + key + '.count ' + sets[key].values().length + ' ' + ts + "\n";
numStats += 1;
}

/* Do the actual sending of data */
for (ns in cwPackets) {
for (var i = 0; i < cwPackets[ns].length; i++) {
cloudwatch.PutMetricData(
cwPackets[ns][i],
function(err, data) {
fmt.dump(err, 'Err');
fmt.dump(data, 'Data');
}
);
}
}

};

exports.init = function(startupTime, config, events) {
var instance = new CloudwatchBackend(startupTime, config, events);
return true;
var instance = new CloudwatchBackend(startupTime, config, events);
return true;
};

Loading