Skip to content

Commit

Permalink
Merge pull request #1 from intellifylearning/optimize
Browse files Browse the repository at this point in the history 
send accumulated metrics as an array instead of one at a time... save…
  • Loading branch information
Prashant Nayak authored Sep 11, 2016
2 parents 446acde + 24e2b1d commit efa7b37
Showing 1 changed file with 125 additions and 90 deletions.
215 changes: 125 additions & 90 deletions lib/aws-cloudwatch-statsd-backend.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
var util = require('util');
var AWS = require('aws-sdk');

function CloudwatchBackend(startupTime, config, emitter){
function CloudwatchBackend(startupTime, config, emitter) {
var self = this;

this.config = config || {};
Expand All @@ -13,22 +13,22 @@ function CloudwatchBackend(startupTime, config, emitter){
}

// if iamRole is set attempt to fetch credentials from the Metadata Service
if(this.config.iamRole) {
if (this.config.iamRole) {
if (this.config.iamRole == 'any') {
// If the iamRole is set to any, then attempt to fetch any available credentials
ms = new AWS.EC2MetadataCredentials();
ms.refresh(function(err) {
if(err) { console.log('Failed to fetch IAM role credentials: '+err); }
if (err) { console.log('Failed to fetch IAM role credentials: ' + err); }
self.config.credentials = ms;
setEmitter();
});
} else {
// however if it's set to specify a role, query it specifically.
ms = new AWS.MetadataService();
ms.request('/latest/meta-data/iam/security-credentials/'+this.config.iamRole, function(err, rdata) {
ms.request('/latest/meta-data/iam/security-credentials/' + this.config.iamRole, function(err, rdata) {
var data = JSON.parse(rdata);

if(err) { console.log('Failed to fetch IAM role credentials: '+err); }
if (err) { console.log('Failed to fetch IAM role credentials: ' + err); }
self.config.credentials = new AWS.Credentials(data.AccessKeyId, data.SecretAccessKey, data.Token);
setEmitter();
});
Expand All @@ -39,67 +39,80 @@ function CloudwatchBackend(startupTime, config, emitter){
};

CloudwatchBackend.prototype.processKey = function(key) {
var parts = key.split(/[\.\/-]/);
var parts = key.split(/[\.\/-]/);

return {
metricName: parts[parts.length-1],
namespace: parts.length > 1 ? parts.splice(0, parts.length-1).join("/") : null
};
}
return {
metricName: parts[parts.length - 1],
namespace: parts.length > 1 ? parts.splice(0, parts.length - 1).join("/") : null
};
};

CloudwatchBackend.prototype.flush = function(timestamp, metrics) {

console.log('Flushing metrics at ' + new Date(timestamp*1000).toISOString());
console.log('Flushing metrics at ' + new Date(timestamp * 1000).toISOString());

var counters = metrics.counters;
var gauges = metrics.gauges;
var timers = metrics.timers;
var sets = metrics.sets;

// put all currently accumulated counter metrics into an array
var currentCounterMetrics = [];
for (key in counters) {
if (key.indexOf('statsd.') == 0)
continue;
if (key.indexOf('statsd.') == 0)
continue;

if (this.config.whitelist && this.config.whitelist.length > 0 && this.config.whitelist.indexOf(key) == -1) {
console.log("Key (counter) " + key + " not in whitelist");
continue;
}

if(this.config.whitelist && this.config.whitelist.length >0 && this.config.whitelist.indexOf(key) == -1) {
console.log("Key (counter) "+key+" not in whitelist");
continue;
}

var names = this.config.processKeyForNamespace ? this.processKey(key) : {};
var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend";
var metricName = this.config.metricName || names.metricName || key;

this.cloudwatch.putMetricData({
MetricData : [{
MetricName : metricName,
Unit : 'Count',
Timestamp: new Date(timestamp*1000).toISOString(),
Value : counters[key]
}],
Namespace : namespace
},
function(err, data) {
console.log(util.inspect(err));
console.log(util.inspect(data));
currentCounterMetrics.push({
MetricName: metricName,
Unit: 'Count',
Timestamp: new Date(timestamp * 1000).toISOString(),
Value: counters[key]
});
}

// send off the array (instead of one at a time)
this.cloudwatch.putMetricData({
MetricData: currentCounterMetrics,
Namespace: namespace
}, function(err, data) {
if (err) {
// log an error
console.log(util.inspect(err));
} else {
// Success
// console.log(util.inspect(data));
}
});

// put all currently accumulated timer metrics into an array
var currentTimerMetrics = [];
for (key in timers) {
if (timers[key].length > 0) {

if(this.config.whitelist && this.config.whitelist.length >0 && this.config.whitelist.indexOf(key) == -1) {
console.log("Key (counter) "+key+" not in whitelist");
continue;
}
if (this.config.whitelist && this.config.whitelist.length > 0 && this.config.whitelist.indexOf(key) == -1) {
console.log("Key (counter) " + key + " not in whitelist");
continue;
}

var values = timers[key].sort(function (a,b) { return a-b; });
var values = timers[key].sort(function(a, b) {
return a - b;
});
var count = values.length;
var min = values[0];
var max = values[count - 1];

var cumulativeValues = [min];
for (var i = 1; i < count; i++) {
cumulativeValues.push(values[i] + cumulativeValues[i-1]);
cumulativeValues.push(values[i] + cumulativeValues[i - 1]);
}

var sum = min;
Expand All @@ -110,87 +123,109 @@ CloudwatchBackend.prototype.flush = function(timestamp, metrics) {

var key2;

sum = cumulativeValues[count-1];
sum = cumulativeValues[count - 1];
mean = sum / count;

var names = this.config.processKeyForNamespace ? this.processKey(key) : {};
var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend";
var metricName = this.config.metricName || names.metricName || key;

this.cloudwatch.putMetricData({
MetricData : [{
MetricName : metricName,
Unit : 'Milliseconds',
Timestamp: new Date(timestamp*1000).toISOString(),
StatisticValues: {
Minimum: min,
Maximum: max,
Sum: sum,
SampleCount: count
}
}],
Namespace : namespace
},
function(err, data) {
console.log(util.inspect(err));
console.log(util.inspect(data));
currentTimerMetrics.push({
MetricName: metricName,
Unit: 'Milliseconds',
Timestamp: new Date(timestamp * 1000).toISOString(),
StatisticValues: {
Minimum: min,
Maximum: max,
Sum: sum,
SampleCount: count
}
});
}
}
}

// send off the array (instead of one at a time)
this.cloudwatch.putMetricData({
MetricData: currentTimerMetrics,
Namespace: namespace
}, function(err, data) {
if (err) {
// log an error
console.log(util.inspect(err));
} else {
// Success
// console.log(util.inspect(data));
}
});

// put all currently accumulated gauge metrics into an array
var currentGaugeMetrics = [];
for (key in gauges) {

if(this.config.whitelist && this.config.whitelist.length >0 && this.config.whitelist.indexOf(key) == -1) {
console.log("Key (counter) "+key+" not in whitelist");
continue;
}
if (this.config.whitelist && this.config.whitelist.length > 0 && this.config.whitelist.indexOf(key) == -1) {
console.log("Key (counter) " + key + " not in whitelist");
continue;
}

var names = this.config.processKeyForNamespace ? this.processKey(key) : {};
var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend";
var metricName = this.config.metricName || names.metricName || key;

this.cloudwatch.putMetricData({
MetricData : [{
MetricName : metricName,
Unit : 'None',
Timestamp: new Date(timestamp*1000).toISOString(),
Value : gauges[key]
}],
Namespace : namespace
},

function(err, data) {
console.log(util.inspect(err));
console.log(util.inspect(data));
currentGaugeMetrics.push({
MetricName: metricName,
Unit: 'None',
Timestamp: new Date(timestamp * 1000).toISOString(),
Value: gauges[key]
});
}

// send off the array (instead of one at a time)
this.cloudwatch.putMetricData({
MetricData: currentGaugeMetrics,
Namespace: namespace
}, function(err, data) {
if (err) {
// log an error
console.log(util.inspect(err));
} else {
// Success
// console.log(util.inspect(data));
}
});

// put all currently accumulated set metrics into an array
var currentSetMetrics = [];
for (key in sets) {

if(this.config.whitelist && this.config.whitelist.length >0 && this.config.whitelist.indexOf(key) == -1) {
console.log("Key (counter) "+key+" not in whitelist");
continue;
}
if (this.config.whitelist && this.config.whitelist.length > 0 && this.config.whitelist.indexOf(key) == -1) {
console.log("Key (counter) " + key + " not in whitelist");
continue;
}

var names = this.config.processKeyForNamespace ? this.processKey(key) : {};
var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend";
var metricName = this.config.metricName || names.metricName || key;

this.cloudwatch.putMetricData({
MetricData : [{
MetricName : metricName,
Unit : 'None',
Timestamp: new Date(timestamp*1000).toISOString(),
Value : sets[key].values().length
}],
Namespace : namespace
},

function(err, data) {
console.log(util.inspect(err));
console.log(util.inspect(data));
currentSetMetrics.push({
MetricName: metricName,
Unit: 'None',
Timestamp: new Date(timestamp * 1000).toISOString(),
Value: sets[key].values().length
});
}

this.cloudwatch.putMetricData({
MetricData: currentSetMetrics,
Namespace: namespace
}, function(err, data) {
if (err) {
// log an error
console.log(util.inspect(err));
} else {
// Success
// console.log(util.inspect(data));
}
});
};

exports.init = function(startupTime, config, events) {
Expand Down

0 comments on commit efa7b37

Please sign in to comment.