From 24e2b1df22fe976b5c464d7e6b5cd8d732b09d68 Mon Sep 17 00:00:00 2001 From: Prashant Nayak Date: Sun, 11 Sep 2016 19:03:13 -0400 Subject: [PATCH] send accumulated metrics as an array instead of one at a time... saves bandwidth and AWS charges... Also, after sending... dont print to console - this fills up /var/log/statsd.log very quickly on busy systems --- lib/aws-cloudwatch-statsd-backend.js | 215 ++++++++++++++++----------- 1 file changed, 125 insertions(+), 90 deletions(-) diff --git a/lib/aws-cloudwatch-statsd-backend.js b/lib/aws-cloudwatch-statsd-backend.js index 1fff377..5becd92 100644 --- a/lib/aws-cloudwatch-statsd-backend.js +++ b/lib/aws-cloudwatch-statsd-backend.js @@ -1,7 +1,7 @@ var util = require('util'); var AWS = require('aws-sdk'); -function CloudwatchBackend(startupTime, config, emitter){ +function CloudwatchBackend(startupTime, config, emitter) { var self = this; this.config = config || {}; @@ -13,22 +13,22 @@ function CloudwatchBackend(startupTime, config, emitter){ } // if iamRole is set attempt to fetch credentials from the Metadata Service - if(this.config.iamRole) { + if (this.config.iamRole) { if (this.config.iamRole == 'any') { // If the iamRole is set to any, then attempt to fetch any available credentials ms = new AWS.EC2MetadataCredentials(); ms.refresh(function(err) { - if(err) { console.log('Failed to fetch IAM role credentials: '+err); } + if (err) { console.log('Failed to fetch IAM role credentials: ' + err); } self.config.credentials = ms; setEmitter(); }); } else { // however if it's set to specify a role, query it specifically. ms = new AWS.MetadataService(); - ms.request('/latest/meta-data/iam/security-credentials/'+this.config.iamRole, function(err, rdata) { + ms.request('/latest/meta-data/iam/security-credentials/' + this.config.iamRole, function(err, rdata) { var data = JSON.parse(rdata); - if(err) { console.log('Failed to fetch IAM role credentials: '+err); } + if (err) { console.log('Failed to fetch IAM role credentials: ' + err); } self.config.credentials = new AWS.Credentials(data.AccessKeyId, data.SecretAccessKey, data.Token); setEmitter(); }); @@ -39,67 +39,80 @@ function CloudwatchBackend(startupTime, config, emitter){ }; CloudwatchBackend.prototype.processKey = function(key) { - var parts = key.split(/[\.\/-]/); + var parts = key.split(/[\.\/-]/); - return { - metricName: parts[parts.length-1], - namespace: parts.length > 1 ? parts.splice(0, parts.length-1).join("/") : null - }; -} + return { + metricName: parts[parts.length - 1], + namespace: parts.length > 1 ? parts.splice(0, parts.length - 1).join("/") : null + }; +}; CloudwatchBackend.prototype.flush = function(timestamp, metrics) { - console.log('Flushing metrics at ' + new Date(timestamp*1000).toISOString()); + console.log('Flushing metrics at ' + new Date(timestamp * 1000).toISOString()); var counters = metrics.counters; var gauges = metrics.gauges; var timers = metrics.timers; var sets = metrics.sets; + // put all currently accumulated counter metrics into an array + var currentCounterMetrics = []; for (key in counters) { - if (key.indexOf('statsd.') == 0) - continue; + if (key.indexOf('statsd.') == 0) + continue; + + if (this.config.whitelist && this.config.whitelist.length > 0 && this.config.whitelist.indexOf(key) == -1) { + console.log("Key (counter) " + key + " not in whitelist"); + continue; + } - if(this.config.whitelist && this.config.whitelist.length >0 && this.config.whitelist.indexOf(key) == -1) { - console.log("Key (counter) "+key+" not in whitelist"); - continue; - } - var names = this.config.processKeyForNamespace ? this.processKey(key) : {}; var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; var metricName = this.config.metricName || names.metricName || key; - this.cloudwatch.putMetricData({ - MetricData : [{ - MetricName : metricName, - Unit : 'Count', - Timestamp: new Date(timestamp*1000).toISOString(), - Value : counters[key] - }], - Namespace : namespace - }, - function(err, data) { - console.log(util.inspect(err)); - console.log(util.inspect(data)); + currentCounterMetrics.push({ + MetricName: metricName, + Unit: 'Count', + Timestamp: new Date(timestamp * 1000).toISOString(), + Value: counters[key] }); } + // send off the array (instead of one at a time) + this.cloudwatch.putMetricData({ + MetricData: currentCounterMetrics, + Namespace: namespace + }, function(err, data) { + if (err) { + // log an error + console.log(util.inspect(err)); + } else { + // Success + // console.log(util.inspect(data)); + } + }); + + // put all currently accumulated timer metrics into an array + var currentTimerMetrics = []; for (key in timers) { if (timers[key].length > 0) { - if(this.config.whitelist && this.config.whitelist.length >0 && this.config.whitelist.indexOf(key) == -1) { - console.log("Key (counter) "+key+" not in whitelist"); - continue; - } + if (this.config.whitelist && this.config.whitelist.length > 0 && this.config.whitelist.indexOf(key) == -1) { + console.log("Key (counter) " + key + " not in whitelist"); + continue; + } - var values = timers[key].sort(function (a,b) { return a-b; }); + var values = timers[key].sort(function(a, b) { + return a - b; + }); var count = values.length; var min = values[0]; var max = values[count - 1]; var cumulativeValues = [min]; for (var i = 1; i < count; i++) { - cumulativeValues.push(values[i] + cumulativeValues[i-1]); + cumulativeValues.push(values[i] + cumulativeValues[i - 1]); } var sum = min; @@ -110,87 +123,109 @@ CloudwatchBackend.prototype.flush = function(timestamp, metrics) { var key2; - sum = cumulativeValues[count-1]; + sum = cumulativeValues[count - 1]; mean = sum / count; var names = this.config.processKeyForNamespace ? this.processKey(key) : {}; var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; var metricName = this.config.metricName || names.metricName || key; - this.cloudwatch.putMetricData({ - MetricData : [{ - MetricName : metricName, - Unit : 'Milliseconds', - Timestamp: new Date(timestamp*1000).toISOString(), - StatisticValues: { - Minimum: min, - Maximum: max, - Sum: sum, - SampleCount: count - } - }], - Namespace : namespace - }, - function(err, data) { - console.log(util.inspect(err)); - console.log(util.inspect(data)); + currentTimerMetrics.push({ + MetricName: metricName, + Unit: 'Milliseconds', + Timestamp: new Date(timestamp * 1000).toISOString(), + StatisticValues: { + Minimum: min, + Maximum: max, + Sum: sum, + SampleCount: count + } }); - } + } } + // send off the array (instead of one at a time) + this.cloudwatch.putMetricData({ + MetricData: currentTimerMetrics, + Namespace: namespace + }, function(err, data) { + if (err) { + // log an error + console.log(util.inspect(err)); + } else { + // Success + // console.log(util.inspect(data)); + } + }); + + // put all currently accumulated gauge metrics into an array + var currentGaugeMetrics = []; for (key in gauges) { - if(this.config.whitelist && this.config.whitelist.length >0 && this.config.whitelist.indexOf(key) == -1) { - console.log("Key (counter) "+key+" not in whitelist"); - continue; - } + if (this.config.whitelist && this.config.whitelist.length > 0 && this.config.whitelist.indexOf(key) == -1) { + console.log("Key (counter) " + key + " not in whitelist"); + continue; + } var names = this.config.processKeyForNamespace ? this.processKey(key) : {}; var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; var metricName = this.config.metricName || names.metricName || key; - this.cloudwatch.putMetricData({ - MetricData : [{ - MetricName : metricName, - Unit : 'None', - Timestamp: new Date(timestamp*1000).toISOString(), - Value : gauges[key] - }], - Namespace : namespace - }, - - function(err, data) { - console.log(util.inspect(err)); - console.log(util.inspect(data)); + currentGaugeMetrics.push({ + MetricName: metricName, + Unit: 'None', + Timestamp: new Date(timestamp * 1000).toISOString(), + Value: gauges[key] }); } + // send off the array (instead of one at a time) + this.cloudwatch.putMetricData({ + MetricData: currentGaugeMetrics, + Namespace: namespace + }, function(err, data) { + if (err) { + // log an error + console.log(util.inspect(err)); + } else { + // Success + // console.log(util.inspect(data)); + } + }); + + // put all currently accumulated set metrics into an array + var currentSetMetrics = []; for (key in sets) { - if(this.config.whitelist && this.config.whitelist.length >0 && this.config.whitelist.indexOf(key) == -1) { - console.log("Key (counter) "+key+" not in whitelist"); - continue; - } + if (this.config.whitelist && this.config.whitelist.length > 0 && this.config.whitelist.indexOf(key) == -1) { + console.log("Key (counter) " + key + " not in whitelist"); + continue; + } var names = this.config.processKeyForNamespace ? this.processKey(key) : {}; var namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; var metricName = this.config.metricName || names.metricName || key; - this.cloudwatch.putMetricData({ - MetricData : [{ - MetricName : metricName, - Unit : 'None', - Timestamp: new Date(timestamp*1000).toISOString(), - Value : sets[key].values().length - }], - Namespace : namespace - }, - - function(err, data) { - console.log(util.inspect(err)); - console.log(util.inspect(data)); + currentSetMetrics.push({ + MetricName: metricName, + Unit: 'None', + Timestamp: new Date(timestamp * 1000).toISOString(), + Value: sets[key].values().length }); } + + this.cloudwatch.putMetricData({ + MetricData: currentSetMetrics, + Namespace: namespace + }, function(err, data) { + if (err) { + // log an error + console.log(util.inspect(err)); + } else { + // Success + // console.log(util.inspect(data)); + } + }); }; exports.init = function(startupTime, config, events) {