Add InfluxDB Line Protocol Support, and Add endpoint for JSON

README.md

@@ -57,9 +57,9 @@ to `/APP_ROOT/v1/send` on whichever port you specify.
 ```bash
 # Install nodejs
 # This assumes you're on a 64 bit machine
-wget http://nodejs.org/dist/v0.10.19/node-v0.10.19-linux-x64.tar.gz
-tar xvf node-v0.10.19-linux-x64.tar.gz 
-sudo ln -s `pwd`/node-v0.10.19-linux-x64/bin/{node,npm} /usr/local/bin/
+wget https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz
+tar xvf node-v4.1.1-linux-x64.tar.gz
+sudo ln -s `pwd`/node-v4.1.1-linux-x64/bin/{node,npm} /usr/local/bin/
 
 # Grab a Bucky release
 # You should use the latest release available at https://github.com/HubSpot/BuckyServer/releases
@@ -101,7 +101,14 @@ If you need more customization, you can write a module:
 
 There are a few of types of modules:
 
-
+- Server - Use to set properties of the Bucky Server
+  - port - Use to set the port that Bucky Server will listen to
+  - appRoot - Use to define the root of the endpoint
+  - https - Use to set options for running Bucky Server in https mode
+    - port - Use to specify the port for https, if not populated the http server port + 1
+    - options - Use to define the certificates for https. [List of all possible options](https://nodejs.org/api/tls.html#tls_tls_createserver_options_secureconnectionlistener)
+      - For all options that accept a buffer you can use the path to the file and they'll be read in
+  - httpsOnly - If this flag is set to true then Bucky Server will not run in http mode
 - Logger - Use to have Bucky log to something other than the console
 - Config - Use to have Bucky pull config from somewhere other than the default file
 - App - Use to do things when Bucky loads and/or on requests.  Auth, monitoring initialization, etc.
@@ -180,7 +187,7 @@ called with like this:
 You are free to implement the `on` method as a dud if live reloading doesn't
 make sense using your config system.  Take a look at [lib/configWrapper.coffee](lib/configWrapper.coffee)
 for an example of how a basic object can be converted (and feel free to use it).
-  
+
 #### App
 
 App modules get loaded once, and can optionally provide a function to be ran with each request.
@@ -236,7 +243,10 @@ module.exports = ({app, logger, config}, next) ->
 
 ### Format
 
-If you are interested in writing new clients, the format of metric data is the same as is used by statsd:
+If you are interested in writing new clients, there are two endpoints for inbound data.
+The default endpoint uses the same format as statsd:
+
+default endpoint: `{hostname}:{port}/v1/{appRoot}` uses
 
 ```
 <metric name>:<metric value>|<unit>[@<sample rate>]
@@ -249,4 +259,24 @@ my.awesome.metric:35|ms
 some.other.metric:3|[email protected]
 ```
 
-All requests are sent with content-type `text/plain`.
+All post reqeusts sent to the default endpoint must use content-type `text/plain`.
+
+JSON endpoint: `{hostname}:{port}/v1/{appRoot}/json` uses
+
+```javascript
+{
+  "<metric name>": "<metric value>[|<unit>[@<sample rate>]"
+}
+```
+
+This allows for the ':' character to be included in your metrics. This is valid for InfluxDB Line Protocol.
+
+For example:
+```javascript
+{
+  "page,browser=Chrome,browserVersion=44,url=http://localhost:3000/#customHash/%7Bexample%3A%22encoded%20data%22%7D,key=domContentLoadedEventEnd": "500|ms",
+  "ajax,browser=Microsoft\\ Internet\\ Explorer,browserVersion=8,url=http://localhost:3000/#customHash/%7Bexample%3A%22encoded%20data%22%7D,endpoint=your/awesome/template.html,method=GET,status=200": "1|c"
+}
+```
+
+All post request to the json endpoint will be *converted* to content-type 'application/json'. This allows for backwards compatibility with IE8 which can't send XDomainRequest with a content-type other than 'plain/text'.

config/default.yaml

@@ -1,6 +1,15 @@
 server:
   port: 5999
   appRoot: "/bucky"
+  # If https options are set then bucky will also listen on https
+  # https:
+    # port: 5599
+    # See full list of options at: https://nodejs.org/api/tls.html#tls_tls_createserver_options_secureconnectionlistener
+    # options:
+      # key: "ssl/key.pem"
+      # cert: "ssl/cert.pem"
+  # If the following is set to true then bucky server will not run in http mode
+  # httpsOnly: true
 
 statsd:
   host: 'localhost'
@@ -18,7 +27,7 @@ influxdb:
   password: 'root'
   use_udp: false
   retentionPolicy: 'default'
-  # Acceptable version are: '0.8' and '0.9'
+  # All version other than '0.9' will default to '0.8'
   version: '0.9'
 
 
@@ -32,7 +41,7 @@ modules:
 
   collectors:
   # Uncomment the modules that you'd like to use
-    # - ./modules/collectionLogger
+    - ./modules/collectionLogger
     # - ./modules/statsd
     # - ./modules/openTSDB
     # - ./modules/influxdb

lib/influxdb.coffee

@@ -1,78 +1,95 @@
-request = require('request')
-dgram = require('dgram')
+request = require "request"
+dgram = require "dgram"
 
 class Client
   constructor: (@config={}, @logger) ->
     do @init
 
   init: ->
-    useUDP = @config.get('influxdb.use_udp').get() ? false
+    useUDP = @config.get("influxdb.use_udp").get() ? false
 
     @send = if useUDP then @sendUDP() else @sendHTTP()
 
   write: (metrics) ->
-    @send @metricsJson metrics
+    @send @formatMetrics metrics
 
   sendHTTP: ->
-    version = @config.get('influxdb.version').get() ? '0.9'
-    host = @config.get('influxdb.host').get() ? 'localhost'
-    port = @config.get('influxdb.port').get() ? 8086
-    database = @config.get('influxdb.database').get() ? 'bucky'
-    username = @config.get('influxdb.username').get() ? 'root'
-    password = @config.get('influxdb.password').get() ? 'root'
+    version = @config.get("influxdb.version").get() ? "0.9"
+    host = @config.get("influxdb.host").get() ? "localhost"
+    port = @config.get("influxdb.port").get() ? 8086
+    database = @config.get("influxdb.database").get() ? "bucky"
+    username = @config.get("influxdb.username").get() ? "root"
+    password = @config.get("influxdb.password").get() ? "root"
+    retentionPolicy = @config.get("influxdb.retentionPolicy").get() ? "default"
     logger = @logger
-    if version == '0.8'
-      endpoint = 'http://' + host + ':' + port + '/db/' + database + '/series'
-    else
-      endpoint = 'http://' + host + ':' + port + '/write'
-    client = request.defaults
-      method: 'POST'
-      url: endpoint
+
+    clientConfig =
+      method: "POST"
       qs:
         u: username
         p: password
 
-    (metricsJson) ->
-      client form: metricsJson, (error, response, body) ->
-        logger.log error if error
+    if version == "0.9"
+      clientConfig.url = "http://" + host + ":" + port + "/write"
+      clientConfig.qs.db = database
+      clientConfig.qs.rp = retentionPolicy
+    else
+      clientConfig.url = "http://" + host + ":" + port + "/db/" + database + "/series"
+
+    client = request.defaults clientConfig
+
+    (formatMetrics) ->
+      if version == "0.9"
+        metrics = formatMetrics.join "\n"
+        # uncomment to see data sent to DB
+        # logger.log "db: " + database + "\n" + metrics
+        client body: metrics, (error, response, body) ->
+          logger.log "Warning:" if body && body.length > 0
+          logger.log "\tresponse:\n", body if body && body.length > 0
+          logger.log error if error
+      else
+        metrics = JSON.stringify formatMetrics
+        # logger.log "db: " + database + "\n" + metrics
+        client form: metrics, (error, response, body) ->
+          logger.log "Warning:" if body && body.length > 0
+          logger.log "\tresponse:\n", body if body && body.length > 0
+          logger.log error if error
 
   sendUDP: ->
-    host = @config.get('influxdb.host').get() ? 'localhost'
-    port = @config.get('influxdb.port').get() ? 4444
-    client = dgram.createSocket 'udp4'
-
-    (metricsJson) ->
-      message = new Buffer metricsJson
+    version = @config.get("influxdb.version").get() ? "0.9"
+    host = @config.get("influxdb.host").get() ? "localhost"
+    port = @config.get("influxdb.port").get() ? 4444
+    client = dgram.createSocket "udp4"
+
+    (formatMetrics) ->
+      if version == "0.9"
+        formatMetrics.forEach (metric) ->
+          message = new Buffer metric
+          client.send message, 0, message.length, port, host
+      else
+        message = new Buffer JSON.stringify formatMetrics
+        client.send message, 0, message.length, port, host
 
-      client.send message, 0, message.length, port, host
+  formatMetrics: (metrics) ->
+    version = @config.get("influxdb.version").get() ? "0.9"
+    data = []
 
-  metricsJson: (metrics) ->
-    version = @config.get('influxdb.version').get() ? '0.9'
-    if version == '0.8'
-      data = []
-    else
-      data =
-        database: @config.get('influxdb.database').get() ? 'bucky'
-        retentionPolicy: @config.get('influxdb.retentionPolicy').get() ? "default"
-        time: new Date().toISOString()
-        points: []
     for key, desc of metrics
       [val, unit, sample] = @parseRow desc
 
-      if version == '0.8'
-        data.push
+      if version == "0.9"
+        fields = key.replace(/\\? /g, "\\ ")
+        fields += " value=" + parseFloat val
+        fields += ',unit="' + unit.replace(/"/g, '\\"') + '"' if unit
+        fields += ",sample=" + sample if sample
+      else
+        fields =
           name: key,
-          columns: ['value'],
+          columns: ["value"],
           points: [[parseFloat val]]
-      else
-        data.points.push
-          measurement: key
-          fields:
-            value: parseFloat val
-            unit: unit
-            sample: sample
-    # @logger.log(JSON.stringify(data, null, 2))
-    JSON.stringify data
+      data.push fields
+
+    data
 
   parseRow: (row) ->
     re = /([0-9\.]+)\|([a-z]+)(?:@([0-9\.]+))?/

package.json

@@ -1,6 +1,6 @@
 {
   "name": "bucky-server",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "Server to collect stats from the client",
   "main": "./start.js",
   "bin": "./start.js",
@@ -23,7 +23,7 @@
     "express": "~3.2.5",
     "coffee-script": "~1.6.2",
     "lynx": "~0.0.11",
-    "underscore": "~1.4.4",
+    "underscore": "~1.8.3",
     "nopents": "~0.1.0",
     "config": "~0.4.27",
     "q": "~0.9.6",

server.coffee

@@ -3,9 +3,10 @@
 Q = require 'q'
 _ = require 'underscore'
 express = require 'express'
+http = require 'http'
 
 # Set cwd for config, and load config file
-process.chdir(__dirname);
+process.chdir __dirname
 config = require 'config'
 
 configWrapper = require './lib/configWrapper'
@@ -14,23 +15,29 @@ load = require './lib/load'
 MODULES = config.modules
 loadLogger = ->
   if MODULES.logger
-    load(MODULES.logger, {config})
+    load MODULES.logger, {config}
   else
     console
 
 # We always have the base config, but the
 # app can optionally swap it out for something else.
 loadConfig = (logger) ->
   if MODULES.config
-    load(MODULES.config, {config, logger})
+    load MODULES.config, {config, logger}
   else
-    configWrapper(config)
+    configWrapper config
 
 setCORSHeaders = (req, res, next) ->
   res.setHeader 'Access-Control-Allow-Origin', '*'
   res.setHeader 'Access-Control-Allow-Methods', 'POST'
   res.setHeader 'Access-Control-Max-Age', '604800'
   res.setHeader 'Access-Control-Allow-Credentials', 'true'
+  res.setHeader 'Access-Control-Allow-Headers', 'content-type'
+
+  next()
+
+setJSONHeader = (req, res, next) ->
+  req.headers['content-type'] = 'application/json'
 
   next()
 
@@ -103,6 +110,7 @@ loadApp = (logger, loadedConfig) ->
 
     for path, handlers of routes
       # Bind all request modules as middleware and install the collectors
+      app.post "#{ path }/json", setJSONHeader, express.json(), setCORSHeaders, handlers...
       app.post path, parser, setCORSHeaders, handlers...
 
       app.options path, setCORSHeaders, (req, res) ->
@@ -111,15 +119,29 @@ loadApp = (logger, loadedConfig) ->
     app.get "#{ APP_ROOT }/v1/health-check", (req, res) ->
       res.send('OK\n')
 
-    port = process.env.PORT ? loadedConfig.get('server.port').get() ? 5000
-    app.listen(port)
-
-    logger.log('Server listening on port %d in %s mode', port, app.settings.env)
+    if loadedConfig.get('server.https.options').get() instanceof Object
+      https = require 'https'
+      fs = require 'fs'
+      httpsOptions = _.mapObject loadedConfig.get('server.https.options').get(), (v, k) ->
+        if _.isString(v)
+          try
+            fs.readFileSync(v)
+          catch
+            v
+        else
+          v
+      httpsPort = loadedConfig.get('server.https.port').get() ? (port + 1)
+      https.createServer(httpsOptions, app).listen httpsPort
+      logger.log "HTTPS Server listening on port %d in %s mode", httpsPort, app.settings.env
+    if !loadedConfig.get('server.httpsOnly').get()
+      port = process.env.PORT ? loadedConfig.get('server.port').get() ? 5000
+      http.createServer(app).listen port
+      logger.log 'HTTP Server listening on port %d in %s mode', port, app.settings.env
 
 Q.when(loadLogger()).then (logger) ->
 
   logger.log "Loading Config"
-  Q.when(loadConfig(logger)).then (loadedConfig) ->
+  Q.when(loadConfig logger).then (loadedConfig) ->
 
     logger.log "Loading App"
     loadApp(logger, loadedConfig)