Added support for additional HTTP headers to HTTP miner

docs/nodeconfig.rst

@@ -192,6 +192,8 @@ Parameters
     character is used as indicator. Default: *null*
 :fields: a dicionary of *extraction dictionaries* to extract
     additional attributes from each line. Default: {}
+:headers: a dictionary of additional headers to add to the HTTP
+    header. Default: {}
 
 Extraction dictionary
 +++++++++++++++++++++
@@ -230,6 +232,8 @@ extract the indicator and additional fields::
         dshield_email:
             regex: '^.*\t.*\t[0-9]+\t[0-9]+\t[^\t]+\t[A-Z]+\t(\S+)'
             transform: '\1'
+    headers:
+        api-key: sample-api-key
 
 Example config in YAML where the text in each line until the first
 whitespace is used as indicator::
@@ -302,6 +306,8 @@ Parameters
 :fields: list of JSON attributes to include in the indicator value.
     If *null* no additional attributes are extracted. Default: *null*
 :prefix: prefix to add to field names. Default: json
+:headers: a dictionary of additional headers to add to the HTTP
+    header. Default: {}
 
 Example
 +++++++
@@ -315,5 +321,7 @@ Example config in YAML::
     fields:
         - region
         - service
+    headers:
+        api-key: sample-api-key
 
 For a complete config example check **aws.AMAZON** prototype.

minemeld/ft/http.py

@@ -51,6 +51,9 @@ class HttpFT(basepoller.BasePollerFT):
         :encoding: encoding of the feed, if not UTF-8. See
             ``str.decode`` for options. Default: *null*, meaning do
             nothing, (Assumes UTF-8).
+        :headers: Header parameters are optional to sepcify a user-agent or an api-token
+        Example: headers = {'user-agent': 'my-app/0.0.1'} or Authorization: Bearer 
+        (curl -H "Authorization: Bearer " "https://api-url.com/api/v1/iocs?first_seen_since=2016-1-1")
 
     **Extraction dictionary**
         Extraction dictionaries contain the following keys:
@@ -109,6 +112,8 @@ def configure(self):
         self.username = self.config.get('username', None)
         self.password = self.config.get('password', None)
 
+        self.headers = self.config.get('headers', None)
+
         self.ignore_regex = self.config.get('ignore_regex', None)
         if self.ignore_regex is not None:
             self.ignore_regex = re.compile(self.ignore_regex)
@@ -196,6 +201,10 @@ def _build_iterator(self, now):
         if self.username is not None and self.password is not None:
             rkwargs['auth'] = (self.username, self.password)
 
+        if self.headers is not None:
+            for key value in self.headers.items():
+                rkwargs[key] = value
+
         r = requests.get(
             self.url,
             **rkwargs