Merge pull request #719 from mozilla/dns_blocklist

Add FQDN/DNS block page

cron/createFDQNBlockList.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2014 Mozilla Corporation
+
+import boto
+import boto.s3
+import logging
+import random
+import re
+import sys
+from datetime import datetime
+from datetime import timedelta
+from configlib import getConfig, OptionParser
+from logging.handlers import SysLogHandler
+from pymongo import MongoClient
+
+import os
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../lib'))
+from utilities.toUTC import toUTC
+
+
+logger = logging.getLogger(sys.argv[0])
+
+
+def loggerTimeStamp(self, record, datefmt=None):
+    return toUTC(datetime.now()).isoformat()
+
+def initLogger():
+    logger.level = logging.INFO
+    formatter = logging.Formatter(
+        '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    formatter.formatTime = loggerTimeStamp
+    if options.output == 'syslog':
+        logger.addHandler(
+            SysLogHandler(
+                address=(options.sysloghostname, options.syslogport)))
+    else:
+        sh = logging.StreamHandler(sys.stderr)
+        sh.setFormatter(formatter)
+        logger.addHandler(sh)
+
+def genMeteorID():
+    return('%024x' % random.randrange(16**24))
+
+def isFQDN(fqdn):
+    try:
+        # We could resolve FQDNs here, but that could tip our hand and it's
+        # possible us investigating could trigger other alerts.
+        # validate using the regex from https://github.com/yolothreat/utilitybelt
+        fqdn_re = re.compile('(?=^.{4,255}$)(^((?!-)[a-zA-Z0-9-]{1,63}(?<!-)\.)+[a-zA-Z]{2,63}$)', re.I | re.S | re.M)
+        return bool(re.match(fqdn_re,fqdn))
+    except:
+        return False
+
+def parse_fqdn_whitelist(fqdn_whitelist_location):
+    fqdns = []
+    with open(fqdn_whitelist_location, "r") as text_file:
+        for line in text_file:
+            line=line.strip().strip("'").strip('"')
+            if isFQDN(line):
+                fqdns.append(line)
+    return fqdns
+
+def main():
+    logger.debug('starting')
+    logger.debug(options)
+    try:
+        # connect to mongo
+        client = MongoClient(options.mongohost, options.mongoport)
+        mozdefdb = client.meteor
+        fqdnblocklist = mozdefdb['fqdnblocklist']
+        # ensure indexes
+        fqdnblocklist.create_index([('dateExpiring',-1)])
+
+        # delete any that expired
+        fqdnblocklist.delete_many({'dateExpiring': {"$lte": datetime.utcnow()-timedelta(days=options.expireage)}})
+
+        # Lastly, export the combined blocklist
+        fqdnCursor=mozdefdb['fqdnblocklist'].aggregate([
+                {"$sort": {"dateAdded": -1}},
+                {"$match": {"address": {"$exists": True}}},
+                {"$match":
+                    {"$or":[
+                        {"dateExpiring": {"$gte": datetime.utcnow()}},
+                        {"dateExpiring": {"$exists": False}},
+                    ]},
+                },
+                {"$project":{"address":1}},
+                {"$limit": options.fqdnlimit}
+            ])
+        FQDNList=[]
+        for fqdn in fqdnCursor:
+            if fqdn not in options.fqdnwhitelist:
+                FQDNList.append(fqdn['address'])
+        # to text
+        with open(options.outputfile, 'w') as outputfile:
+            for fqdn in FQDNList:
+                outputfile.write("{0}\n".format(fqdn))
+        outputfile.close()
+        # to s3?
+        if len(options.aws_bucket_name)>0:
+            s3_upload_file(options.outputfile, options.aws_bucket_name, options.aws_document_key_name)
+
+
+    except ValueError as e:
+        logger.error("Exception %r generating FQDN block list" % e)
+
+
+def initConfig():
+    # output our log to stdout or syslog
+    options.output = getConfig('output', 'stdout', options.configfile)
+    # syslog hostname
+    options.sysloghostname = getConfig('sysloghostname',
+                                       'localhost',
+                                       options.configfile)
+    # syslog port
+    options.syslogport = getConfig('syslogport', 514, options.configfile)
+
+    # mongo instance
+    options.mongohost = getConfig('mongohost', 'localhost', options.configfile)
+    options.mongoport = getConfig('mongoport', 3001, options.configfile)
+
+    # FQDN whitelist as a \n separted file of example.com or foo.bar.com style names
+    options.fqdn_whitelist_file = getConfig('fqdn_whitelist_file', '/dev/null', options.configfile)
+    options.fqdnwhitelist = parse_fqdn_whitelist(options.fqdn_whitelist_file)
+
+    # Output File Name
+    options.outputfile = getConfig('outputfile', 'fqdnblocklist.txt', options.configfile)
+
+    # Days after expiration that we purge an fqdnblocklist entry (from the ui, they don't end up in the export after expiring)
+    options.expireage = getConfig('expireage',1,options.configfile)
+
+    # Max FQDNs to emit
+    options.fqdnlimit = getConfig('fqdnlimit', 1000, options.configfile)
+
+    # AWS creds
+    options.aws_access_key_id=getConfig('aws_access_key_id','',options.configfile)          #aws credentials to use to connect to mozilla_infosec_blocklist
+    options.aws_secret_access_key=getConfig('aws_secret_access_key','',options.configfile)
+    options.aws_bucket_name=getConfig('aws_bucket_name','',options.configfile)
+    options.aws_document_key_name=getConfig('aws_document_key_name','',options.configfile)
+
+
+def s3_upload_file(file_path, bucket_name, key_name):
+    """
+    Upload a file to the given s3 bucket and return a template url.
+    """
+    conn = boto.connect_s3(aws_access_key_id=options.aws_access_key_id,aws_secret_access_key=options.aws_secret_access_key)
+    try:
+        bucket = conn.get_bucket(bucket_name, validate=False)
+    except boto.exception.S3ResponseError as e:
+        conn.create_bucket(bucket_name)
+        bucket = conn.get_bucket(bucket_name, validate=False)
+
+
+    key = boto.s3.key.Key(bucket)
+    key.key = key_name
+    key.set_contents_from_filename(file_path)
+
+    key.set_acl('public-read')
+    url = "https://s3.amazonaws.com/{}/{}".format(bucket.name, key.name)
+    print( "URL: {}".format(url))
+    return url
+
+if __name__ == '__main__':
+    parser = OptionParser()
+    parser.add_option(
+        "-c",
+        dest='configfile',
+        default=sys.argv[0].replace('.py', '.conf'),
+        help="configuration file to use")
+    (options, args) = parser.parse_args()
+    initConfig()
+    initLogger()
+    main()

cron/createFQDNBlockList.conf

@@ -0,0 +1,9 @@
+[options]
+mongoport = 3002
+outputfile = /opt/mozdef/envs/mozdef/static/fqdnblocklist.txt
+fqdn_whitelist_file = /opt/mozdef/envs/mozdef/static/fqdnlist.txt
+aws_access_key_id = <add_aws_access_key_id>
+aws_secret_access_key = <add_aws_secret_access_key>
+aws_bucket_name = <add_aws_bucket_name>
+aws_document_key_name = fqdnblocklist
+iplimit = 3000

cron/createFQDNBlockList.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2014 Mozilla Corporation
+
+source  /opt/mozdef/envs/mozdef/bin/activate
+/opt/mozdef/envs/mozdef/cron/createFQDNBlockList.py -c /opt/mozdef/envs/mozdef/cron/createFQDNBlockList.conf
+

meteor/app/client/blockFDQN.js

@@ -0,0 +1,63 @@
+/*
+This Source Code Form is subject to the terms of the Mozilla Public
+License, v. 2.0. If a copy of the MPL was not distributed with this
+file, You can obtain one at http://mozilla.org/MPL/2.0/.
+Copyright (c) 2014 Mozilla Corporation
+ */
+
+if (Meteor.isClient) {
+
+    Template.blockFQDNform.rendered = function() {
+        $('#fqdn')[0].value = Session.get('blockFQDN');
+    };
+
+    Template.blockFQDNform.events({
+        "submit form": function(event, template) {
+            event.preventDefault();
+            formobj=formToObject("#blockFQDN :input");
+            formobj.push({userid:Meteor.user().profile.email});
+            Meteor.call('blockfqdn',formobj,
+                onResultReceived = function(err,result){
+                    if (typeof err == 'undefined') {
+                        //console.log(result);
+                        if ( result == true){
+                            Session.set('displayMessage','blocked & successfully ');
+                        }else{
+                            Session.set('errorMessage','block failed, returned & ' + JSON.stringify(result) );
+                        }
+                    }else{
+                        Session.set('errorMessage','block failed & ' + JSON.stringify(err));
+                    }
+                });
+            Router.go('/fqdnblocklist');
+        }
+    });
+
+    Template.blockFQDNModal.rendered = function(){
+        Deps.autorun(function() {
+            $('#fqdn')[0].value = Session.get('blockFQDN');
+        }); //end deps.autorun
+    };
+
+    Template.blockFQDNModal.events({
+        "submit form": function(event, template) {
+            event.preventDefault();
+            formobj=formToObject("#blockFQDNform :input");
+            formobj.push({userid:Meteor.user().profile.email});
+            Meteor.call('blockfqdn',formobj,
+                onResultReceived = function(err,result){
+                    if (typeof err == 'undefined') {
+                        //console.log(result);
+                        if ( result == true){
+                            Session.set('displayMessage','blocked & successfully ');
+                        }else{
+                            Session.set('errorMessage','block failed, returned & ' + JSON.stringify(result) );
+                        }
+                    }else{
+                        Session.set('errorMessage','block failed & ' + JSON.stringify(err));
+                    }
+                });
+            $('#modalBlockFQDNWindow').modal('hide')
+        }
+    });
+}