Merge pull request #1342 from mozilla/python_3_upgrade

Python 3 upgrade

.travis.yml

@@ -1,6 +1,6 @@
 language: python
 python:
-  - "2.7"
+  - "3.6"
 sudo: required
 services:
   - docker

CHANGELOG

@@ -5,6 +5,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased]
 
+## [v3.0.0] - 2019-07-08
+### Added
+- Support for Python3
+
+### Removed
+- Support for Python2
+- Usage of boto (boto3 now preferred)
+
+
 ## [v2.0.1] - 2019-07-08
 ### Fixed
 - Ensure all print statements use parenthesis
@@ -123,7 +132,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added checks on sending SQS messages to only accept intra-account messages
 - Improved docker performance and disk space requirements
 
-[Unreleased]: https://github.com/mozilla/MozDef/compare/v2.0.1...HEAD
+[Unreleased]: https://github.com/mozilla/MozDef/compare/v3.0.0...HEAD
+[v3.0.0]: https://github.com/mozilla/MozDef/compare/v2.0.1...v3.0.0
 [v2.0.1]: https://github.com/mozilla/MozDef/compare/v2.0.0...v2.0.1
 [v2.0.0]: https://github.com/mozilla/MozDef/compare/v1.40.0...v2.0.0
 [v1.40.0]: https://github.com/mozilla/MozDef/compare/v1.40.0...v1.39.0

Makefile

@@ -66,6 +66,7 @@ build: build-from-cwd
 
 .PHONY: build-from-cwd
 build-from-cwd:  ## Build local MozDef images (use make NO_CACHE=--no-cache build to disable caching)
+	docker-compose -f docker/compose/docker-compose.yml -p $(NAME) $(BUILD_MODE) $(PARALLEL) $(NO_CACHE) base
 	docker-compose -f docker/compose/docker-compose.yml -p $(NAME) $(BUILD_MODE) $(PARALLEL) $(NO_CACHE)
 
 .PHONY: build-from-github
@@ -76,6 +77,7 @@ build-from-github:  ## Build local MozDef images from the github branch (use mak
 
 .PHONY: build-tests
 build-tests:  ## Build end-to-end test environment only
+	docker-compose -f docker/compose/docker-compose-tests.yml -p test-$(NAME) $(NO_CACHE) $(BUILD_MODE) base
 	docker-compose -f docker/compose/docker-compose-tests.yml -p test-$(NAME) $(NO_CACHE) $(BUILD_MODE)
 
 .PHONY: stop

alerts/actions/dashboard_geomodel.py

@@ -85,30 +85,30 @@ def onMessage(self, message):
             whois = IPWhois(source_ip).lookup_whois()
             whois_str = whois['nets'][0]['description']
             source_ip_isp = whois_str.replace('\n', ', ').replace('\r', '')
-            new_ip_info = u'{} ({})'.format(source_ip, source_ip_isp)
+            new_ip_info = '{} ({})'.format(source_ip, source_ip_isp)
         except Exception:
-            new_ip_info = u'{}'.format(source_ip)
+            new_ip_info = '{}'.format(source_ip)
 
-        new_location_str = u""
+        new_location_str = ""
         if city.lower() == 'unknown':
-            new_location_str += u'{0}'.format(country)
+            new_location_str += '{0}'.format(country)
         else:
-            new_location_str += u'{0}, {1}'.format(city, country)
+            new_location_str += '{0}, {1}'.format(city, country)
 
         event_timestamp = toUTC(message['events'][0]['documentsource']['details']['event_time'])
         event_day = event_timestamp.strftime('%B %d, %Y')
-        summary = u'On {0} (UTC), did you login from {1} ({2})?'.format(event_day, new_location_str, source_ip)
+        summary = 'On {0} (UTC), did you login from {1} ({2})?'.format(event_day, new_location_str, source_ip)
 
         previous_city = message['details']['previous_locality_details']['city']
         previous_country = message['details']['previous_locality_details']['country']
         if previous_city.lower() == 'unknown':
-            previous_location_str = u'{0}'.format(previous_country)
+            previous_location_str = '{0}'.format(previous_country)
         else:
-            previous_location_str = u'{0}, {1}'.format(previous_city, previous_country)
+            previous_location_str = '{0}, {1}'.format(previous_city, previous_country)
 
         alert_record = {
-            'alert_id': b2a_hex(os.urandom(15)),
-            'alert_code': b2a_hex(self.alert_classname),
+            'alert_id': b2a_hex(os.urandom(15)).decode(),
+            'alert_code': b2a_hex(self.alert_classname.encode()).decode(),
             'user_id': auth_full_username,
             'risk': self.config['risk'],
             'summary': summary,

alerts/alert_actions_worker.py

@@ -47,7 +47,7 @@ def on_message(self, body, message):
             # just to be safe..check what we were sent.
             if isinstance(body, dict):
                 bodyDict = body
-            elif isinstance(body, str) or isinstance(body, unicode):
+            elif isinstance(body, str):
                 try:
                     bodyDict = json.loads(body)  # lets assume it's json
                 except ValueError as e:

alerts/celeryconfig.py

@@ -72,7 +72,7 @@
         alert_class = getattr(alert_module, alert_classname)
         app.register_task(alert_class())
     except ImportError as e:
-        print("Error importing {}").format(alert_namespace)
+        print("Error importing {}".format(alert_namespace))
         print(e)
         pass
     except Exception as e:

alerts/feedback_events.py

@@ -31,9 +31,9 @@ def onEvent(self, event):
         user = event['_source']['details']['alert_information']['user_id']
         event_summary = event['_source']['summary']
         event_date = event['_source']['details']['alert_information']['date']
-        summary = u"{} escalated alert within single-sign on (SSO) dashboard. Event Date: {} Summary: \"{}\"".format(user, event_date, event_summary)
+        summary = "{} escalated alert within single-sign on (SSO) dashboard. Event Date: {} Summary: \"{}\"".format(user, event_date, event_summary)
 
-        for alert_code, tag in self._config.iteritems():
+        for alert_code, tag in self._config.items():
             if event['_source']['details']['alert_information']['alert_code'] == alert_code:
                 tags.append(tag)
 

alerts/lib/alert_plugin_set.py

@@ -6,7 +6,7 @@ class AlertPluginSet(PluginSet):
 
     def send_message_to_plugin(self, plugin_class, message, metadata=None):
         if 'utctimestamp' in message and 'summary' in message:
-            message_log_str = u'{0} received message: ({1}) {2}'.format(plugin_class.__module__, message['utctimestamp'], message['summary'])
+            message_log_str = '{0} received message: ({1}) {2}'.format(plugin_class.__module__, message['utctimestamp'], message['summary'])
             logger.info(message_log_str)
 
         return plugin_class.onMessage(message), metadata

alerts/lib/alerttask.py

@@ -18,12 +18,14 @@
 from collections import Counter
 from celery import Task
 from celery.utils.log import get_task_logger
-from config import RABBITMQ, ES, ALERT_PLUGINS
 
 from mozdef_util.utilities.toUTC import toUTC
 from mozdef_util.elasticsearch_client import ElasticsearchClient
 from mozdef_util.query_models import TermMatch, ExistsMatch
 
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../"))
+from lib.config import RABBITMQ, ES, ALERT_PLUGINS
+
 sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))
 from lib.alert_plugin_set import AlertPluginSet
 
@@ -35,7 +37,7 @@ def keypaths(nested):
     """ return a list of nested dict key paths
         like: [u'_source', u'details', u'program']
     """
-    for key, value in nested.iteritems():
+    for key, value in nested.items():
         if isinstance(value, collections.Mapping):
             for subkey, subvalue in keypaths(value):
                 yield [key] + subkey, subvalue
@@ -123,6 +125,9 @@ def parse_config(self, config_filename, config_keys):
             temp_value = getConfig(config_key, "", config_filename)
             setattr(self.config, config_key, temp_value)
 
+    def close_connections(self):
+        self.mqConn.release()
+
     def _discover_task_exchange(self):
         """Use configuration information to understand the message queue protocol.
         return: amqp, sqs
@@ -344,11 +349,7 @@ def searchEventsAggregated(self, aggregationPath, samplesLimit=5):
             for i in Counter(aggregationValues).most_common():
                 idict = {"value": i[0], "count": i[1], "events": [], "allevents": []}
                 for r in results:
-                    if (
-                        getValueByPath(r["_source"], aggregationPath).encode(
-                            "ascii", "ignore"
-                        ) == i[0]
-                    ):
+                    if getValueByPath(r["_source"], aggregationPath) == i[0]:
                         # copy events detail into this aggregation up to our samples limit
                         if len(idict["events"]) < samplesLimit:
                             idict["events"].append(r)
@@ -510,11 +511,9 @@ def tagEventsAlert(self, events, alertResultES):
                     event["_source"]["alert_names"] = []
                 event["_source"]["alert_names"].append(self.determine_alert_classname())
 
-                self.es.save_event(
-                    index=event["_index"], body=event["_source"], doc_id=event["_id"]
-                )
-            # We refresh here to ensure our changes to the events will show up for the next search query results
-            self.es.refresh(event["_index"])
+                self.es.save_event(index=event["_index"], body=event["_source"], doc_id=event["_id"])
+                # We refresh here to ensure our changes to the events will show up for the next search query results
+                self.es.refresh(event["_index"])
         except Exception as e:
             self.log.error("Error while updating events in ES: {0}".format(e))
 

alerts/lib/deadman_alerttask.py

@@ -1,9 +1,10 @@
-from alerttask import AlertTask
+from .alerttask import AlertTask
 
 
 class DeadmanAlertTask(AlertTask):
 
     def executeSearchEventsSimple(self):
         # We override this method to specify the size as 1
         # since we only care about if ANY events are found or not
-        return self.main_query.execute(self.es, indices=self.event_indices, size=1)
+        results = self.main_query.execute(self.es, indices=self.event_indices, size=1)
+        return results

alerts/plugins/ip_source_enrichment.py

@@ -8,6 +8,7 @@
 import os
 import re
 
+import functools
 import netaddr
 
 
@@ -23,10 +24,11 @@ def _find_ip_addresses(string):
     ipv6_rx = '(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))'
 
     ipv4 = re.findall(ipv4_rx, string)
-    ipv6 = map(
+    ipv6_map = map(
         lambda match: match[0] if isinstance(match, tuple) else match,
         re.findall(ipv6_rx, string))
 
+    ipv6 = [x for x in ipv6_map]
     return ipv4 + ipv6
 
 
@@ -42,11 +44,11 @@ def find_ips(value):
 
         if isinstance(value, list) or isinstance(value, tuple):
             found = [find_ips(item) for item in value]
-            return reduce(add, found, [])
+            return functools.reduce(add, found, [])
 
         if isinstance(value, dict):
             found = [find_ips(item) for item in value.values()]
-            return reduce(add, found, [])
+            return functools.reduce(add, found, [])
 
         return []
 

alerts/ssh_access_signreleng.py

@@ -28,7 +28,7 @@ def main(self):
 
         for exclusion in self.config['exclusions']:
             exclusion_query = None
-            for key, value in exclusion.iteritems():
+            for key, value in exclusion.items():
                 phrase_exclusion = PhraseMatch(key, value)
                 if exclusion_query is None:
                     exclusion_query = phrase_exclusion

benchmarking/workers/json2Mozdef.py

@@ -15,7 +15,7 @@
 from requests_futures.sessions import FuturesSession
 from multiprocessing import Process, Queue
 import logging
-from Queue import Empty
+from queue import Empty
 from requests.packages.urllib3.exceptions import ClosedPoolError
 import time
 

bot/irc/mozdefbot.py

@@ -147,7 +147,7 @@ def formatAlert(jsonDictIn):
     return colorify('{0}: {1} {2}'.format(
         severity,
         colors['blue'] + category + colors['normal'],
-        summary.encode('ascii', 'replace')
+        summary
     ))
 
 
@@ -219,7 +219,7 @@ def priv_handler(client, actor, recipient, message):
                             ip = netaddr.IPNetwork(field)[0]
                             if (not ip.is_loopback() and not ip.is_private() and not ip.is_reserved()):
                                 whois = IPWhois(ip).lookup_whois()
-                                description = whois['nets'][0]['description'].encode('string_escape')
+                                description = whois['nets'][0]['description']
                                 self.client.msg(
                                     recipient, "{0} description: {1}".format(field, description))
                             else:
@@ -290,7 +290,7 @@ def on_message(self, body, message):
             # just to be safe..check what we were sent.
             if isinstance(body, dict):
                 bodyDict = body
-            elif isinstance(body, str) or isinstance(body, unicode):
+            elif isinstance(body, str):
                 try:
                     bodyDict = json.loads(body)  # lets assume it's json
                 except ValueError as e:
@@ -402,7 +402,7 @@ def initConfig():
 
     # Our config parser stomps out the '#' so we gotta readd
     channelkeys = {}
-    for key, value in options.channelkeys.iteritems():
+    for key, value in options.channelkeys.items():
         if not key.startswith('#'):
             key = '#{0}'.format(key)
         channelkeys[key] = value

bot/slack/bot_plugin_set.py

@@ -1,5 +1,6 @@
 import os
 import pynsive
+import importlib
 
 from mozdef_util.utilities.logger import logger
 
@@ -32,7 +33,7 @@ def identify_plugins(self, enabled_plugins):
                 continue
 
             module_obj = pynsive.import_module(found_module)
-            reload(module_obj)
+            importlib.reload(module_obj)
             plugin_class_obj = module_obj.Command()
             logger.info('Plugin {0} registered to receive command with {1}'.format(module_name, plugin_class_obj.command_name))
             plugins.append(

bot/slack/commands/ip_whois.py

@@ -16,7 +16,7 @@ def handle_command(self, parameters):
                 ip = netaddr.IPNetwork(ip_token)[0]
                 if (not ip.is_loopback() and not ip.is_private() and not ip.is_reserved()):
                     whois = IPWhois(ip).lookup_whois()
-                    description = str(whois['nets'][0]['description']).encode('string_escape')
+                    description = whois['nets'][0]['description']
                     response += "{0} description: {1}\n".format(ip_token, description)
                 else:
                     response += "{0}: hrm...loopback? private ip?\n".format(ip_token)

bot/slack/mozdefbot.py

@@ -49,7 +49,7 @@ def on_message(self, body, message):
             # just to be safe..check what we were sent.
             if isinstance(body, dict):
                 body_dict = body
-            elif isinstance(body, str) or isinstance(body, unicode):
+            elif isinstance(body, str):
                 try:
                     body_dict = json.loads(body)  # lets assume it's json
                 except ValueError as e:

bot/slack/slack_bot.py

@@ -50,7 +50,7 @@ def delegate_command(self, message_text):
 
         if command == '!help':
             response = "\nHelp is on it's way...try these:\n"
-            for command_name, plugin in self.plugins.iteritems():
+            for command_name, plugin in self.plugins.items():
                 response += "\n{0} -- {1}".format(
                     command_name,
                     plugin['help_text']

cloudy_mozdef/cloudformation/mozdef-alert-developer.yml

@@ -23,7 +23,7 @@ Resources:
                 S3Bucket: public.us-west-2.security.allizom.org
                 S3Key: mozdef-lambda-layer/layer-latest.zip
             CompatibleRuntimes:
-              - python2.7
+              - python3.6
             LicenseInfo: 'MPL 2.0'
     LambdalertIAMRole:
       Type: AWS::IAM::Role
@@ -36,16 +36,16 @@ Resources:
                 Service: lambda.amazonaws.com
               Action: sts:AssumeRole
         ManagedPolicyArns:
-          -  arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole 
-    AlertWritersEnv: 
+          -  arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole
+    AlertWritersEnv:
       Type: "AWS::Lambda::Function"
-      Properties: 
+      Properties:
         Handler: "lambdalert.handle"
-        Role: 
-          Fn::GetAtt: 
+        Role:
+          Fn::GetAtt:
             - "LambdalertIAMRole"
             - "Arn"
-        Code: 
+        Code:
           S3Bucket: public.us-west-2.security.allizom.org
           S3Key: mozdef-lambda-layer/function-latest.zip
         Layers:
@@ -55,9 +55,9 @@ Resources:
             OPTIONS_ESSERVERS: !Ref ESUrl
             OPTIONS_MQPROTOCOL: sqs
         VpcConfig:
-          SecurityGroupIds: 
+          SecurityGroupIds:
             - !Ref MozDefSecurityGroup
           SubnetIds: !Ref PublicSubnetIds
         ReservedConcurrentExecutions: 1
-        Runtime: "python2.7"
+        Runtime: "python3.6"
         Timeout: 120