Merge pull request PaloAltoNetworks#83 from jtschichold/fix-77

Fixes PaloAltoNetworks#77

minemeld/ft/basepoller.py

@@ -366,6 +366,7 @@ def _polling_loop(self):
                     raise
 
                 except:
+                    self.statistics['error.parsing'] += 1
                     LOG.exception('%s - Exception parsing %s', self.name, item)
                     continue
 

minemeld/ft/taxii.py

@@ -27,6 +27,7 @@
 import gevent
 import gevent.event
 import netaddr
+import werkzeug.urls
 
 import libtaxii
 import libtaxii.clients
@@ -835,11 +836,16 @@ def _add_indicator(self, score, indicator, value):
                 uuid.uuid4()
             )
 
+            if value['type'] == 'URL':
+                eindicator = werkzeug.urls.iri_to_uri(indicator, safe_conversion=True)
+            else:
+                eindicator = indicator
+
             sindicator = stix.indicator.indicator.Indicator(
                 id_=id_,
                 title='{}: {}'.format(
                     value['type'],
-                    indicator
+                    eindicator
                 ),
                 description='{} indicator from {}'.format(
                     value['type'],

tests/test_ft_taxii.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 #  Copyright 2016 Palo Alto Networks, Inc
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -385,3 +387,56 @@ def test_datafeed_update_url(self, glet_mock, SR_mock):
         SR_mock.reset_mock()
 
         b.stop()
+
+    @mock.patch.object(redis, 'StrictRedis')
+    @mock.patch.object(gevent, 'Greenlet')
+    def test_datafeed_unicode_url(self, glet_mock, SR_mock):
+        config = {}
+        chassis = mock.Mock()
+
+        chassis.request_sub_channel.return_value = None
+        ochannel = mock.Mock()
+        chassis.request_pub_channel.return_value = ochannel
+        chassis.request_rpc_channel.return_value = None
+        rpcmock = mock.Mock()
+        rpcmock.get.return_value = {'error': None, 'result': 'OK'}
+        chassis.send_rpc.return_value = rpcmock
+
+        b = minemeld.ft.taxii.DataFeed(FTNAME, chassis, config)
+
+        inputs = ['a']
+        output = False
+
+        b.connect(inputs, output)
+        b.mgmtbus_initialize()
+
+        b.start()
+        # __init__ + get chkp + delete chkp
+        self.assertEqual(len(SR_mock.mock_calls), 5)
+        SR_mock.reset_mock()
+
+        # unicast
+        b.update(
+            'a',
+            indicator=u'☃.net/påth',
+            value={
+                'type': 'URL',
+                'confidence': 100,
+                'share_level': 'green',
+                'sources': ['test.1']
+            }
+        )
+        for call in SR_mock.mock_calls:
+            name, args, kwargs = call
+            if name == '().pipeline().__enter__().hset':
+                break
+        else:
+            self.fail(msg='hset not found')
+
+        stixdict = xmltodict.parse(args[2])
+        indicator = stixdict['stix:STIX_Package']['stix:Indicators']['stix:Indicator']
+        cyboxprops = indicator['indicator:Observable']['cybox:Object']['cybox:Properties']
+        self.assertEqual(cyboxprops['URIObj:Value'], u'\u2603.net/p\xe5th')
+        SR_mock.reset_mock()
+
+        b.stop()