diff --git a/minemeld/ft/basepoller.py b/minemeld/ft/basepoller.py index 5bc48c7c..f2abb296 100644 --- a/minemeld/ft/basepoller.py +++ b/minemeld/ft/basepoller.py @@ -366,6 +366,7 @@ def _polling_loop(self): raise except: + self.statistics['error.parsing'] += 1 LOG.exception('%s - Exception parsing %s', self.name, item) continue diff --git a/minemeld/ft/taxii.py b/minemeld/ft/taxii.py index c47e91b4..d7cb3112 100644 --- a/minemeld/ft/taxii.py +++ b/minemeld/ft/taxii.py @@ -27,6 +27,7 @@ import gevent import gevent.event import netaddr +import werkzeug.urls import libtaxii import libtaxii.clients @@ -835,11 +836,16 @@ def _add_indicator(self, score, indicator, value): uuid.uuid4() ) + if value['type'] == 'URL': + eindicator = werkzeug.urls.iri_to_uri(indicator, safe_conversion=True) + else: + eindicator = indicator + sindicator = stix.indicator.indicator.Indicator( id_=id_, title='{}: {}'.format( value['type'], - indicator + eindicator ), description='{} indicator from {}'.format( value['type'], diff --git a/tests/test_ft_taxii.py b/tests/test_ft_taxii.py index 40570bfd..02b666ad 100644 --- a/tests/test_ft_taxii.py +++ b/tests/test_ft_taxii.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -385,3 +387,56 @@ def test_datafeed_update_url(self, glet_mock, SR_mock): SR_mock.reset_mock() b.stop() + + @mock.patch.object(redis, 'StrictRedis') + @mock.patch.object(gevent, 'Greenlet') + def test_datafeed_unicode_url(self, glet_mock, SR_mock): + config = {} + chassis = mock.Mock() + + chassis.request_sub_channel.return_value = None + ochannel = mock.Mock() + chassis.request_pub_channel.return_value = ochannel + chassis.request_rpc_channel.return_value = None + rpcmock = mock.Mock() + rpcmock.get.return_value = {'error': None, 'result': 'OK'} + chassis.send_rpc.return_value = rpcmock + + b = minemeld.ft.taxii.DataFeed(FTNAME, chassis, config) + + inputs = ['a'] + output = False + + b.connect(inputs, output) + b.mgmtbus_initialize() + + b.start() + # __init__ + get chkp + delete chkp + self.assertEqual(len(SR_mock.mock_calls), 5) + SR_mock.reset_mock() + + # unicast + b.update( + 'a', + indicator=u'☃.net/påth', + value={ + 'type': 'URL', + 'confidence': 100, + 'share_level': 'green', + 'sources': ['test.1'] + } + ) + for call in SR_mock.mock_calls: + name, args, kwargs = call + if name == '().pipeline().__enter__().hset': + break + else: + self.fail(msg='hset not found') + + stixdict = xmltodict.parse(args[2]) + indicator = stixdict['stix:STIX_Package']['stix:Indicators']['stix:Indicator'] + cyboxprops = indicator['indicator:Observable']['cybox:Object']['cybox:Properties'] + self.assertEqual(cyboxprops['URIObj:Value'], u'\u2603.net/p\xe5th') + SR_mock.reset_mock() + + b.stop()