From 23e8fa6256758c736765e4fbad68c3b0e3bebf06 Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Tue, 13 Dec 2016 14:51:24 -0500 Subject: [PATCH 01/20] asynchronouns bap This is a big commit, that severely rewrites many aspects of our IDA intergration. 1. BAP now is run asynchronounsly 2. Multiple instances of BAP can be run at once 3. Task Manager is provided for a primitive job control 4. A view is created for each BAP instance 5. View Manager helps to switch between different views 6. Each plugin can be run from the Edit/Plugins menu 7. Each plugin can be run from Ctrl-3 (fast plugin) menu --- plugins/bap/plugins/bap_bir_attr.py | 142 +++-- plugins/bap/plugins/bap_clear_comments.py | 40 ++ plugins/bap/plugins/bap_functions.py | 87 ++-- plugins/bap/plugins/bap_taint.py | 123 +++-- plugins/bap/plugins/bap_taint_ptr.py | 10 + plugins/bap/plugins/bap_taint_reg.py | 10 + plugins/bap/plugins/bap_task_manager.py | 70 +++ plugins/bap/plugins/bap_view.py | 174 ++++--- plugins/bap/plugins/pseudocode_bap_comment.py | 7 +- plugins/bap/plugins/pseudocode_bap_taint.py | 14 +- plugins/bap/utils/abstract_ida_plugins.py | 4 +- plugins/bap/utils/config.py | 77 ++- plugins/bap/utils/ida.py | 46 +- plugins/bap/utils/run.py | 492 ++++++++++++------ plugins/bap/utils/sexpr.py | 21 +- 15 files changed, 840 insertions(+), 477 deletions(-) create mode 100644 plugins/bap/plugins/bap_clear_comments.py create mode 100644 plugins/bap/plugins/bap_taint_ptr.py create mode 100644 plugins/bap/plugins/bap_taint_reg.py create mode 100644 plugins/bap/plugins/bap_task_manager.py diff --git a/plugins/bap/plugins/bap_bir_attr.py b/plugins/bap/plugins/bap_bir_attr.py index e6264eb..98d05f8 100644 --- a/plugins/bap/plugins/bap_bir_attr.py +++ b/plugins/bap/plugins/bap_bir_attr.py @@ -14,7 +14,31 @@ with the format (BAP (k1 v1) (k2 v2) ...) """ import idautils - +from bap.utils.run import BapIda + +class BapScripter(BapIda): + + def __init__(self, user_args, attrs): + super(BapScripter,self).__init__() + if attrs != []: + self.action = 'extracting ' + ','.join(attrs) + else: + self.action = 'running bap ' + user_args + self.script = self.tmpfile('py') + self.args += user_args.split(' ') + self.args.append('--emit-ida-script') + self.args += [ + '--emit-ida-script-file', self.script.name + ] + self.args += [ + '--emit-ida-script-attr='+attr.strip() + for attr in attrs + ] + + +# perfectly random numbers +ARGS_HISTORY=324312 +ATTR_HISTORY=234345 class BAP_BIR_Attr(idaapi.plugin_t): """ @@ -22,109 +46,70 @@ class BAP_BIR_Attr(idaapi.plugin_t): Also supports installation of callbacks using install_callback() """ + flags = 0 + comment = "Run BAP " + help = "Runs BAP and extracts data from the output" + wanted_name = "BAP: Run" + wanted_hotkey = "Shift-S" _callbacks = [] - @classmethod - def _do_callbacks(cls): - data = { - 'ea': idc.ScreenEA(), - } - for callback in cls._callbacks: - callback(data) + recipes = {} - @classmethod - def run_bap(cls): + def _do_callbacks(self,ea): + for callback in BAP_BIR_Attr._callbacks: + callback({'ea': ea}) + + def run(self, arg): """ Ask user for BAP args to pass, BIR attributes to print; and run BAP. Allows users to also use {screen_ea} in the BAP args to get the address at the location pointed to by the cursor. """ - import tempfile - from bap.utils.run import run_bap_with - - args = { - 'screen_ea': "0x{:X}".format(idc.ScreenEA()), - 'ida_script_location': tempfile.mkstemp(suffix='.py', - prefix='ida-bap-')[1], - 'args_from_user': idaapi.askstr(0, '', 'Args to pass to BAP'), - 'bir_attr': idaapi.askstr(0, 'comment', - 'BIR Attributes (comma separated)') - } - - if args['args_from_user'] is None: - args['args_from_user'] = '' - - if args['bir_attr'] is not None: - for attr in args['bir_attr'].split(','): - attr = attr.strip() # For users who prefer "x, y, z" style - args['args_from_user'] += " --emit-ida-script-attr=" + attr - args['args_from_user'] += "\ - --emit-ida-script-file={ida_script_location} \ - --emit-ida-script \ - ".format(**args) - - idc.SetStatus(IDA_STATUS_WAITING) - idaapi.refresh_idaview_anyway() - - run_bap_with(args['args_from_user'].format(**args)) - - idc.SetStatus(IDA_STATUS_READY) - - idaapi.IDAPython_ExecScript(args['ida_script_location'], globals()) - idc.Exec("rm -f \"{ida_script_location}\"".format(**args)) # Cleanup + args_msg = "Arguments that will be passed to `bap'" - idc.Refresh() # Force the updated information to show up - - cls._do_callbacks() + args = idaapi.askstr(ARGS_HISTORY, '--passes=', args_msg) + if args is None: + return + attr_msg = "A comma separated list of attributes,\n" + attr_msg += "that should be propagated to comments" + attr_def = BAP_BIR_Attr.recipes.get(args, '') + attr = idaapi.askstr(ATTR_HISTORY, attr_def, attr_msg) - @classmethod - def clear_bap_comments(cls): - """Ask user for confirmation and then clear (BAP ..) comments.""" - from bap.utils.bap_comment import get_bap_comment - from bap.utils.ida import all_valid_ea - from idaapi import ASKBTN_CANCEL, ASKBTN_YES - - if idaapi.askyn_c(ASKBTN_CANCEL, - "Delete all (BAP ..) comments?") != ASKBTN_YES: + if attr is None: return - for ea in all_valid_ea(): - old_comm = idaapi.get_cmt(ea, 0) - if old_comm is None: - continue - _, start_loc, end_loc = get_bap_comment(old_comm) - new_comm = old_comm[:start_loc] + old_comm[end_loc:] - idaapi.set_cmt(ea, new_comm, 0) + # store a choice of attributes for the given set of arguments + BAP_BIR_Attr.recipes[args] = attr + ea = idc.ScreenEA() + attrs = [] + if attr != '': + attrs = attr.split(',') + analysis = BapScripter(args, attrs) + analysis.on_finish(lambda bap: self.load_script(bap,ea)) + analysis.run() + + def load_script(self, bap, ea): + idc.SetStatus(idc.IDA_STATUS_WORK) + idaapi.IDAPython_ExecScript(bap.script.name, globals()) + self._do_callbacks(ea) + idc.Refresh() + # do we really need to call this? + idaapi.refresh_idaview_anyway() + idc.SetStatus(idc.IDA_STATUS_READY) - cls._do_callbacks() - flags = idaapi.PLUGIN_FIX - comment = "BAP BIR Attr Plugin" - help = "BAP BIR Attr Plugin" - wanted_name = "BAP BIR Attr Plugin" - wanted_hotkey = "" def init(self): """Initialize Plugin.""" - from bap.utils.ida import add_hotkey - add_hotkey("Shift-S", self.run_bap) - add_hotkey("Ctrl-Shift-S", self.clear_bap_comments) return idaapi.PLUGIN_KEEP def term(self): """Terminate Plugin.""" pass - def run(self, arg): - """ - Run Plugin. - - Ignored since keybindings are installed. - """ - pass @classmethod def install_callback(cls, callback_fn): @@ -136,6 +121,7 @@ def install_callback(cls, callback_fn): Dict is guaranteed to get the following keys: 'ea': The value of EA at point where user propagated taint from. """ + idc.Message('a callback is installed\n') cls._callbacks[ptr_or_reg].append(callback_fn) diff --git a/plugins/bap/plugins/bap_clear_comments.py b/plugins/bap/plugins/bap_clear_comments.py new file mode 100644 index 0000000..bdc3f68 --- /dev/null +++ b/plugins/bap/plugins/bap_clear_comments.py @@ -0,0 +1,40 @@ +from idaapi import ASKBTN_YES +from bap.utils.bap_comment import get_bap_comment +from bap.utils.ida import all_valid_ea + + +class Main(idaapi.plugin_t): + flags = idaapi.PLUGIN_DRAW + comment = "removes all BAP comments" + help="" + wanted_name = "BAP: Clear comments" + wanted_hotkey = "Ctrl-Shift-S" + + + def clear_bap_comments(self): + """Ask user for confirmation and then clear (BAP ..) comments.""" + + if idaapi.askyn_c(ASKBTN_YES, + "Delete all (BAP ..) comments?") != ASKBTN_YES: + return + + for ea in all_valid_ea(): + old_comm = idaapi.get_cmt(ea, 0) + if old_comm is None: + continue + _, start_loc, end_loc = get_bap_comment(old_comm) + new_comm = old_comm[:start_loc] + old_comm[end_loc:] + idaapi.set_cmt(ea, new_comm, 0) + + + def init(self): + return idaapi.PLUGIN_KEEP + + def run(self, arg): + self.clear_bap_comments() + + def term(self): pass + + +def PLUGIN_ENTRY(): + return Main() diff --git a/plugins/bap/plugins/bap_functions.py b/plugins/bap/plugins/bap_functions.py index 52dd95f..f25c45e 100644 --- a/plugins/bap/plugins/bap_functions.py +++ b/plugins/bap/plugins/bap_functions.py @@ -8,55 +8,59 @@ Shift-P : Run BAP and mark code as functions in IDA """ import idautils +import idaapi +import idc +from bap.utils import config -class BAP_Functions(idaapi.plugin_t): - """Plugin to get functions from BAP and mark them in IDA.""" +from bap.utils.run import BapIda - @classmethod - def mark_functions(cls): - """Run BAP, get functions, and mark them in IDA.""" - import tempfile - from bap.utils.run import run_bap_with +class FunctionFinder(BapIda): + def __init__(self): + idc.Message('in the FunctionFinder constructor\n') + super(FunctionFinder,self).__init__() + self.action = 'looking for function starts' + self.syms = self.tmpfile('syms', mode='r') + self.args += [ + '--print-symbol-format', 'addr', + '--dump', 'symbols:{0}'.format(self.syms.name) + ] - idc.SetStatus(IDA_STATUS_WAITING) - idaapi.refresh_idaview_anyway() - - args = { - 'symbol_file': tempfile.mkstemp(suffix='.symout', - prefix='ida-bap-')[1], - } - - run_bap_with( - "\ - --print-symbol-format=addr \ - --dump=symbols:\"{symbol_file}\" \ - ".format(**args), no_extras=True - ) - - with open(args['symbol_file'], 'r') as f: - for line in f: - line = line.strip() - if len(line) == 0: - continue - addr = int(line, 16) - end_addr = idaapi.BADADDR # Lets IDA decide the end - idaapi.add_func(addr, end_addr) - - idc.SetStatus(IDA_STATUS_READY) - - idc.Refresh() # Force the updated information to show up +class BAP_Functions(idaapi.plugin_t): + """Plugin to get functions from BAP and mark them in IDA.""" flags = idaapi.PLUGIN_FIX comment = "BAP Functions Plugin" help = "BAP Functions Plugin" - wanted_name = "BAP Functions Plugin" - wanted_hotkey = "" + wanted_name = "BAP: Discover functions" + wanted_hotkey = "Shift-P" + + + def mark_functions(self): + """Run BAP, get functions, and mark them in IDA.""" + idc.Message('creating BAP instance\n') + analysis = FunctionFinder() + idc.Message('installing handler\n') + analysis.on_finish(lambda x: self.add_starts(x)) + idc.Message('running the analysis\n') + analysis.run() + + def add_starts(self, bap): + idc.Message('Analysis has finished\n') + idaapi.refresh_idaview_anyway() + for line in bap.syms: + line = line.strip() + if len(line) == 0: + continue + addr = int(line, 16) + end_addr = idaapi.BADADDR + idc.Message('Adding function at {0}\n'.format(addr)) + idaapi.add_func(addr, end_addr) + + idc.Refresh() def init(self): """Initialize Plugin.""" - from bap.utils.ida import add_hotkey - add_hotkey("Shift-P", self.mark_functions) return idaapi.PLUGIN_KEEP def term(self): @@ -64,12 +68,7 @@ def term(self): pass def run(self, arg): - """ - Run Plugin. - - Ignored since keybindings are installed. - """ - pass + self.mark_functions() def PLUGIN_ENTRY(): diff --git a/plugins/bap/plugins/bap_taint.py b/plugins/bap/plugins/bap_taint.py index 5581131..73fe820 100644 --- a/plugins/bap/plugins/bap_taint.py +++ b/plugins/bap/plugins/bap_taint.py @@ -15,9 +15,50 @@ "Normal" White : Lines that were visited, but didn't get tainted """ import idautils +import idaapi +import idc +from bap.utils.run import BapIda +from bap.utils.abstract_ida_plugins import DoNothing + +patterns = [ + ('true', 'gray'), + ('is-visited', 'white'), + ('has-taints', 'red'), + ('taints', 'yellow') +] + + +class PropagateTaint(BapIda): + "Propagate taint information using BAP" + def __init__(self, addr, kind): + super(PropagateTaint,self).__init__() + self.action = 'taint propagating from {:s}0x{:X}'.format( + '*' if kind == 'ptr' else '', + addr) + self.passes = ['taint','propagate-taint','map-terms','emit-ida-script'] + self.script = self.tmpfile('py') + scheme = self.tmpfile('scm') + for (pat,color) in patterns: + scheme.write('(({0}) (color {1}))\n'.format(pat,color)) + scheme.close() + + self.args += [ + '--taint-'+kind, '0x{:X}'.format(addr), + '--passes', ','.join(self.passes), + '--map-terms-using', scheme.name, + '--emit-ida-script-attr', 'color', + '--emit-ida-script-file', self.script.name + ] + + + +class BapTaint(idaapi.plugin_t): + flags = 0 + comment = "BAP Taint Plugin" + wanted_name = "BAP: Taint" -class BAP_Taint(idaapi.plugin_t): + help = "" """ Plugin to use BAP to propagate taint information. @@ -31,7 +72,6 @@ class BAP_Taint(idaapi.plugin_t): @classmethod def _do_callbacks(cls, ptr_or_reg): - assert(ptr_or_reg == 'reg' or ptr_or_reg == 'ptr') data = { 'ea': idc.ScreenEA(), 'ptr_or_reg': ptr_or_reg @@ -39,66 +79,24 @@ def _do_callbacks(cls, ptr_or_reg): for callback in cls._callbacks[ptr_or_reg]: callback(data) - def _taint_and_color(self, ptr_or_reg): - import tempfile - from bap.utils.run import run_bap_with - - args = { - 'taint_location': idc.ScreenEA(), - 'ida_script_location': tempfile.mkstemp(suffix='.py', - prefix='ida-bap-')[1], - 'ptr_or_reg': ptr_or_reg - } - - idc.Message('-------- STARTING TAINT ANALYSIS --------------\n') - idc.SetStatus(IDA_STATUS_WAITING) + def start(self): + tainter = PropagateTaint(idc.ScreenEA(), self.kind) + tainter.on_finish(lambda bap: self.finish(bap)) + tainter.run() + def finish(self, bap): + idaapi.IDAPython_ExecScript(bap.script.name, globals()) idaapi.refresh_idaview_anyway() + BapTaint._do_callbacks(self.kind) + idc.Refresh() - run_bap_with( - "\ - --taint-{ptr_or_reg}=0x{taint_location:X} \ - --taint \ - --propagate-taint \ - --map-terms-with='((true) (color gray))' \ - --map-terms-with='((is-visited) (color white))' \ - --map-terms-with='((has-taints) (color red))' \ - --map-terms-with='((taints) (color yellow))' \ - --map-terms \ - --emit-ida-script-attr=color \ - --emit-ida-script-file={ida_script_location} \ - --emit-ida-script \ - ".format(**args) - ) - - idc.Message('-------- DONE WITH TAINT ANALYSIS -------------\n\n') - idc.SetStatus(IDA_STATUS_READY) - - idaapi.IDAPython_ExecScript(args['ida_script_location'], globals()) - - idc.Exec("rm -f \"{ida_script_location}\"".format(**args)) # Cleanup - - idc.Refresh() # Force the color information to show up - - self._do_callbacks(ptr_or_reg) - - def _taint_reg_and_color(self): - self._taint_and_color('reg') - - def _taint_ptr_and_color(self): - self._taint_and_color('ptr') - - flags = idaapi.PLUGIN_FIX - comment = "BAP Taint Plugin" - help = "BAP Taint Plugin" - wanted_name = "BAP Taint Plugin" - wanted_hotkey = "" + def __init__(self, kind): + assert(kind in ('ptr', 'reg')) + self.kind = kind + self.wanted_name += 'pointer' if kind == 'ptr' else 'value' def init(self): """Initialize Plugin.""" - from bap.utils.ida import add_hotkey - add_hotkey("Shift-A", self._taint_reg_and_color) - add_hotkey("Ctrl-Shift-A", self._taint_ptr_and_color) return idaapi.PLUGIN_KEEP def term(self): @@ -108,10 +106,8 @@ def term(self): def run(self, arg): """ Run Plugin. - - Ignored since keybindings are installed. """ - pass + self.start() @classmethod def install_callback(cls, callback_fn, ptr_or_reg=None): @@ -130,9 +126,12 @@ def install_callback(cls, callback_fn, ptr_or_reg=None): elif ptr_or_reg == 'ptr' or ptr_or_reg == 'reg': cls._callbacks[ptr_or_reg].append(callback_fn) else: - print "Invalid ptr_or_reg value passed {}".format(repr(ptr_or_reg)) + idc.Fatal("Invalid ptr_or_reg value passed {}". + format(repr(ptr_or_reg))) + +class BapTaintStub(DoNothing): + pass def PLUGIN_ENTRY(): - """Install BAP_Taint upon entry.""" - return BAP_Taint() + return BapTaintStub() diff --git a/plugins/bap/plugins/bap_taint_ptr.py b/plugins/bap/plugins/bap_taint_ptr.py new file mode 100644 index 0000000..02b852e --- /dev/null +++ b/plugins/bap/plugins/bap_taint_ptr.py @@ -0,0 +1,10 @@ +from bap.plugins.bap_taint import BapTaint + +class BapTaintPtr(BapTaint): + wanted_hotkey = "Ctrl-Shift-A" + def __init__(self): + super(BapTaintPtr,self).__init__('ptr') + + +def PLUGIN_ENTRY(): + return BapTaintPtr() diff --git a/plugins/bap/plugins/bap_taint_reg.py b/plugins/bap/plugins/bap_taint_reg.py new file mode 100644 index 0000000..85d796f --- /dev/null +++ b/plugins/bap/plugins/bap_taint_reg.py @@ -0,0 +1,10 @@ +from bap.plugins.bap_taint import BapTaint + +class BapTaintReg(BapTaint): + wanted_hotkey = "Shift-A" + def __init__(self): + super(BapTaintReg,self).__init__('reg') + + +def PLUGIN_ENTRY(): + return BapTaintReg() diff --git a/plugins/bap/plugins/bap_task_manager.py b/plugins/bap/plugins/bap_task_manager.py new file mode 100644 index 0000000..b1bbb7e --- /dev/null +++ b/plugins/bap/plugins/bap_task_manager.py @@ -0,0 +1,70 @@ +""" BAP Task Manager Form """ + +#pylint: disable=missing-docstring,unused-argument,no-self-use,invalid-name + +from __future__ import print_function + +import idaapi #pylint: disable=import-error + +from bap.utils.run import BapIda + +class BapSelector(idaapi.Choose2): + #pylint: disable=invalid-name,missing-docstring,no-self-use + def __init__(self): + idaapi.Choose2.__init__(self, 'Choose instances to kill', [ + ['#', 2], + ['PID', 4], + ['Action', 40], + ], flags=idaapi.Choose2.CH_MULTI) + self.selection = [] + self.instances = list(BapIda.instances) + + def select(self): + choice = self.Show(modal=True) + if choice < 0: + return [self.instances[i] for i in self.selection] + else: + return [self.instances[choice]] + + def OnClose(self): + pass + + def OnGetLine(self, n): + bap = self.instances[n] + return [str(n), str(bap.proc.pid), bap.action] + + def OnGetSize(self): + return len(self.instances) + + def OnSelectionChange(self, selected): + self.selection = selected + + +class BapTaskManager(idaapi.plugin_t): + #pylint: disable=no-init + flags = idaapi.PLUGIN_DRAW + wanted_hotkey = "Ctrl-Alt-Shift-F5" + comment = "bap task manager" + help = "Open BAP Task Manager" + wanted_name = "BAP: Task Manager" + + def run(self, arg): + chooser = BapSelector() + selected = chooser.select() + for bap in selected: + if bap in BapIda.instances: + print('BAP> terminating '+str(bap.proc.pid)) + bap.cancel() + else: + print("BAP> instance {0} has already finised". + format(bap.proc.pid)) + + def term(self): + pass + + def init(self): + return idaapi.PLUGIN_KEEP + + +def PLUGIN_ENTRY(): + return BapTaskManager() diff --git a/plugins/bap/plugins/bap_view.py b/plugins/bap/plugins/bap_view.py index dcbe48b..a3c46c3 100644 --- a/plugins/bap/plugins/bap_view.py +++ b/plugins/bap/plugins/bap_view.py @@ -1,81 +1,133 @@ """BAP View Plugin to read latest BAP execution trace.""" -import idaapi +from __future__ import print_function +from bap.utils.run import BapIda +import re +import idaapi # pylint: disable=import-error -class BAP_View(idaapi.plugin_t): +class BapViews(idaapi.Choose2): + #pylint: disable=invalid-name,missing-docstring,no-self-use + def __init__(self, views): + idaapi.Choose2.__init__(self, 'Choose BAP view', [ + ['PID', 4], + ['Status', 5], + ['Action', 40] + ]) + self.views = views + + def OnClose(self): + pass + + def OnGetLine(self, n): + view = self.views[self.views.keys()[n]] + code = view.instance.proc.returncode + return [ + str(view.instance.proc.pid), + "running" if code is None else str(code), + view.instance.action + ] + + def OnGetSize(self): + return len(self.views) + +class View(idaapi.simplecustviewer_t): + #pylint: disable=invalid-name,missing-docstring,no-self-use + #pylint: disable=super-on-old-class,no-member + def __init__(self, caption, instance, on_close=None): + super(View, self).__init__() + self.Create(caption) + self.instance = instance + self.on_close = on_close + + def update(self): + self.ClearLines() + with open(self.instance.out.name, 'r') as src: + for line in src.read().split('\n'): + self.AddLine(recolorize(line)) + self.Refresh() # Ensure latest information gets to the screen + + def OnClose(self): + self.ClearLines() + if self.on_close: + self.on_close() + + +class BapView(idaapi.plugin_t): """ BAP View Plugin. Keybindings: - Ctrl-Alt-Shift-S : Open/Refresh BAP View + Ctrl-Shift-F5 : Open/Refresh BAP View """ + flags = idaapi.PLUGIN_DRAW + wanted_hotkey = "Ctrl-Shift-F5" + comment = "bap output viewer" + help = "View BAP output" + wanted_name = "BAP: Show output" - _view = None - - @classmethod - def _get_store_path(cls): - from tempfile import gettempdir - from idaapi import get_root_filename - return "{}/ida-bap-{}.out".format(gettempdir(), get_root_filename()) - - @classmethod - def _get_view(cls): - """Get the BAP View, creating it if necessary.""" - if cls._view is None: - cls._view = idaapi.simplecustviewer_t() - created_view = cls._view.Create('BAP View') - if not created_view: - cls._view = None - return cls._view - - @classmethod - def update(cls, text): - """Replace BAP View storage with the text.""" - with open(cls._get_store_path(), 'w') as f: - f.write(text) - - @classmethod - def show(cls): - """Display BAP View to the user.""" - v = cls._get_view() - if v is not None: - import re - ansi_escape = re.compile(r'\x1b[^m]*m([^\x1b]*)\x1b[^m]*m') - recolorize = lambda s: ansi_escape.sub('\1\x22\\1\2\x22', s) - v.ClearLines() - with open(cls._get_store_path(), 'r') as f: - for line in f.read().split('\n'): - v.AddLine(recolorize(line)) - v.Refresh() # Ensure latest information gets to the screen - v.Show() # Actually show it on the screen - - flags = idaapi.PLUGIN_PROC - wanted_hotkey = "" - comment = "BAP View" - help = "BAP View" - wanted_name = "BAP View" + def __init__(self): + self.views = {} + + + def create_view(self, bap): + "creates a new view" + pid = bap.proc.pid + name = 'BAP-{0}'.format(pid) + view = View(name, bap, on_close=lambda: self.delete_view(pid)) + view.instance = bap + curr = idaapi.get_current_tform() + self.views[pid] = view + view.Show() #pylint: disable=no-member + idaapi.switchto_tform(curr, True) + + def delete_view(self, pid): + "deletes a view associated with the provided pid" + del self.views[pid] + + def update_view(self, bap): + """updates the view associated with the given bap instance""" + view = self.views.get(bap.proc.pid, None) + if view: + view.update() + + def finished(self, bap): + "final update" + self.update_view(bap) + view = self.views[bap.proc.pid] + if bap.proc.returncode > 0: + view.Show() #pylint: disable=no-member def init(self): """Initialize BAP view to load whenever hotkey is pressed.""" - from bap.utils import ida - ida.add_hotkey('Ctrl-Alt-Shift-S', self.show) - self.update('\n BAP has not been run yet.') + BapIda.observers['instance_created'].append(self.create_view) + BapIda.observers['instance_updated'].append(self.update_view) + BapIda.observers['instance_finished'].append(self.finished) return idaapi.PLUGIN_KEEP def term(self): """Close BAP View, if it exists.""" - import idc - v = self._get_view() - if v is not None: - v.Close() - idc.Exec("rm -f {}".format(self._get_store_path())) # Cleanup - - def run(self, arg): - """Ignore, since callbacks are installed.""" - pass + for pid in self.views: + self.views[pid].Close() + + def show_view(self): + "Switch to one of the BAP views" + chooser = BapViews(self.views) + choice = chooser.Show(modal=True) #pylint: disable=no-member + if choice >= 0: + view = self.views[self.views.keys()[choice]] + view.Show() + + + def run(self, arg): #pylint: disable=unused-argument + "invokes the plugin" + self.show_view() +def recolorize(line): + """fix ansi colors""" + ansi_escape = re.compile(r'\x1b[^m]*m([^\x1b]*)\x1b[^m]*m') + return ansi_escape.sub('\1\x22\\1\2\x22', line) -def PLUGIN_ENTRY(): +def PLUGIN_ENTRY(): #pylint: disable=invalid-name """Install BAP_View upon entry.""" - return BAP_View() + return BapView() diff --git a/plugins/bap/plugins/pseudocode_bap_comment.py b/plugins/bap/plugins/pseudocode_bap_comment.py index 36548ed..1b85a68 100644 --- a/plugins/bap/plugins/pseudocode_bap_comment.py +++ b/plugins/bap/plugins/pseudocode_bap_comment.py @@ -6,6 +6,10 @@ class Pseudocode_BAP_Comment(abstract_ida_plugins.SimpleLine_Modifier_Hexrays): """Propagate comments from Text/Graph view to Pseudocode view.""" + flags = idaapi.PLUGIN_HIDE + comment = "BAP Comment on Pseudocode" + help = "BAP Comment on Pseudocode" + wanted_name = "BAP Comment on Pseudocode" @classmethod def _simpleline_modify(cls, cfunc, sl): @@ -32,9 +36,6 @@ def _simpleline_modify(cls, cfunc, sl): sl.line += sexpr.from_list(BAP_dict) sl.line += '\x02\x0c\x02\x0c' # stop comment coloring - comment = "BAP Comment on Pseudocode" - help = "BAP Comment on Pseudocode" - wanted_name = "BAP Comment on Pseudocode" def PLUGIN_ENTRY(): diff --git a/plugins/bap/plugins/pseudocode_bap_taint.py b/plugins/bap/plugins/pseudocode_bap_taint.py index 7ab299d..722d82d 100644 --- a/plugins/bap/plugins/pseudocode_bap_taint.py +++ b/plugins/bap/plugins/pseudocode_bap_taint.py @@ -17,11 +17,15 @@ } from bap.utils import abstract_ida_plugins, ida +from bap.plugins.bap_taint import BapTaint + +import idc class Pseudocode_BAP_Taint(abstract_ida_plugins.SimpleLine_Modifier_Hexrays): """Propagate taint information from Text/Graph view to Pseudocode view.""" + flags=idaapi.PLUGIN_HIDE comment = "BAP Taint Plugin for Pseudocode View" help = "BAP Taint Plugin for Pseudocode View" wanted_name = "BAP Taint Pseudocode" @@ -79,17 +83,13 @@ def autocolorize_callback(data): return self.run_over_cfunc(cfunc) - idaapi.load_plugin('BAP_Taint') - BAP_Taint.install_callback(autocolorize_callback) - - print ("Finished installing callbacks for Taint Analysis" + - " in Hex-Rays") - + # idaapi.load_plugin('bap_taint') + BapTaint.install_callback(autocolorize_callback) else: return idaapi.PLUGIN_SKIP except AttributeError: - print "init_hexrays_plugin() not found. Skipping Hex-Rays plugin." + idc.Warning("init_hexrays_plugin() not found. Skipping Hex-Rays plugin.") return abstract_ida_plugins.SimpleLine_Modifier_Hexrays.init(self) # Call superclass init() diff --git a/plugins/bap/utils/abstract_ida_plugins.py b/plugins/bap/utils/abstract_ida_plugins.py index 8620ffb..072594d 100644 --- a/plugins/bap/utils/abstract_ida_plugins.py +++ b/plugins/bap/utils/abstract_ida_plugins.py @@ -19,7 +19,7 @@ def PLUGIN_ENTRY(): return DoNothing() """ - flags = idaapi.PLUGIN_UNL + flags = idaapi.PLUGIN_HIDE comment = "Does Nothing" help = "Does Nothing" wanted_name = "Do Nothing" @@ -27,7 +27,7 @@ def PLUGIN_ENTRY(): def init(self): """Skip plugin.""" - return idaapi.PLUGIN_OK + return idaapi.PLUGIN_SKIP def term(self): """Do nothing.""" diff --git a/plugins/bap/utils/config.py b/plugins/bap/utils/config.py index 7844940..15c6fe5 100644 --- a/plugins/bap/utils/config.py +++ b/plugins/bap/utils/config.py @@ -1,19 +1,20 @@ """Module for reading from and writing to the bap.cfg config file.""" import os -import idaapi +import idaapi #pylint: disable=import-error -cfg_dir = idaapi.idadir('cfg') -cfg_path = os.path.join(cfg_dir, 'bap.cfg') +CFG_DIR = idaapi.idadir('cfg') +CFG_PATH = os.path.join(CFG_DIR, 'bap.cfg') def _read(): - if not os.path.exists(cfg_path): + "parse the config file" + if not os.path.exists(CFG_PATH): return {} cfg = {'default': []} - with open(cfg_path, 'r') as f: + with open(CFG_PATH, 'r') as src: current_section = 'default' - for line in f.read().split('\n'): + for line in src.read().split('\n'): if len(line) == 0: # Empty line continue elif line[0] == '.': # Section @@ -26,45 +27,79 @@ def _read(): def _write(cfg): + "dump config into the file" new_config = [] for section in cfg: new_config.append('.' + section) for line in cfg[section]: new_config.append(line) new_config.append('') - if not os.path.exists(cfg_dir): - os.makedirs(cfg_dir) - with open(cfg_path, 'w') as f: - f.write('\n'.join(new_config)) + if not os.path.exists(CFG_DIR): + os.makedirs(CFG_DIR) + with open(CFG_PATH, 'w') as out: + out.write('\n'.join(new_config)) -def get(key, default=None, section='default'): +def get(path, default=None): """Get value from key:value in the config file.""" + key = Key(path) cfg = _read() - if section not in cfg: + if key.section not in cfg: return default - for line in cfg[section]: + for line in cfg[key.section]: if line[0] == ';': # Comment continue - elif line.split()[0] == key: + elif line.split()[0] == key.value: return line.split()[1] return default -def set(key, value, section='default'): +def is_set(key): + """returns True if the value is set, + i.e., if it is `1`, `true` or `yes`. + returns False, if key is not present in the dictionary, + or has any other value. + """ + return get(key, default='0').lower() in ('1', 'true', 'yes') + +def set(path, value): #pylint: disable=redefined-builtin """Set key:value in the config file.""" cfg = _read() + key = Key(path) - if section not in cfg: - cfg[section] = [] - for i, line in enumerate(cfg[section]): + if key.section not in cfg: + cfg[key.section] = [] + for i, line in enumerate(cfg[key.section]): if line[0] == ';': # Comment continue elif line.split()[0] == key: - cfg[section][i] = '{}\t{}\t; Previously: {}'.format( - key, value, line) + cfg[key.section][i] = '{}\t{}\t; Previously: {}'.format( + key.value, value, line) break else: # Key not previously set - cfg[section].append('{}\t{}'.format(key, value)) + cfg[key.section].append('{}\t{}'.format(key.value, value)) _write(cfg) + + +class Key(object): #pylint: disable=too-few-public-methods + "Configuration key" + def __init__(self, path): + elts = path.split('.') + if len(elts) > 2: + raise InvalidKey(path) + simple = len(elts) == 1 + self.section = 'default' if simple else elts[0] + self.value = elts[0] if simple else elts[1] + + +class InvalidKey(Exception): + "Raised when the key is badly formated" + def __init__(self, path): + super(InvalidKey, self).__init__() + self.path = path + + def __str__(self): + return 'Invalid key syntax. \ + Expected `` or `
.`, got {0}'.format( + self.path) diff --git a/plugins/bap/utils/ida.py b/plugins/bap/utils/ida.py index 175a5b7..b9fad36 100644 --- a/plugins/bap/utils/ida.py +++ b/plugins/bap/utils/ida.py @@ -52,8 +52,8 @@ def dump_loader_info(output_filename): out.write("))\n") -def dump_symbol_info(output_filename): - """Dump information for BAP's symbolizer into output_filename.""" +def dump_symbol_info(out): + """Dump information for BAP's symbolizer into the out file object.""" from idautils import Segments, Functions from idc import ( SegStart, SegEnd, GetFunctionAttr, @@ -89,17 +89,16 @@ def func_name_propagate_thunk(ea): idaapi.autoWait() - with open(output_filename, 'w+') as out: - for ea in Segments(): - fs = Functions(SegStart(ea), SegEnd(ea)) - for f in fs: - out.write('("%s" 0x%x 0x%x)\n' % ( - func_name_propagate_thunk(f), - GetFunctionAttr(f, FUNCATTR_START), - GetFunctionAttr(f, FUNCATTR_END))) + for ea in Segments(): + fs = Functions(SegStart(ea), SegEnd(ea)) + for f in fs: + out.write('("%s" 0x%x 0x%x)\n' % ( + func_name_propagate_thunk(f), + GetFunctionAttr(f, FUNCATTR_START), + GetFunctionAttr(f, FUNCATTR_END))) -def dump_c_header(output_filename): +def dump_c_header(out): """Dump type information as a C header.""" def local_type_info(): class my_sink(idaapi.text_sink_t): @@ -155,10 +154,9 @@ def preprocess(line): line = pp_wd(line) return line - with open(output_filename, 'w+') as out: - for line in local_type_info() + function_sigs(): - line = preprocess(line) - out.write(line + '\n') + for line in local_type_info() + function_sigs(): + line = preprocess(line) + out.write(line + '\n') def dump_brancher_info(output_filename): @@ -182,21 +180,3 @@ def pp(l): pp(dest(ea, True) - branch_dests), pp(branch_dests) )) - - -def add_hotkey(hotkey, func): - """ - Assign hotkey to run func. - - If a pre-existing action for the hotkey exists, then this function will - remove that action and replace it with func. - - Arguments: - - hotkey : string (for example 'Ctrl-Shift-A') - - func : unit function (neither accepts arguments, nor returns values) - """ - hotkey_ctx = idaapi.add_hotkey(hotkey, func) - if hotkey_ctx is None: - print("Failed to register {} for {}".format(hotkey, func)) - else: - print("Registered {} for {}".format(hotkey, func)) diff --git a/plugins/bap/utils/run.py b/plugins/bap/utils/run.py index 907e793..3bc7b9d 100644 --- a/plugins/bap/utils/run.py +++ b/plugins/bap/utils/run.py @@ -1,5 +1,296 @@ """Utilities that interact with BAP.""" +from __future__ import print_function + +import tempfile +import subprocess +import os +import sys + +import traceback + +import idc #pylint: disable=import-error +import idaapi #pylint: disable=import-error +from bap.utils import ida, config + + +#pylint: disable=missing-docstring + +class Bap(object): + """Bap instance base class. + + Instantiate a subprocess with BAP. + + We will try to keep it clean from IDA + specifics, so that later we can lift it to the bap-python library + """ + + + DEBUG = False + + + def __init__(self, bap, input_file): + """Sandbox for the BAP process. + + Each process is sandboxed, so that all intermediated data is + stored in a temporary directory. + + instance variables: + + - `tmpdir` -- a folder where we will put all our intermediate + files. Might be removed on the cleanup (see cleanup for more); + + - `proc` an instance of `Popen` class if process has started, + None otherwise + + - `args` an argument list that was passed to the `Popen`. + + - `action` a gerund describing the action, that is perfomed by + the analysis + + - `fds` a list of opened filedescriptors to be closed on the exit + + """ + self.args = [bap, input_file] + self.proc = None + self.fds = [] + self.out = self.tmpfile("out") + self.action = "running bap" + + + def run(self): + "starts BAP process" + if self.DEBUG: + print("BAP> {0}\n".format(' '.join(self.args))) + self.proc = subprocess.Popen( + self.args, + stdout=self.out, + stderr=subprocess.STDOUT, + env={ + 'BAP_LOG_DIR' : self.tmpdir + }) + + def finished(self): + "true is the process has finished" + return self.proc is not None and self.proc.poll() is not None + + def close(self): + "terminate the process if needed and cleanup" + if not self.finished(): + if self.proc is not None: + self.proc.terminate() + self.proc.wait() + self.cleanup() + + def cleanup(self): + """Close and remove all created temporary files. + + For the purposes of debugging, files are not removed + if BAP finished with a positive nonzero code. I.e., + they are removed only if BAP terminated normally, or was + killed by a signal (terminated). + + All opened file descriptros are closed in any case.""" + for desc in self.fds: + desc.close() + + if not self.DEBUG and (self.proc is None or + self.proc.returncode <= 0): + for path in os.listdir(self.tmpdir): + os.remove(os.path.join(self.tmpdir, path)) + os.rmdir(self.tmpdir) + + def tmpfile(self, suffix, *args, **kwargs): + "creates a new temporary files in the self.tmpdir" + if getattr(self, 'tmpdir', None) is None: + #pylint: disable=attribute-defined-outside-init + self.tmpdir = tempfile.mkdtemp(prefix="bap") + + tmp = tempfile.NamedTemporaryFile( + delete=False, + prefix='bap-ida', + suffix="."+suffix, + dir=self.tmpdir, + *args, + **kwargs) + self.fds.append(tmp) + return tmp + +class BapIda(Bap): + """BAP instance in IDA. + + Uses timer even to poll the ready status of the process. + + """ + instances = [] + poll_interval_ms = 200 + + # class level handlers to observe BAP instances, + # useful, for handling gui. See also, on_finished + # and on_cancel, for user specific handlers. + observers = { + 'instance_created' : [], + 'instance_updated' : [], + 'instance_finished' : [], + } + + + def __init__(self, symbols=True): + try: + check_and_configure_bap() + except: + idc.Message('BAP> configuration failed\n{0}\n'. + format(str(sys.exc_info()))) + traceback.print_exc() + raise BapIdaError() + bap = config.get('bap_executable_path') + if bap is None: + idc.Warning("Can't locate BAP\n") + raise BapNotFound() + binary = idc.GetInputFilePath() + super(BapIda, self).__init__(bap, binary) + # if you run IDA inside IDA you will crash IDA + self.args.append('--no-ida') + self._on_finish = [] + self._on_cancel = [] + if symbols: + self._setup_symbols() + + headers = config.is_set('ida_api.enabled') + + if headers: + self._setup_headers(bap) + + def run(self): + "run BAP instance" + if len(BapIda.instances) > 0: + answer = idaapi.askyn_c( + idaapi.ASKBTN_YES, + "Previous instances of BAP didn't finish yet.\ + Do you really want to start a new one?". + format(len(BapIda.instances))) + + if answer == idaapi.ASKBTN_YES: + self._do_run() + else: + self._do_run() + + idc.Message("BAP> total number of running instances: {0}\n". + format(len(BapIda.instances))) + + def _setup_symbols(self): + "pass symbol information from IDA to BAP" + with self.tmpfile("sym") as out: + ida.dump_symbol_info(out) + self.args += [ + "--read-symbols-from", out.name, + "--symbolizer=file", + "--rooter=file" + ] + + + def _setup_headers(self, bap): + "pass type information from IDA to BAP" + # this is very fragile, and may break in case + # if we have several BAP instances, especially + # when they are running on different binaries. + # Will leave it as it is until issue #588 is + # resolved in the upstream + with self.tmpfile("h") as out: + ida.dump_c_header(out) + subprocess.call(bap, [ + '--api-add', 'c:"{0}"'.format(out.name), + ]) + + def cleanup(): + subprocess.call(bap, [ + "--api-remove", "c:{0}". + format(os.path.basename(out.name)) + ]) + self._on_cancel.append(cleanup) + self._on_finish.append(cleanup) + + + def _do_run(self): + try: + super(BapIda, self).run() + BapIda.instances.append(self) + idaapi.register_timer(200, self.update) + idc.SetStatus(idc.IDA_STATUS_THINKING) + self.run_handlers(self.observers['instance_created']) + idc.Message("BAP> created new instance with PID {0}\n". + format(self.proc.pid)) + except: #pylint: disable=bare-except + idc.Message("BAP> failed to create instance\nError: {0}\n". + format(str(sys.exc_info()[1]))) + traceback.print_exc() + + + def run_handlers(self, handlers): + failures = 0 + for handler in handlers: + try: + handler(self) + except: #pylint: disable=bare-except + failures += 1 + idc.Message("BAP> {0} failed because {1}\n". + format(self.action, str(sys.exc_info()[1]))) + traceback.print_exc() + if failures != 0: + idc.Warning("Some BAP handlers failed") + + + def close(self): + super(BapIda, self).close() + BapIda.instances.remove(self) + + def update(self): + if self.finished(): + if self.proc.returncode == 0: + self.run_handlers(self._on_finish) + self.run_handlers(self.observers['instance_finished']) + self.close() + idc.Message("BAP> finished " + self.action + '\n') + elif self.proc.returncode > 0: + idc.Message("BAP> an error has occured while {0}\n". + format(self.action)) + with open(self.out.name) as out: + idc.Message('BAP> output:\n{0}\n'.format(out.read())) + else: + idc.Message("BAP> was killed by signal {0}\n". + format(-self.proc.returncode)) + return -1 + else: + self.run_handlers(self.observers['instance_updated']) + thinking = False + for bap in BapIda.instances: + if bap.finished(): + idc.SetStatus(idc.IDA_STATUS_THINKING) + thinking = True + if not thinking: + idc.SetStatus(idc.IDA_STATUS_READY) + return 200 + + def cancel(self): + self.run_handlers(self._on_cancel) + self.run_handlers(self.observers['instance_finished']) + self.close() + + def on_finish(self, callback): + self._on_finish.append(callback) + + def on_cancel(self, callback): + self._on_cancel.append(callback) + +class BapIdaError(Exception): + pass + +class BapNotFound(BapIdaError): + pass + + +BAP_FINDERS = [] + def check_and_configure_bap(): """ @@ -12,173 +303,58 @@ def check_and_configure_bap(): Also, this specifically enables the BAP API option in the config if it is unspecified. """ - from bap.utils import config - import idaapi + if config.get('bap_executable_path') is not None: + return - def config_path(): - if config.get('bap_executable_path') is not None: - return - default_bap_path = '' + bap_path = '' - from subprocess import check_output, CalledProcessError - import os - try: - default_bap_path = check_output(['which', 'bap']).strip() - except (OSError, CalledProcessError) as e: - # Cannot run 'which' command OR - # 'which' could not find 'bap' - try: - default_bap_path = os.path.join( - check_output(['opam', 'config', 'var', 'bap:bin']).strip(), - 'bap' - ) - except OSError: - # Cannot run 'opam' - pass - if not default_bap_path.endswith('bap'): - default_bap_path = '' - - def confirm(msg): - from idaapi import askyn_c, ASKBTN_CANCEL, ASKBTN_YES - return askyn_c(ASKBTN_CANCEL, msg) == ASKBTN_YES - - while True: - bap_path = idaapi.askfile_c(False, default_bap_path, 'Path to bap') - if bap_path is None: - if confirm('Are you sure you don\'t want to set path?'): - return - else: - continue - if not bap_path.endswith('bap'): - if not confirm("Path does not end with bap. Confirm?"): - continue - if not os.path.isfile(bap_path): - if not confirm("Path does not point to a file. Confirm?"): - continue + for find in BAP_FINDERS: + path = find() + if path: + bap_path = path break - config.set('bap_executable_path', bap_path) + # always ask a user to confirm the path that was found using heuristics + user_path = ask_user(bap_path) + if user_path: + bap_path = user_path + config.set('bap_executable_path', bap_path) - def config_bap_api(): - if config.get('enabled', section='bap_api') is None: - config.set('enabled', '1', section='bap_api') - config_path() - config_bap_api() +def system_path(): + try: + return subprocess.check_output(['which', 'bap']).strip() + except (OSError, subprocess.CalledProcessError): + return None -def run_bap_with(argument_string, no_extras=False): - """ - Run bap with the given argument_string. - Uses the currently open file, dumps latest symbols from IDA and runs - BAP with the argument_string +def opam_path(): + try: + cmd = ['opam', 'config', 'var', 'bap:bin'] + res = subprocess.check_output(cmd).strip() + return os.path.join(res, 'bap') + except OSError: + return None - Also updates the 'BAP View' +def ask_user(default_path): + def confirm(msg): + return idaapi.askyn_c(idaapi.ASKBTN_YES, msg) == idaapi.ASKBTN_YES - Note: If no_extras is set to True, then none of the extra work mentioned - above is done, and instead, bap is just run purely using - bap - """ - from bap.plugins.bap_view import BAP_View - from bap.utils import config - import ida - import idc - import tempfile - - check_and_configure_bap() - bap_executable_path = config.get('bap_executable_path') - if bap_executable_path is None: - return # The user REALLY doesn't want us to run it - - args = { - 'bap_executable_path': bap_executable_path, - 'bap_output_file': tempfile.mkstemp(suffix='.out', - prefix='ida-bap-')[1], - 'input_file_path': idc.GetInputFilePath(), - 'symbol_file_location': tempfile.mkstemp(suffix='.sym', - prefix='ida-bap-')[1], - 'header_path': tempfile.mkstemp(suffix='.h', prefix='ida-bap-')[1], - 'remaining_args': argument_string - } + while True: + bap_path = idaapi.askfile_c(False, default_path, 'Path to bap') + if bap_path is None: + if confirm('Are you sure you don\'t want to set path?'): + return None + else: + continue + if not bap_path.endswith('bap'): + if not confirm("Path does not end with bap. Confirm?"): + continue + if not os.path.isfile(bap_path): + if not confirm("Path does not point to a file. Confirm?"): + continue + return bap_path - if no_extras: - - command = ( - "\ - \"{bap_executable_path}\" \"{input_file_path}\" \ - {remaining_args} \ - > \"{bap_output_file}\" 2>&1 \ - ".format(**args) - ) - - else: - - bap_api_enabled = (config.get('enabled', - default='0', - section='bap_api').lower() in - ('1', 'true', 'yes')) - - ida.dump_symbol_info(args['symbol_file_location']) - - if bap_api_enabled: - ida.dump_c_header(args['header_path']) - idc.Exec( - "\ - \"{bap_executable_path}\" \ - --api-add=c:\"{header_path}\" \ - ".format(**args) - ) - - command = ( - "\ - \"{bap_executable_path}\" \"{input_file_path}\" \ - --read-symbols-from=\"{symbol_file_location}\" --symbolizer=file \ - {remaining_args} \ - -d > \"{bap_output_file}\" 2>&1 \ - ".format(**args) - ) - - idc.Exec(command) - - with open(args['bap_output_file'], 'r') as f: - BAP_View.update( - "BAP execution string\n" + - "--------------------\n" + - "\n" + - '\n --'.join(command.strip().split('--')) + - "\n" + - "\n" + - "Output\n" + - "------\n" + - "\n" + - f.read() - ) - - # Force close BAP View - # This forces the user to re-open the new view if needed - # This "hack" is needed since IDA decides to give a different BAP_View - # class here, than the cls parameter it sends to BAP_View - # TODO: Fix this - import idaapi - tf = idaapi.find_tform("BAP View") - if tf: - idaapi.close_tform(tf, 0) - - # Do a cleanup of all the temporary files generated/added - if not no_extras: - if bap_api_enabled: - idc.Exec( - "\ - \"{bap_executable_path}\" \ - --api-remove=c:`basename \"{header_path}\"` \ - ".format(**args) - ) - idc.Exec( - "\ - rm -f \ - \"{symbol_file_location}\" \ - \"{header_path}\" \ - \"{bap_output_file}\" \ - ".format(**args) - ) +BAP_FINDERS.append(system_path) +BAP_FINDERS.append(opam_path) diff --git a/plugins/bap/utils/sexpr.py b/plugins/bap/utils/sexpr.py index 0aadece..e1d3b4d 100644 --- a/plugins/bap/utils/sexpr.py +++ b/plugins/bap/utils/sexpr.py @@ -42,16 +42,21 @@ def from_list(l): def is_valid(s): """Return True if s is a valid S-Expression.""" in_str = False + escaped = False bb = 0 for c in s: - if c == '(' and not in_str: - bb += 1 - elif c == ')' and not in_str: - bb -= 1 - if bb < 0: - return False - elif c == '\"': - in_str = not in_str + if not escaped and in_str and c == '\\': + escaped = True + else: + if not escaped and c == '\"': + in_str = not in_str + elif c == '(' and not in_str: + bb += 1 + elif c == ')' and not in_str: + bb -= 1 + if bb < 0: + return False + escaped=False return bb == 0 From 63168345993336ff7f00caef79e7d3b63cfac402 Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Fri, 16 Dec 2016 08:43:08 -0500 Subject: [PATCH 02/20] added pytest based testsuite --- plugins/bap/plugins/bap_clear_comments.py | 22 +- plugins/bap/utils/bap_comment.py | 280 ++++++++++++++++------ plugins/bap/utils/config.py | 9 +- plugins/bap/utils/ida.py | 26 +- setup.py | 22 ++ tests/conftest.py | 5 + tests/mockidaapi.py | 12 + tests/mockidc.py | 0 tests/test_bap_comment.py | 30 +++ tests/test_config.py | 11 + tests/test_ida.py | 56 +++++ tox.ini | 6 + 12 files changed, 385 insertions(+), 94 deletions(-) create mode 100644 setup.py create mode 100644 tests/conftest.py create mode 100644 tests/mockidaapi.py create mode 100644 tests/mockidc.py create mode 100644 tests/test_bap_comment.py create mode 100644 tests/test_config.py create mode 100644 tests/test_ida.py create mode 100644 tox.ini diff --git a/plugins/bap/plugins/bap_clear_comments.py b/plugins/bap/plugins/bap_clear_comments.py index bdc3f68..be3719c 100644 --- a/plugins/bap/plugins/bap_clear_comments.py +++ b/plugins/bap/plugins/bap_clear_comments.py @@ -1,16 +1,18 @@ +import idaapi from idaapi import ASKBTN_YES -from bap.utils.bap_comment import get_bap_comment + + +from bap.utils import bap_comment from bap.utils.ida import all_valid_ea -class Main(idaapi.plugin_t): +class BapClearComments(idaapi.plugin_t): flags = idaapi.PLUGIN_DRAW comment = "removes all BAP comments" - help="" + help = "" wanted_name = "BAP: Clear comments" wanted_hotkey = "Ctrl-Shift-S" - def clear_bap_comments(self): """Ask user for confirmation and then clear (BAP ..) comments.""" @@ -19,13 +21,9 @@ def clear_bap_comments(self): return for ea in all_valid_ea(): - old_comm = idaapi.get_cmt(ea, 0) - if old_comm is None: - continue - _, start_loc, end_loc = get_bap_comment(old_comm) - new_comm = old_comm[:start_loc] + old_comm[end_loc:] - idaapi.set_cmt(ea, new_comm, 0) - + comm = idaapi.get_cmt(ea, 0) + if bap_comment.parse(comm): + idaapi.set_cmt(ea, '', 0) def init(self): return idaapi.PLUGIN_KEEP @@ -37,4 +35,4 @@ def term(self): pass def PLUGIN_ENTRY(): - return Main() + return BapClearComments() diff --git a/plugins/bap/utils/bap_comment.py b/plugins/bap/utils/bap_comment.py index cd8b63b..d9fb131 100644 --- a/plugins/bap/utils/bap_comment.py +++ b/plugins/bap/utils/bap_comment.py @@ -1,92 +1,232 @@ -""" -BAP Comment. +"""BAP Comment. + +We use comments to annotate code in IDA with the semantic information +extracted from BAP analyses. The comments are machine-readable, and a +simple syntax is used, to make the parser robust and comments human +readable. We will define the syntax formally later, but we will start +with an example: + + BAP: saluki-sat,saluki-unsat, beagle-strings="hello, world",nice + +Basically, the comment string includes an arbitrary amount of +key=value pairs. If a value contains whitespaces, punctuation or any +non-word character, then it should be delimited with double quotes. If +a value contains quote character, then it should be escaped with the +backslash character (the backslash character can escape +itself). Properties that doesn't have values (or basically has a +property of a unit type, so called boolean properties) are represented +with their names only, e.g., ``saluki-sat``. A property can have +multiple values, separated by a comma. Properties wihtout values, can +be also separated with the comma. In fact you can always trade off +space for comma, if you like, e.g., ``saluki-sat,saluki-unsat`` is +equivalent to ``saluki-sat saluki-unsat``: + +>>> assert(parse('BAP: saluki-sat,saluki-unsat') == \ +parse('BAP: saluki-sat saluki-unsat')) + + +Comments are parsed into a dictionary, that maps properties into their +values. A property that doesn't have a value is mapped to an empty +list. + +>>> parse('BAP: saluki-sat,saluki-unsat beagle-chars=ajdladasn,asd \ + beagle-strings="{hello world}"') +{'saluki-sat': [], 'beagle-chars': ['ajdladasn', 'asd'], + 'saluki-unsat': [], 'beagle-strings': ['{hello world}']} + +They can be modifed, and dumped back into a string: + +>>> dumps({'saluki-sat': [], 'beagle-chars': ['ajdladasn', 'asd'], + 'saluki-unsat': [], 'beagle-strings': ['{hello world}']}) +'BAP: saluki-sat,saluki-unsat beagle-chars=ajdladasn,asd \ + beagle-strings="{hello world}"' + + +Any special characters inside the property value must be properly +escaped: + +>>> parse('BAP: beagle-chars="abc\\'"') +{'beagle-chars': ["abc'"]} + +Note: In the examples, we need to escape the backslash, as they are +intended to be run by the doctest system, that will perform one layer +of the expension. So, in the real life, to escape a quote we will +write only one backslash, e.g., "abc\'". Probably, this should be +considered as a bug on the doctest side, as it is assumed, that you +can copy paste an example from the doc to the interpreter and see the +identical results. Here we will get a syntax error from the python +interpreter. + +>>> dumps(parse('BAP: beagle-chars="abc\\'"')) +'BAP: beagle-chars="abc\\'"' + +Syntactically incorrect code will raise the ``SyntaxError`` exception, +e.g., + +>>> parse('BAP: beagle-words=hello=world') +Traceback (most recent call last): + ... +SyntaxError: in state key expected got = + +## Grammar -A BAP Comment is an S-Expression which is of the form (BAP ...) +comm ::= "BAP:" +props ::= + | +prop ::= + | = +values ::= | "," +value ::= +key ::= + + +Where ```` is any sequence of word-characters (see WORDCHARS) +constant (letters, numbers and the following two characters: "-" and +":"), e.g., `my-property-name`, or `analysis:property`. + + +Note: the parser usually accepts more languages that are formally recognized +by the grammar. -This module defines commonly used utility functions to interact with -BAP Comments. """ +import string +from shlex import shlex + +WORDCHARS = ''.join(['-:', string.ascii_letters, string.digits]) -def get_bap_comment(comm): + +def parse(comment): + """ Parse comment string. + + Returns a dictionary that maps properties to their values. + Raises SyntaxError if the comment is syntactically incorrect. + Returns None if comment doesn't start with the `BAP:` prefix. """ - Get '(BAP )' style comment from given string. + lexer = shlex(comment) + lexer.wordchars = WORDCHARS + result = {} + key = '' + values = [] + state = 'init' + + def error(exp, token): + "raise a nice error message" + raise SyntaxError('in state {0} expected {1} got {2}'. + format(state, exp, token)) + + def push(result, key, values): + "push binding into the stack" + if key != '': + result[key] = values + + for token in lexer: + if state == 'init': + if token != 'BAP:': + return None + state = 'key' + elif state == 'key': + if token == '=': + error('', token) + elif token == ',': + state = 'value' + else: + push(result, key, values) + values = [] + key = token + state = 'eq' + elif state == 'eq': + if token == '=': + state = 'value' + else: + push(result, key, values) + key = '' + values = [] + if token == ',': + state = 'key' + else: + key = token + state = 'eq' + elif state == 'value': + values.append(unquote(token)) + state = 'key' + + push(result, key, values) + return result + + +def dumps(comm): + """ + Dump dictionary into a comment string. - Returns tuple (BAP_dict, start_loc, end_loc) - BAP_dict: The '(BAP )' style comment - start_loc: comm[:start_loc] was before the BAP comment - end_loc: comm[end_loc:] was after the BAP comment + The representation is parseable with the parse function. """ - if '(BAP ' in comm: - start_loc = comm.index('(BAP ') - bracket_count = 0 - in_str = False - for i in range(start_loc, len(comm)): - if comm[i] == '(' and not in_str: - bracket_count += 1 - elif comm[i] == ')' and not in_str: - bracket_count -= 1 - if bracket_count == 0: - end_loc = i + 1 - BAP_dict = comm[start_loc:end_loc] - break - elif comm[i] == '\"': - in_str = not in_str + keys = [] + elts = [] + for (key, values) in comm.items(): + if values: + elts.append('{0}={1}'.format(key, ','.join( + quote(x) for x in values))) else: - # Invalid bracketing. - # Someone messed up the dict. - # Correct by inserting enough close brackets. - end_loc = len(comm) - BAP_dict = comm[start_loc:end_loc] + (')' * bracket_count) + keys.append(key) + keys.sort() + elts.sort() + return ' '.join(x for x in + ('BAP:', ','.join(keys), ' '.join(elts)) if x) + + +def quote(token): + """delimit a token with quotes if needed. + + The function guarantees that the string representation of the + token will be parsed into the same token. In case if a token + contains characters that are no in the set of WORDCHARS symbols, + that will lead to the splittage of the token during the lexing, + a pair of double quotes are added to prevent this. + + >>> quote('hello, world') + '"hello, world"' + """ + if set(token) - set(WORDCHARS): + return '"{0}"'.format(token) else: - start_loc = len(comm) - end_loc = len(comm) - BAP_dict = '(BAP )' + return token - return (BAP_dict, start_loc, end_loc) +def unquote(word, quotes='\'"'): + """removes quotes from both sides of the word. -def get_bap_list(BAP_dict): - """Return a list containing all the values in the BAP comment.""" - import sexpr - assert(BAP_dict[:5] == '(BAP ') - assert(sexpr.is_valid(BAP_dict)) - outer_removed = BAP_dict[5:-1] # Remove outermost '(BAP', ')' - return sexpr.to_list(outer_removed) + The quotes should occur on both sides of the word: + >>> unquote('"hello"') + 'hello' -def add_to_comment_string(comm, key, value): - """Add key:value to comm string.""" - import sexpr + If a quote occurs only on one side of the word, then + the word is left intact: - BAP_dict, start_loc, end_loc = get_bap_comment(comm) + >>> unquote('"hello') + '"hello' - if value == '()': - kv = ['BAP', [key]] # Make unit tags easier to read - else: - kv = ['BAP', [key, value]] + The quotes that delimites the world should be equal, i.e., + if the word is delimited with double quotes on the left and + a quote on the right, then it is not considered as delimited, + so it is not dequoted: - for e in get_bap_list(BAP_dict): - if isinstance(e, list) and len(e) <= 2: - # It is of the '(k v)' or '(t)' type - if e[0] != key: # Don't append if same as required key - kv.append(e) - else: - kv.append(e) + >>> unquote('"hello\\'') + '"hello\\'' - return comm[:start_loc] + sexpr.from_list(kv) + comm[end_loc:] + Finally, only one layer of quotes is removed, + >>> unquote('""hello""') + '"hello"' + """ + if len(word) > 1 and word[0] == word[-1] \ + and word[0] in quotes and word[-1] in quotes: + return word[1:-1] + else: + return word -def get_value(comm, key, default=None): - """Get value from key:value pair in comm string.""" - BAP_dict, _, _ = get_bap_comment(comm) - for e in get_bap_list(BAP_dict): - if isinstance(e, list) and len(e) <= 2: - # It is of the '(k v)' or '(t)' type - if e[0] == key: - try: - return e[1] - except IndexError: - return True - return default +if __name__ == "__main__": + import doctest + doctest.testmod() diff --git a/plugins/bap/utils/config.py b/plugins/bap/utils/config.py index 15c6fe5..0cb85c7 100644 --- a/plugins/bap/utils/config.py +++ b/plugins/bap/utils/config.py @@ -1,7 +1,7 @@ """Module for reading from and writing to the bap.cfg config file.""" import os -import idaapi #pylint: disable=import-error +import idaapi # pylint: disable=import-error CFG_DIR = idaapi.idadir('cfg') CFG_PATH = os.path.join(CFG_DIR, 'bap.cfg') @@ -62,7 +62,8 @@ def is_set(key): """ return get(key, default='0').lower() in ('1', 'true', 'yes') -def set(path, value): #pylint: disable=redefined-builtin + +def set(path, value): # pylint: disable=redefined-builtin """Set key:value in the config file.""" cfg = _read() key = Key(path) @@ -72,7 +73,7 @@ def set(path, value): #pylint: disable=redefined-builtin for i, line in enumerate(cfg[key.section]): if line[0] == ';': # Comment continue - elif line.split()[0] == key: + elif line.split()[0] == key.value: cfg[key.section][i] = '{}\t{}\t; Previously: {}'.format( key.value, value, line) break @@ -82,7 +83,7 @@ def set(path, value): #pylint: disable=redefined-builtin _write(cfg) -class Key(object): #pylint: disable=too-few-public-methods +class Key(object): # pylint: disable=too-few-public-methods "Configuration key" def __init__(self, path): elts = path.split('.') diff --git a/plugins/bap/utils/ida.py b/plugins/bap/utils/ida.py index b9fad36..5c45d01 100644 --- a/plugins/bap/utils/ida.py +++ b/plugins/bap/utils/ida.py @@ -1,15 +1,25 @@ """Utilities that interact with IDA.""" import idaapi +import idc +from bap.utils import bap_comment def add_to_comment(ea, key, value): """Add key:value to comm string at EA.""" - from bap_comment import add_to_comment_string - old_comm = idaapi.get_cmt(ea, 0) - if old_comm is None: - old_comm = '' - new_comm = add_to_comment_string(old_comm, key, value) - idaapi.set_cmt(ea, new_comm, 0) + cmt = idaapi.get_cmt(ea, 0) + comm = {} + if cmt: + comm = bap_comment.parse(cmt) + if comm is None: + comm = {} + if value == '()': + comm.setdefault(key, []) + else: + if key in comm: + comm[key].append(value) + else: + comm[key] = [value] + idaapi.set_cmt(ea, bap_comment.dumps(comm), 0) def cfunc_from_ea(ea): @@ -37,7 +47,7 @@ def dump_loader_info(output_filename): from idautils import Segments import idc - idaapi.autoWait() + idc.Wait() with open(output_filename, 'w+') as out: info = idaapi.get_inf_structure() @@ -163,7 +173,7 @@ def dump_brancher_info(output_filename): """Dump information for BAP's brancher into output_filename.""" from idautils import CodeRefsFrom - idaapi.autoWait() + idc.Wait() def dest(ea, flow): # flow denotes whether normal flow is also taken return set(CodeRefsFrom(ea, flow)) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..72ab1e7 --- /dev/null +++ b/setup.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python2.7 + +from setuptools import setup + +setup( + name='bap-ida-python', + version='0.2.0', + description='BAP IDA Plugin', + author='BAP Team', + url='https://github.com/BinaryAnalysisPlatform/bap-ida-python', + maintainer='Ivan Gotovchits', + maintainer_email='ivg@ieee.org', + license='MIT', + package_dir={'': 'plugins'}, + packages=['bap', 'bap.utils', 'bap.plugins'], + classifiers=[ + 'Development Status :: 3 - Alpha', + 'License :: OSI Approved :: MIT License', + 'Topic :: Software Development :: Disassemblers', + 'Topic :: Security' + ] +) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..58936ba --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,5 @@ +import sys + + +sys.modules['idaapi'] = __import__('mockidaapi') +sys.modules['idc'] = __import__('mockidc') diff --git a/tests/mockidaapi.py b/tests/mockidaapi.py new file mode 100644 index 0000000..b91feef --- /dev/null +++ b/tests/mockidaapi.py @@ -0,0 +1,12 @@ +# flake8: noqa + +ASKBTN_YES = NotImplemented +ASKBTN_NO = NotImplemented +ASKBTN_CANCEL = NotImplemented +PLUGIN_DRAW = NotImplemented +PLUGIN_KEEP = NotImplemented +class plugin_t(): NotImplemented +def idadir(sub): NotImplemented +def get_cmt(ea, off): NotImplemented +def set_cmt(ea, off): NotImplemented +def askyn_c(dflt, title): NotImplemented diff --git a/tests/mockidc.py b/tests/mockidc.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_bap_comment.py b/tests/test_bap_comment.py new file mode 100644 index 0000000..80252c1 --- /dev/null +++ b/tests/test_bap_comment.py @@ -0,0 +1,30 @@ +from bap.utils.bap_comment import parse, dumps + + +def test_parse(): + assert(parse('hello') is None) + assert(parse('BAP: hello') == {'hello': []}) + assert(parse('BAP: hello,world') == {'hello': [], 'world': []}) + assert(parse('BAP: hello=cruel,world') == {'hello': ['cruel', 'world']}) + assert(parse('BAP: hello="hello, world"') == {'hello': ['hello, world']}) + assert(parse('BAP: hello=cruel,world goodbye=real,life') == + {'hello': ['cruel', 'world'], 'goodbye': ['real', 'life']}) + assert(parse('BAP: hello="f\'"') == {'hello': ["f'"]}) + + +def test_dumps(): + assert('BAP:' in dumps({'hello': []})) + assert(dumps({'hello': ['cruel', 'world'], 'nice': [], 'thing': []}) == + 'BAP: nice,thing hello=cruel,world') + assert(dumps({'hello': ["world\'"]}) == 'BAP: hello="world\'"') + + +def test_roundup(): + comm = { + 'x': [], 'y': [], 'z': [], + 'a': ['1', '2', '3'], + 'b': ['thing\''], + 'c': ['many things'], + 'd': ['strange \\ things'], + } + assert(parse(dumps(parse(dumps(comm)))) == comm) diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..76799bd --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,11 @@ +def test_set_and_get(monkeypatch, tmpdir): + monkeypatch.setattr('idaapi.idadir', lambda x: + str(tmpdir.mkdir(x))) + from bap.utils.config import get, set, is_set + for path in ('foo', 'foo.bar'): + assert get(path) is None + set(path, 'hello') + assert get(path) == 'hello' + assert not is_set(path) + set(path, 'true') + assert is_set(path) diff --git a/tests/test_ida.py b/tests/test_ida.py new file mode 100644 index 0000000..5b4a1d7 --- /dev/null +++ b/tests/test_ida.py @@ -0,0 +1,56 @@ +import pytest + + +@pytest.fixture +def addresses(monkeypatch): + addresses = (0xDEADBEAF, 0xDEADBEEF) + monkeypatch.setattr('bap.utils.ida.all_valid_ea', lambda: addresses) + return addresses + + +@pytest.fixture +def comments(monkeypatch): + comments = {} + + def get_cmt(ea, off): + return comments.get(ea) + + def set_cmt(ea, val, off): + comments[ea] = val + monkeypatch.setattr('idaapi.get_cmt', get_cmt) + monkeypatch.setattr('idaapi.set_cmt', set_cmt) + return comments + + +@pytest.fixture(params=['yes', 'no', 'cancel']) +def choice(request, monkeypatch): + choice = request.param + monkeypatch.setattr('idaapi.ASKBTN_YES', 'yes') + monkeypatch.setattr('idaapi.ASKBTN_NO', 'no') + monkeypatch.setattr('idaapi.ASKBTN_CANCEL', 'cancel') + monkeypatch.setattr('idaapi.askyn_c', lambda d, t: request.param) + return choice + + +def test_comments(addresses, comments, choice): + import sys + sys.modules['idaapi'] = __import__('mockidaapi') + from bap.utils.ida import add_to_comment + from bap.plugins.bap_clear_comments import PLUGIN_ENTRY + for key in addresses: + add_to_comment(key, 'foo', 'bar') + assert comments[key] == 'BAP: foo=bar' + add_to_comment(key, 'foo', 'baz') + assert comments[key] == 'BAP: foo=bar,baz' + add_to_comment(key, 'bar', '()') + assert comments[key] == 'BAP: bar foo=bar,baz' + plugin = PLUGIN_ENTRY() + plugin.init() + plugin.run(0) + bap_cmts = [c for c in comments.values() if 'BAP:' in c] + expected = { + 'yes': 0, + 'no': len(addresses), + 'cancel': len(addresses), + } + assert len(bap_cmts) == expected[choice] diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..94bb50a --- /dev/null +++ b/tox.ini @@ -0,0 +1,6 @@ +[tox] +envlist=py27,py24,py35 + +[testenv] +deps=pytest +commands=py.test \ No newline at end of file From e4d7b03a30ff21b469c6f4b6bf7fcb8422cd9bd4 Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Fri, 16 Dec 2016 08:49:37 -0500 Subject: [PATCH 03/20] added travis file to trigger the test --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..bf92d2a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,4 @@ +sudo: false +language: python +install: pip install tox +script: tox \ No newline at end of file From 1453d0f593d662b4b97f0be5aea2d26ce42041ae Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Fri, 16 Dec 2016 08:52:47 -0500 Subject: [PATCH 04/20] added explicit requirement for py35 --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bf92d2a..1903249 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,8 @@ sudo: false language: python +python: + - "2.4" + - "2.7" + - "3.5" install: pip install tox -script: tox \ No newline at end of file +script: tox From 591f1c7c3e838ae9b391e6eb01e9e21f856e925b Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Fri, 16 Dec 2016 08:55:49 -0500 Subject: [PATCH 05/20] it looks like 3.5 includes previous one Basically tox here competes with travis. Tox is capable of running tests in multiple environments, and the same feature is provided by Travis. I would like to use Tox here, to have the same setup as I have on my local machine. It is possible to use tox-travis, but it will still have a setup that would be different from the local one. --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1903249..383828e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,6 @@ sudo: false language: python python: - - "2.4" - - "2.7" - "3.5" install: pip install tox script: tox From a16f4d0c7bf979a0e01acd0ac6f300f2da37c3f3 Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Fri, 16 Dec 2016 09:20:43 -0500 Subject: [PATCH 06/20] refactored ida specific mocking into idapatch fixture --- tests/test_ida.py | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/tests/test_ida.py b/tests/test_ida.py index 5b4a1d7..b99be5a 100644 --- a/tests/test_ida.py +++ b/tests/test_ida.py @@ -1,6 +1,14 @@ import pytest +@pytest.fixture +def idapatch(monkeypatch): + def patch(attrs, ns='idaapi'): + for (k, v) in attrs.items(): + monkeypatch.setattr(ns + '.' + k, v, raising=False) + return patch + + @pytest.fixture def addresses(monkeypatch): addresses = (0xDEADBEAF, 0xDEADBEEF) @@ -9,26 +17,27 @@ def addresses(monkeypatch): @pytest.fixture -def comments(monkeypatch): - comments = {} - - def get_cmt(ea, off): - return comments.get(ea) +def comments(idapatch): + cmts = {} def set_cmt(ea, val, off): - comments[ea] = val - monkeypatch.setattr('idaapi.get_cmt', get_cmt) - monkeypatch.setattr('idaapi.set_cmt', set_cmt) - return comments + cmts[ea] = val + idapatch({ + 'get_cmt': lambda ea, off: cmts.get(ea), + 'set_cmt': set_cmt + }) + return cmts @pytest.fixture(params=['yes', 'no', 'cancel']) -def choice(request, monkeypatch): +def choice(request, idapatch): choice = request.param - monkeypatch.setattr('idaapi.ASKBTN_YES', 'yes') - monkeypatch.setattr('idaapi.ASKBTN_NO', 'no') - monkeypatch.setattr('idaapi.ASKBTN_CANCEL', 'cancel') - monkeypatch.setattr('idaapi.askyn_c', lambda d, t: request.param) + idapatch({ + 'ASKBTN_YES': 'yes', + 'ASKBTN_NO': 'no', + 'ASKBTN_CANCEL': 'cancel', + 'askyn_c': lambda d, t: request.param + }) return choice From 85baf5213cad1aef511153a952347ce055efa4fd Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Fri, 16 Dec 2016 09:22:54 -0500 Subject: [PATCH 07/20] we can mock the import on the conftest level --- tests/test_ida.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_ida.py b/tests/test_ida.py index b99be5a..95d25dc 100644 --- a/tests/test_ida.py +++ b/tests/test_ida.py @@ -42,8 +42,6 @@ def choice(request, idapatch): def test_comments(addresses, comments, choice): - import sys - sys.modules['idaapi'] = __import__('mockidaapi') from bap.utils.ida import add_to_comment from bap.plugins.bap_clear_comments import PLUGIN_ENTRY for key in addresses: From 1670b9da4bf05156cf4af30fe3729d41631597fd Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Fri, 16 Dec 2016 09:28:10 -0500 Subject: [PATCH 08/20] moved fixtures away from tests also pep8ted bap.utils.run module --- plugins/bap/utils/run.py | 40 +++++++++++++++++--------------------- tests/conftest.py | 42 +++++++++++++++++++++++++++++++++++++++- tests/test_ida.py | 42 ---------------------------------------- 3 files changed, 59 insertions(+), 65 deletions(-) diff --git a/plugins/bap/utils/run.py b/plugins/bap/utils/run.py index 3bc7b9d..3e42c50 100644 --- a/plugins/bap/utils/run.py +++ b/plugins/bap/utils/run.py @@ -9,12 +9,12 @@ import traceback -import idc #pylint: disable=import-error -import idaapi #pylint: disable=import-error +import idc # pylint: disable=import-error +import idaapi # pylint: disable=import-error from bap.utils import ida, config -#pylint: disable=missing-docstring +# pylint: disable=missing-docstring class Bap(object): """Bap instance base class. @@ -25,10 +25,8 @@ class Bap(object): specifics, so that later we can lift it to the bap-python library """ - DEBUG = False - def __init__(self, bap, input_file): """Sandbox for the BAP process. @@ -57,7 +55,6 @@ def __init__(self, bap, input_file): self.out = self.tmpfile("out") self.action = "running bap" - def run(self): "starts BAP process" if self.DEBUG: @@ -67,7 +64,7 @@ def run(self): stdout=self.out, stderr=subprocess.STDOUT, env={ - 'BAP_LOG_DIR' : self.tmpdir + 'BAP_LOG_DIR': self.tmpdir }) def finished(self): @@ -103,7 +100,7 @@ def cleanup(self): def tmpfile(self, suffix, *args, **kwargs): "creates a new temporary files in the self.tmpdir" if getattr(self, 'tmpdir', None) is None: - #pylint: disable=attribute-defined-outside-init + # pylint: disable=attribute-defined-outside-init self.tmpdir = tempfile.mkdtemp(prefix="bap") tmp = tempfile.NamedTemporaryFile( @@ -116,6 +113,7 @@ def tmpfile(self, suffix, *args, **kwargs): self.fds.append(tmp) return tmp + class BapIda(Bap): """BAP instance in IDA. @@ -129,12 +127,11 @@ class BapIda(Bap): # useful, for handling gui. See also, on_finished # and on_cancel, for user specific handlers. observers = { - 'instance_created' : [], - 'instance_updated' : [], - 'instance_finished' : [], + 'instance_created': [], + 'instance_updated': [], + 'instance_finished': [], } - def __init__(self, symbols=True): try: check_and_configure_bap() @@ -176,7 +173,7 @@ def run(self): self._do_run() idc.Message("BAP> total number of running instances: {0}\n". - format(len(BapIda.instances))) + format(len(BapIda.instances))) def _setup_symbols(self): "pass symbol information from IDA to BAP" @@ -188,7 +185,6 @@ def _setup_symbols(self): "--rooter=file" ] - def _setup_headers(self, bap): "pass type information from IDA to BAP" # this is very fragile, and may break in case @@ -210,7 +206,6 @@ def cleanup(): self._on_cancel.append(cleanup) self._on_finish.append(cleanup) - def _do_run(self): try: super(BapIda, self).run() @@ -219,27 +214,25 @@ def _do_run(self): idc.SetStatus(idc.IDA_STATUS_THINKING) self.run_handlers(self.observers['instance_created']) idc.Message("BAP> created new instance with PID {0}\n". - format(self.proc.pid)) - except: #pylint: disable=bare-except + format(self.proc.pid)) + except: # pylint: disable=bare-except idc.Message("BAP> failed to create instance\nError: {0}\n". format(str(sys.exc_info()[1]))) traceback.print_exc() - def run_handlers(self, handlers): failures = 0 for handler in handlers: try: handler(self) - except: #pylint: disable=bare-except + except: # pylint: disable=bare-except failures += 1 idc.Message("BAP> {0} failed because {1}\n". - format(self.action, str(sys.exc_info()[1]))) + format(self.action, str(sys.exc_info()[1]))) traceback.print_exc() if failures != 0: idc.Warning("Some BAP handlers failed") - def close(self): super(BapIda, self).close() BapIda.instances.remove(self) @@ -282,9 +275,11 @@ def on_finish(self, callback): def on_cancel(self, callback): self._on_cancel.append(callback) + class BapIdaError(Exception): pass + class BapNotFound(BapIdaError): pass @@ -321,7 +316,6 @@ def check_and_configure_bap(): config.set('bap_executable_path', bap_path) - def system_path(): try: return subprocess.check_output(['which', 'bap']).strip() @@ -337,6 +331,7 @@ def opam_path(): except OSError: return None + def ask_user(default_path): def confirm(msg): return idaapi.askyn_c(idaapi.ASKBTN_YES, msg) == idaapi.ASKBTN_YES @@ -356,5 +351,6 @@ def confirm(msg): continue return bap_path + BAP_FINDERS.append(system_path) BAP_FINDERS.append(opam_path) diff --git a/tests/conftest.py b/tests/conftest.py index 58936ba..c62f2e4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,45 @@ import sys - +import pytest sys.modules['idaapi'] = __import__('mockidaapi') sys.modules['idc'] = __import__('mockidc') + + +@pytest.fixture +def idapatch(monkeypatch): + def patch(attrs, ns='idaapi'): + for (k, v) in attrs.items(): + monkeypatch.setattr(ns + '.' + k, v, raising=False) + return patch + + +@pytest.fixture +def addresses(monkeypatch): + addresses = (0xDEADBEAF, 0xDEADBEEF) + monkeypatch.setattr('bap.utils.ida.all_valid_ea', lambda: addresses) + return addresses + + +@pytest.fixture +def comments(idapatch): + cmts = {} + + def set_cmt(ea, val, off): + cmts[ea] = val + idapatch({ + 'get_cmt': lambda ea, off: cmts.get(ea), + 'set_cmt': set_cmt + }) + return cmts + + +@pytest.fixture(params=['yes', 'no', 'cancel']) +def choice(request, idapatch): + choice = request.param + idapatch({ + 'ASKBTN_YES': 'yes', + 'ASKBTN_NO': 'no', + 'ASKBTN_CANCEL': 'cancel', + 'askyn_c': lambda d, t: request.param + }) + return choice diff --git a/tests/test_ida.py b/tests/test_ida.py index 95d25dc..4a22970 100644 --- a/tests/test_ida.py +++ b/tests/test_ida.py @@ -1,45 +1,3 @@ -import pytest - - -@pytest.fixture -def idapatch(monkeypatch): - def patch(attrs, ns='idaapi'): - for (k, v) in attrs.items(): - monkeypatch.setattr(ns + '.' + k, v, raising=False) - return patch - - -@pytest.fixture -def addresses(monkeypatch): - addresses = (0xDEADBEAF, 0xDEADBEEF) - monkeypatch.setattr('bap.utils.ida.all_valid_ea', lambda: addresses) - return addresses - - -@pytest.fixture -def comments(idapatch): - cmts = {} - - def set_cmt(ea, val, off): - cmts[ea] = val - idapatch({ - 'get_cmt': lambda ea, off: cmts.get(ea), - 'set_cmt': set_cmt - }) - return cmts - - -@pytest.fixture(params=['yes', 'no', 'cancel']) -def choice(request, idapatch): - choice = request.param - idapatch({ - 'ASKBTN_YES': 'yes', - 'ASKBTN_NO': 'no', - 'ASKBTN_CANCEL': 'cancel', - 'askyn_c': lambda d, t: request.param - }) - return choice - def test_comments(addresses, comments, choice): from bap.utils.ida import add_to_comment From a87078feb971f11ff77af76f20ff3621f16ae851 Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Fri, 16 Dec 2016 16:10:02 -0500 Subject: [PATCH 09/20] added test of bap configuration --- plugins/bap/utils/run.py | 21 ++++++++----- tests/conftest.py | 66 ++++++++++++++++++++++++++++++++++++++++ tests/test_config.py | 4 +-- tests/test_ida.py | 1 - tests/test_run.py | 10 ++++++ 5 files changed, 90 insertions(+), 12 deletions(-) create mode 100644 tests/test_run.py diff --git a/plugins/bap/utils/run.py b/plugins/bap/utils/run.py index 3e42c50..1671621 100644 --- a/plugins/bap/utils/run.py +++ b/plugins/bap/utils/run.py @@ -310,25 +310,30 @@ def check_and_configure_bap(): break # always ask a user to confirm the path that was found using heuristics - user_path = ask_user(bap_path) - if user_path: - bap_path = user_path - config.set('bap_executable_path', bap_path) + bap_path = ask_user(bap_path) + if bap_path and len(bap_path) > 0: + config.set('bap_executable_path', bap_path) -def system_path(): + +def preadline(cmd): try: - return subprocess.check_output(['which', 'bap']).strip() + res = subprocess.check_output(cmd, universal_newlines=True) + return res.strip() except (OSError, subprocess.CalledProcessError): return None +def system_path(): + return preadline(['which', 'bap']) + + def opam_path(): try: cmd = ['opam', 'config', 'var', 'bap:bin'] - res = subprocess.check_output(cmd).strip() + res = preadline(cmd).strip() return os.path.join(res, 'bap') - except OSError: + except: return None diff --git a/tests/conftest.py b/tests/conftest.py index c62f2e4..6bb42ba 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,9 @@ import sys +import subprocess + import pytest + sys.modules['idaapi'] = __import__('mockidaapi') sys.modules['idc'] = __import__('mockidc') @@ -43,3 +46,66 @@ def choice(request, idapatch): 'askyn_c': lambda d, t: request.param }) return choice + + +BAP_PATH = '/opt/bin/bap' + + +@pytest.fixture(params=[ + ('stupid', None, 'what?', 'oh, okay', 'bap'), + ('clever', )]) +def askbap(request, idapatch, monkeypatch): + param = list(request.param) + user = param.pop(0) + + monkeypatch.setattr('os.path.isfile', lambda p: p == BAP_PATH) + idapatch({'ASKBTN_YES': 'yes', 'askyn_c': lambda d, t: 'yes'}) + + def ask(unk, path, msg): + if user == 'clever': + return path + elif user == 'stupid': + if len(param) > 0: + return param.pop(0) + idapatch({'askfile_c': ask}) + return {'user': user, 'path': BAP_PATH} + + +@pytest.fixture +def idadir(idapatch, tmpdir): + idapatch({'idadir': lambda x: str(tmpdir.mkdir(x))}) + return tmpdir.dirname + + +class Popen(subprocess.Popen): + patches = {} + + def __init__(self, args, **kwargs): + cmd = ' '.join(args) + if cmd in Popen.patches: + super(Popen, self).__init__( + Popen.patches[cmd], + shell=True, + **kwargs) + else: + super(Popen, self).__init__(args, **kwargs) + + +@pytest.fixture +def popenpatch(monkeypatch): + monkeypatch.setattr('subprocess.Popen', Popen) + + def patch(cmd, script): + Popen.patches[cmd] = script + return patch + + +@pytest.fixture(params=[None, BAP_PATH]) +def bappath(request, popenpatch): + path = request.param + if path: + popenpatch('which bap', 'echo {}'.format(path)) + else: + popenpatch('which bap', 'false') + popenpatch('opam config var bap:bin', 'echo undefind; false') + return path diff --git a/tests/test_config.py b/tests/test_config.py index 76799bd..dd7d78e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,6 +1,4 @@ -def test_set_and_get(monkeypatch, tmpdir): - monkeypatch.setattr('idaapi.idadir', lambda x: - str(tmpdir.mkdir(x))) +def test_set_and_get(idadir): from bap.utils.config import get, set, is_set for path in ('foo', 'foo.bar'): assert get(path) is None diff --git a/tests/test_ida.py b/tests/test_ida.py index 4a22970..8681aec 100644 --- a/tests/test_ida.py +++ b/tests/test_ida.py @@ -1,4 +1,3 @@ - def test_comments(addresses, comments, choice): from bap.utils.ida import add_to_comment from bap.plugins.bap_clear_comments import PLUGIN_ENTRY diff --git a/tests/test_run.py b/tests/test_run.py new file mode 100644 index 0000000..b05f9ee --- /dev/null +++ b/tests/test_run.py @@ -0,0 +1,10 @@ +def test_check_and_configure_bap(bappath, askbap, idadir): + from bap.utils.run import check_and_configure_bap + from bap.utils import config + check_and_configure_bap() + bap = config.get('bap_executable_path') + expected = { + 'clever': bappath, + 'stupid': None + } + assert bap == expected[askbap['user']] From 258867d9852d554adc6293c03de8d443270f03d5 Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Mon, 19 Dec 2016 14:11:04 -0500 Subject: [PATCH 10/20] make popen patches more generic now it is possible to specify a test function, instead of just a string, that will test for matching command invokation. Also, fixes a hidden bug. We didn't perform the tear down of the added patches after the fixture was used. --- tests/conftest.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6bb42ba..46f2bd2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -78,13 +78,23 @@ def idadir(idapatch, tmpdir): class Popen(subprocess.Popen): - patches = {} + patches = [] def __init__(self, args, **kwargs): cmd = ' '.join(args) - if cmd in Popen.patches: + patch = None + for p in Popen.patches: + if 'test' in p: + if p['test'](args) or p['test'](cmd): + patch = p + break + else: + if p['cmd'] == cmd: + patch = p + break + if patch: super(Popen, self).__init__( - Popen.patches[cmd], + patch['script'], shell=True, **kwargs) else: @@ -94,10 +104,17 @@ def __init__(self, args, **kwargs): @pytest.fixture def popenpatch(monkeypatch): monkeypatch.setattr('subprocess.Popen', Popen) - + def patch(cmd, script): - Popen.patches[cmd] = script - return patch + patch = {} + if callable(cmd): + patch['test'] = cmd + else: + patch['cmd'] = cmd + patch['script'] = script + Popen.patches.append(patch) + yield patch + Popen.patches = [] @pytest.fixture(params=[None, BAP_PATH]) @@ -109,3 +126,5 @@ def bappath(request, popenpatch): popenpatch('which bap', 'false') popenpatch('opam config var bap:bin', 'echo undefind; false') return path + + From a594d2b14ec5ffb364869a8e07f6d6f2d26653e2 Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Mon, 19 Dec 2016 14:12:58 -0500 Subject: [PATCH 11/20] do not hardcode a specific version of python3 for portability across different systems. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 94bb50a..24bd0c8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py27,py24,py35 +envlist=py27,py24,py3 [testenv] deps=pytest From 03554288c86351bba1d40f336aaeedc1c91963cc Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Mon, 19 Dec 2016 17:08:26 -0500 Subject: [PATCH 12/20] added more tests test that BAP is started, and that handlers are called --- plugins/bap/utils/run.py | 2 +- tests/conftest.py | 111 ++++++++++++++++++++++++++++++--------- tests/mockidaapi.py | 1 + tests/mockidc.py | 9 ++++ tests/test_run.py | 58 ++++++++++++++++++++ 5 files changed, 156 insertions(+), 25 deletions(-) diff --git a/plugins/bap/utils/run.py b/plugins/bap/utils/run.py index 1671621..3fcae26 100644 --- a/plugins/bap/utils/run.py +++ b/plugins/bap/utils/run.py @@ -144,7 +144,7 @@ def __init__(self, symbols=True): if bap is None: idc.Warning("Can't locate BAP\n") raise BapNotFound() - binary = idc.GetInputFilePath() + binary = idaapi.get_input_file_path() super(BapIda, self).__init__(bap, binary) # if you run IDA inside IDA you will crash IDA self.args.append('--no-ida') diff --git a/tests/conftest.py b/tests/conftest.py index 46f2bd2..c37d4bc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -81,22 +81,11 @@ class Popen(subprocess.Popen): patches = [] def __init__(self, args, **kwargs): - cmd = ' '.join(args) - patch = None - for p in Popen.patches: - if 'test' in p: - if p['test'](args) or p['test'](cmd): - patch = p - break - else: - if p['cmd'] == cmd: - patch = p - break - if patch: - super(Popen, self).__init__( - patch['script'], - shell=True, - **kwargs) + for patch in Popen.patches: + script = patch(args) + if script: + super(Popen, self).__init__(script, shell=True, **kwargs) + break else: super(Popen, self).__init__(args, **kwargs) @@ -104,15 +93,21 @@ def __init__(self, args, **kwargs): @pytest.fixture def popenpatch(monkeypatch): monkeypatch.setattr('subprocess.Popen', Popen) - - def patch(cmd, script): - patch = {} - if callable(cmd): - patch['test'] = cmd - else: - patch['cmd'] = cmd - patch['script'] = script + + def same_cmd(cmd, args): + return cmd == ' '.join(args) + + def add(patch): Popen.patches.append(patch) + + def patch(*args): + if len(args) == 1: + add(args[0]) + elif len(args) == 2: + add(lambda pargs: args[1] if same_cmd(args[0], pargs) else None) + else: + raise TypeError('popenpatch() takes 1 or two arguments ({} given)'. + format(len(args))) yield patch Popen.patches = [] @@ -128,3 +123,71 @@ def bappath(request, popenpatch): return path +class Ida(object): + + def __init__(self): + self.time = 0 + self.callbacks = [] + self.log = [] + self.status = 'ready' + + def register_timer(self, interval, cb): + self.callbacks.append({ + 'time': self.time + interval, + 'call': cb + }) + + def message(self, msg): + self.log.append(msg) + + def set_status(self, status): + self.status = status + + def run(self): + while self.callbacks: + self.time += 1 + for cb in self.callbacks: + if cb['time'] < self.time: + time = cb['call']() + if time < 0: + self.callbacks.remove(cb) + else: + cb['time'] = self.time + time + + +class Bap(object): + + def __init__(self, path): + self.path = path + self.calls = [] + + def call(self, args): + self.calls.append({'args': args}) + return True + + +@pytest.fixture +def bapida(idapatch, popenpatch, monkeypatch, idadir): + from bap.utils import config + ida = Ida() + bap = Bap(BAP_PATH) + + def run_bap(args): + if args[0] == BAP_PATH: + if bap.call(args): + return 'true' + else: + return 'false' + + config.set('bap_executable_path', bap.path) + idapatch({ + 'register_timer': ida.register_timer, + 'get_input_ida_file_path': lambda: '/bin/true' + }) + idapatch(ns='idc', attrs={ + 'Message': ida.message, + 'SetStatus': ida.set_status, + }) + popenpatch(run_bap) + monkeypatch.setattr('bap.utils.ida.dump_symbol_info', lambda out: None) + return (bap, ida) diff --git a/tests/mockidaapi.py b/tests/mockidaapi.py index b91feef..f2929bb 100644 --- a/tests/mockidaapi.py +++ b/tests/mockidaapi.py @@ -10,3 +10,4 @@ def idadir(sub): NotImplemented def get_cmt(ea, off): NotImplemented def set_cmt(ea, off): NotImplemented def askyn_c(dflt, title): NotImplemented +def get_input_file_path() : NotImplemented diff --git a/tests/mockidc.py b/tests/mockidc.py index e69de29..58d1421 100644 --- a/tests/mockidc.py +++ b/tests/mockidc.py @@ -0,0 +1,9 @@ +# flake8: noqa + +IDA_STATUS_THINKING="thinking" +IDA_STATUS_READY='ready' +IDA_STATUS_WAITING='waiting' +IDA_STATUS_WORK='work' + +def Message(msg): NotImplemented +def SetStatus(s): NotImplemented diff --git a/tests/test_run.py b/tests/test_run.py index b05f9ee..73186b5 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -1,3 +1,6 @@ +from functools import partial + + def test_check_and_configure_bap(bappath, askbap, idadir): from bap.utils.run import check_and_configure_bap from bap.utils import config @@ -8,3 +11,58 @@ def test_check_and_configure_bap(bappath, askbap, idadir): 'stupid': None } assert bap == expected[askbap['user']] + + +def test_run_without_args(bapida): + from bap.utils.run import BapIda + backend, frontend = bapida + bap = BapIda() + bap.run() + frontend.run() + assert len(backend.calls) == 1 + args = backend.calls[0]['args'] + assert args[0] == backend.path + assert '--no-ida' in args + assert '--read-symbols-from' in args + assert '--symbolizer=file' in args + + +def test_disable_symbols(bapida): + from bap.utils.run import BapIda + backend, frontend = bapida + bap = BapIda(symbols=False) + bap.run() + frontend.run() + assert len(backend.calls) == 1 + args = backend.calls[0]['args'] + assert args[0] == backend.path + assert '--no-ida' in args + assert '--read-symbols-from' not in args + assert '--symbolizer=file' not in args + + +def test_event_handlers(bapida): + from bap.utils.run import BapIda + backend, frontend = bapida + bap = BapIda() + bap.events = [] + + def occured(bap, event): + bap.events.append(event) + + events = ('instance_created', 'instance_updated', 'instance_finished') + for event in events: + BapIda.observers[event].append(partial(occured, event=event)) + + bap.on_finish(lambda bap: occured(bap, 'success')) + + bap.run() + frontend.run() + + for msg in frontend.log: + print(msg) + + for event in events: + assert event in bap.events + + assert 'success' in bap.events From a85d8a6f2dd0049dc0fda41105bbe03bdd0027f4 Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Mon, 19 Dec 2016 17:41:58 -0500 Subject: [PATCH 13/20] added failure test --- plugins/bap/utils/run.py | 2 -- tests/conftest.py | 15 +++++++++------ tests/test_run.py | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/plugins/bap/utils/run.py b/plugins/bap/utils/run.py index 3fcae26..2cb7073 100644 --- a/plugins/bap/utils/run.py +++ b/plugins/bap/utils/run.py @@ -247,8 +247,6 @@ def update(self): elif self.proc.returncode > 0: idc.Message("BAP> an error has occured while {0}\n". format(self.action)) - with open(self.out.name) as out: - idc.Message('BAP> output:\n{0}\n'.format(out.read())) else: idc.Message("BAP> was killed by signal {0}\n". format(-self.proc.returncode)) diff --git a/tests/conftest.py b/tests/conftest.py index c37d4bc..4fbcb04 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -160,10 +160,16 @@ class Bap(object): def __init__(self, path): self.path = path self.calls = [] + self.on_call = [] def call(self, args): - self.calls.append({'args': args}) - return True + proc = {'args': args} + self.calls.append(proc) + for call in self.on_call: + res = call(self, proc) + if res is not None: + return res + return 0 @pytest.fixture @@ -174,10 +180,7 @@ def bapida(idapatch, popenpatch, monkeypatch, idadir): def run_bap(args): if args[0] == BAP_PATH: - if bap.call(args): - return 'true' - else: - return 'false' + return 'exit ' + str(bap.call(args)) config.set('bap_executable_path', bap.path) idapatch({ diff --git a/tests/test_run.py b/tests/test_run.py index 73186b5..fc9fe6e 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -66,3 +66,21 @@ def occured(bap, event): assert event in bap.events assert 'success' in bap.events + + +def test_failure(bapida): + from bap.utils.run import BapIda + backend, frontend = bapida + bap = BapIda() + bap.events = [] + + backend.on_call.append(lambda bap, args: 1) + bap.on_finish(lambda bap: bap.events.append('success')) + + bap.run() + frontend.run() + + for msg in frontend.log: + print(msg) + + assert 'success' not in bap.events From 76e65f59f53439b1eafa7469bbe83a198d04234c Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Tue, 20 Dec 2016 11:59:13 -0500 Subject: [PATCH 14/20] redesigned event system --- plugins/bap/utils/run.py | 64 +++++++++++++++++++++++++---------- tests/conftest.py | 72 +++++++++++++++++++++++++++++++++++++--- tests/mockidc.py | 1 + tests/test_run.py | 22 ++++++++++++ 4 files changed, 138 insertions(+), 21 deletions(-) diff --git a/plugins/bap/utils/run.py b/plugins/bap/utils/run.py index 2cb7073..ba4591d 100644 --- a/plugins/bap/utils/run.py +++ b/plugins/bap/utils/run.py @@ -49,11 +49,13 @@ def __init__(self, bap, input_file): - `fds` a list of opened filedescriptors to be closed on the exit """ + self.tmpdir = None self.args = [bap, input_file] self.proc = None self.fds = [] self.out = self.tmpfile("out") self.action = "running bap" + self.closed = False def run(self): "starts BAP process" @@ -68,7 +70,7 @@ def run(self): }) def finished(self): - "true is the process has finished" + "true if the process is no longer running" return self.proc is not None and self.proc.poll() is not None def close(self): @@ -78,6 +80,7 @@ def close(self): self.proc.terminate() self.proc.wait() self.cleanup() + self.closed = True def cleanup(self): """Close and remove all created temporary files. @@ -99,8 +102,7 @@ def cleanup(self): def tmpfile(self, suffix, *args, **kwargs): "creates a new temporary files in the self.tmpdir" - if getattr(self, 'tmpdir', None) is None: - # pylint: disable=attribute-defined-outside-init + if self.tmpdir is None: self.tmpdir = tempfile.mkdtemp(prefix="bap") tmp = tempfile.NamedTemporaryFile( @@ -129,6 +131,8 @@ class BapIda(Bap): observers = { 'instance_created': [], 'instance_updated': [], + 'instance_canceled': [], + 'instance_failed': [], 'instance_finished': [], } @@ -141,8 +145,12 @@ def __init__(self, symbols=True): traceback.print_exc() raise BapIdaError() bap = config.get('bap_executable_path') - if bap is None: - idc.Warning("Can't locate BAP\n") + if bap is None or not os.access(bap, os.X_OK): + idc.Warning(''' + The bap application is either not found or is not an executable. + Please install bap or, if it is installed, provide a path to it. + Installation instructions are available at: http://bap.ece.cmu.edu. + ''') raise BapNotFound() binary = idaapi.get_input_file_path() super(BapIda, self).__init__(bap, binary) @@ -150,6 +158,7 @@ def __init__(self, symbols=True): self.args.append('--no-ida') self._on_finish = [] self._on_cancel = [] + self._on_failed = [] if symbols: self._setup_symbols() @@ -203,16 +212,15 @@ def cleanup(): "--api-remove", "c:{0}". format(os.path.basename(out.name)) ]) - self._on_cancel.append(cleanup) - self._on_finish.append(cleanup) + self.on_cleanup(cleanup) def _do_run(self): try: super(BapIda, self).run() BapIda.instances.append(self) - idaapi.register_timer(200, self.update) + idaapi.register_timer(self.poll_interval_ms, self.update) idc.SetStatus(idc.IDA_STATUS_THINKING) - self.run_handlers(self.observers['instance_created']) + self.run_handlers('instance_created') idc.Message("BAP> created new instance with PID {0}\n". format(self.proc.pid)) except: # pylint: disable=bare-except @@ -220,7 +228,18 @@ def _do_run(self): format(str(sys.exc_info()[1]))) traceback.print_exc() - def run_handlers(self, handlers): + def run_handlers(self, event): + assert event in self.observers + handlers = [] + instance_handlers = { + 'instance_canceled': self._on_cancel, + 'instance_failed': self._on_failed, + 'instance_finished': self._on_finish, + } + + handlers += self.observers[event] + handlers += instance_handlers.get(event, []) + failures = 0 for handler in handlers: try: @@ -240,19 +259,22 @@ def close(self): def update(self): if self.finished(): if self.proc.returncode == 0: - self.run_handlers(self._on_finish) - self.run_handlers(self.observers['instance_finished']) + self.run_handlers('instance_finished') self.close() idc.Message("BAP> finished " + self.action + '\n') elif self.proc.returncode > 0: + self.run_handlers('instance_failed') + self.close() idc.Message("BAP> an error has occured while {0}\n". format(self.action)) else: + if not self.closed: + self.run_handlers('instance_canceled') idc.Message("BAP> was killed by signal {0}\n". format(-self.proc.returncode)) return -1 else: - self.run_handlers(self.observers['instance_updated']) + self.run_handlers('instance_updated') thinking = False for bap in BapIda.instances: if bap.finished(): @@ -260,26 +282,34 @@ def update(self): thinking = True if not thinking: idc.SetStatus(idc.IDA_STATUS_READY) - return 200 + return self.poll_interval_ms def cancel(self): - self.run_handlers(self._on_cancel) - self.run_handlers(self.observers['instance_finished']) + self.run_handlers('instance_canceled') self.close() + def on_cleanup(self, callback): + self.on_finish(callback) + self.on_cancel(callback) + self.on_failed(callback) + def on_finish(self, callback): self._on_finish.append(callback) def on_cancel(self, callback): self._on_cancel.append(callback) + def on_failed(self, callback): + self._on_failed.append(callback) + class BapIdaError(Exception): pass class BapNotFound(BapIdaError): - pass + def __str__(self): + return 'Unable to detect bap executable ' BAP_FINDERS = [] diff --git a/tests/conftest.py b/tests/conftest.py index 4fbcb04..59044a4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ import sys import subprocess +from time import sleep, time as current_time import pytest @@ -124,14 +125,26 @@ def bappath(request, popenpatch): class Ida(object): + '''IDA instance imitator. ''' def __init__(self): self.time = 0 self.callbacks = [] self.log = [] + self.warnings = [] self.status = 'ready' def register_timer(self, interval, cb): + '''add a recurrent callback. + + Registers a function that will be called after the specified + ``interval``. The function may return a positive value, that + will effectively re-register itself. If a negative value is + returned, then the callback will not be called anymore. + + Note: the realtime clocks are imitated with the ``sleep`` + function using 10ms increments. + ''' self.callbacks.append({ 'time': self.time + interval, 'call': cb @@ -140,29 +153,73 @@ def register_timer(self, interval, cb): def message(self, msg): self.log.append(msg) + def warning(self, msg): + self.warnings.append(msg) + def set_status(self, status): self.status = status def run(self): + '''Runs IDA event cycle. + The function will return if there are no more callbacks. + ''' while self.callbacks: - self.time += 1 + sleep(0.1) + self.time += 100 for cb in self.callbacks: if cb['time'] < self.time: time = cb['call']() - if time < 0: + if time is None or time < 0: self.callbacks.remove(cb) else: cb['time'] = self.time + time class Bap(object): + '''BAP utility mock. + + From the perspective of the IDA integration, the bap frontend + utility is considered a backend. So, we will refer to it as a + backend here and later. + + This mock class mimicks the behavior of the bap backend, as the + unit tests must not dependend on the presence of the bap + framework. + + The instances of the backend has the following attributes: + + - ``path`` a path to bap executable + - ``calls`` a list of calls made to backend, each call is + a dictionary that has at least these fields: + - args - arguments passed to the Popen call + - ``on_call`` a list of the call event handlers. An event + handler should be a callable, that accepts two arguments. + The first argument is a reference to the bap backend instance, + the second is a reference to the ``proc`` dictionary (as described + above). If a callback returns a non None value, then this value is + used as a return value of the call to BAP. See ``call`` method + description for more information about the return values. + ''' def __init__(self, path): + '''initializes BAP backend instance. + + Once instance corresponds to one bap installation (not to a + process instance). See class descriptions for information about + the instance attributes. + ''' self.path = path self.calls = [] self.on_call = [] def call(self, args): + '''mocks a call to a bap executable. + + If a call returns with an integer, then it is passed to the + shell's exit command, otherwise a string representation of a + returned value is used to form a command, that is then passed + to a Popen constructor. + ''' proc = {'args': args} self.calls.append(proc) for call in self.on_call: @@ -180,15 +237,22 @@ def bapida(idapatch, popenpatch, monkeypatch, idadir): def run_bap(args): if args[0] == BAP_PATH: - return 'exit ' + str(bap.call(args)) + res = bap.call(args) or 0 + if isinstance(res, int): + return 'exit ' + str(res) + else: + return str(res) config.set('bap_executable_path', bap.path) + monkeypatch.setattr('os.access', lambda p, x: p == BAP_PATH) + idapatch({ 'register_timer': ida.register_timer, - 'get_input_ida_file_path': lambda: '/bin/true' + 'get_input_ida_file_path': lambda: 'true' }) idapatch(ns='idc', attrs={ 'Message': ida.message, + 'Warning': ida.warning, 'SetStatus': ida.set_status, }) popenpatch(run_bap) diff --git a/tests/mockidc.py b/tests/mockidc.py index 58d1421..e78f684 100644 --- a/tests/mockidc.py +++ b/tests/mockidc.py @@ -6,4 +6,5 @@ IDA_STATUS_WORK='work' def Message(msg): NotImplemented +def Warning(msg): NotImplemented def SetStatus(s): NotImplemented diff --git a/tests/test_run.py b/tests/test_run.py index fc9fe6e..127457f 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -54,6 +54,7 @@ def occured(bap, event): for event in events: BapIda.observers[event].append(partial(occured, event=event)) + backend.on_call.append(lambda bap, args: 'sleep 1') bap.on_finish(lambda bap: occured(bap, 'success')) bap.run() @@ -84,3 +85,24 @@ def test_failure(bapida): print(msg) assert 'success' not in bap.events + assert len(BapIda.instances) == 0 + + +def test_cancel(bapida): + from bap.utils.run import BapIda + backend, frontend = bapida + bap = BapIda() + bap.events = [] + + backend.on_call.append(lambda bap, args: 'sleep 100') + frontend.register_timer(600, lambda: bap.cancel()) + + bap.on_finish(lambda bap: bap.events.append('success')) + bap.on_cancel(lambda bap: bap.events.append('canceled')) + + bap.run() + frontend.run() + + assert 'success' not in bap.events + assert 'canceled' in bap.events + assert len(BapIda.instances) == 0 From 6b587243fe97675ae685602b28d9703569c634fa Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Wed, 21 Dec 2016 13:55:24 -0500 Subject: [PATCH 15/20] refactored ida module * made comment handlers extensible * provided a generic interface to IDA service * rewrote type fixing code for psedocode emition * factored out hexrays plugin for stuff that handles hexrays specifics --- plugins/bap/plugins/bap_clear_comments.py | 5 +- plugins/bap/plugins/bap_comments.py | 80 ++++++ plugins/bap/plugins/pseudocode_bap_comment.py | 53 ++-- plugins/bap/plugins/pseudocode_bap_taint.py | 13 +- plugins/bap/utils/_comment_handler.py | 26 ++ plugins/bap/utils/_ctyperewriter.py | 25 ++ plugins/bap/utils/_service.py | 57 ++++ plugins/bap/utils/abstract_ida_plugins.py | 118 -------- plugins/bap/utils/bap_comment.py | 7 + plugins/bap/utils/hexrays.py | 104 +++++++ plugins/bap/utils/ida.py | 257 +++++++----------- plugins/bap/utils/run.py | 4 +- tests/conftest.py | 18 +- tests/mockidaapi.py | 4 +- tests/mockidautils.py | 1 + tests/test_bap_comment.py | 36 ++- tests/test_ida.py | 35 ++- 17 files changed, 498 insertions(+), 345 deletions(-) create mode 100644 plugins/bap/plugins/bap_comments.py create mode 100644 plugins/bap/utils/_comment_handler.py create mode 100644 plugins/bap/utils/_ctyperewriter.py create mode 100644 plugins/bap/utils/_service.py create mode 100644 plugins/bap/utils/hexrays.py create mode 100644 tests/mockidautils.py diff --git a/plugins/bap/plugins/bap_clear_comments.py b/plugins/bap/plugins/bap_clear_comments.py index be3719c..3d90ef3 100644 --- a/plugins/bap/plugins/bap_clear_comments.py +++ b/plugins/bap/plugins/bap_clear_comments.py @@ -1,9 +1,8 @@ import idaapi from idaapi import ASKBTN_YES - from bap.utils import bap_comment -from bap.utils.ida import all_valid_ea +from bap.utils import ida class BapClearComments(idaapi.plugin_t): @@ -20,7 +19,7 @@ def clear_bap_comments(self): "Delete all (BAP ..) comments?") != ASKBTN_YES: return - for ea in all_valid_ea(): + for ea in ida.addresses(): # TODO: store actually commented addresses comm = idaapi.get_cmt(ea, 0) if bap_comment.parse(comm): idaapi.set_cmt(ea, '', 0) diff --git a/plugins/bap/plugins/bap_comments.py b/plugins/bap/plugins/bap_comments.py new file mode 100644 index 0000000..30044a3 --- /dev/null +++ b/plugins/bap/plugins/bap_comments.py @@ -0,0 +1,80 @@ +import idaapi +import idc +from bap.utils import bap_comment, ida + + +class CommentIterator(object): + + def __init__(self, storage): + self.storage = storage + self.index = idc.GetFirstIndex(idc.AR_STR, storage) + + def __iter__(self): + return self + + def next(self): + value = idc.GetArrayElement(idc.AR_STR, self.storage) + if value == 0: + raise StopIteration() + else: + self.index = idc.GetNextIndex(idc.AR_STR, self.storage, self.index) + return value + + +class CommentStorage(object): + + def __init__(self): + name = 'bap-comments' + existing = idc.GetArrayId(name) + if existing < 0: + self.storage = idc.CreateArray(name) + else: + self.storage = existing + + def __iter__(self): + return CommentIterator(self.storage) + + def __len__(self): + n = 0 + for elt in self: + n += 1 + return n + + +class BapComment(idaapi.plugin_t): + flags = idaapi.PLUGIN_HIDE + help = 'propagate comments to IDA Views' + comment = '' + wanted_name = 'BAP: Comment code' + wanted_hotkey = '' + + def init(self): + ida.comment.register_handler(self.update) + return idaapi.PLUGIN_KEEP + + def run(self, arg): + pass + + def term(self): + pass + + def update(self, ea, key, value): + """Add key:value to comm string at EA.""" + cmt = idaapi.get_cmt(ea, 0) + comm = {} + if cmt: + comm = bap_comment.parse(cmt) + if comm is None: + comm = {} + if not value or value == '()': + comm.setdefault(key, []) + else: + if key in comm: + comm[key].append(value) + else: + comm[key] = [value] + idaapi.set_cmt(ea, bap_comment.dumps(comm), 0) + + +def PLUGIN_ENTRY(): + return BapComment() diff --git a/plugins/bap/plugins/pseudocode_bap_comment.py b/plugins/bap/plugins/pseudocode_bap_comment.py index 1b85a68..adc9013 100644 --- a/plugins/bap/plugins/pseudocode_bap_comment.py +++ b/plugins/bap/plugins/pseudocode_bap_comment.py @@ -1,43 +1,32 @@ """Hex-Rays Plugin to propagate comments to Pseudocode View.""" -from bap.utils import abstract_ida_plugins -from bap.utils import bap_comment, sexpr +import idc +import idaapi +from bap.utils import hexrays +from bap.utils import bap_comment -class Pseudocode_BAP_Comment(abstract_ida_plugins.SimpleLine_Modifier_Hexrays): + +COLOR_START = '\x01\x0c // \x01\x0c' +COLOR_END = '\x02\x0c\x02\x0c' + + +class PseudocodeBapComment(hexrays.PseudocodeVisitor): """Propagate comments from Text/Graph view to Pseudocode view.""" flags = idaapi.PLUGIN_HIDE - comment = "BAP Comment on Pseudocode" - help = "BAP Comment on Pseudocode" - wanted_name = "BAP Comment on Pseudocode" - - @classmethod - def _simpleline_modify(cls, cfunc, sl): - sl_dict = {} - - for ea in set(cls.get_ea_list(cfunc, sl)): - ea_comm = GetCommentEx(ea, repeatable=0) - if ea_comm is None: - continue - ea_BAP_dict, _, _ = bap_comment.get_bap_comment(ea_comm) - for e in bap_comment.get_bap_list(ea_BAP_dict): - if isinstance(e, list): - val_list = sl_dict.get(e[0], []) - if len(e) >= 2: # i.e. '(k v)' type - if e[1:] not in val_list: - val_list.append(e[1:]) - sl_dict[e[0]] = val_list - - if len(sl_dict) > 0: - BAP_dict = ['BAP'] - for k, v in sl_dict.items(): - BAP_dict += [[k] + v] - sl.line += '\x01\x0c // \x01\x0c' # start comment coloring - sl.line += sexpr.from_list(BAP_dict) - sl.line += '\x02\x0c\x02\x0c' # stop comment coloring + comment = "" + help = "Propagate BAP comments to pseudocode view" + wanted_name = "BAP: " + def visit_line(self, widget): + for address in widget.extract_addresses(): + comm = idc.Comment(address) + if comm and bap_comment.is_valid(comm): + widget.line += COLOR_START + widget.line += comm + widget.line += COLOR_END def PLUGIN_ENTRY(): """Install Pseudocode_BAP_Comment upon entry.""" - return Pseudocode_BAP_Comment() + return PseudocodeBapComment() diff --git a/plugins/bap/plugins/pseudocode_bap_taint.py b/plugins/bap/plugins/pseudocode_bap_taint.py index 722d82d..bfdecf2 100644 --- a/plugins/bap/plugins/pseudocode_bap_taint.py +++ b/plugins/bap/plugins/pseudocode_bap_taint.py @@ -4,6 +4,12 @@ Requires BAP_Taint plugin, and installs callbacks into it. """ +import idc +import idaapi + +from bap.utils import abstract_ida_plugins, ida, hexrays +from bap.plugins.bap_taint import BapTaint + bap_color = { 'black': 0x000000, 'red': 0xCCCCFF, @@ -16,11 +22,6 @@ 'gray': 0xEAEAEA, } -from bap.utils import abstract_ida_plugins, ida -from bap.plugins.bap_taint import BapTaint - -import idc - class Pseudocode_BAP_Taint(abstract_ida_plugins.SimpleLine_Modifier_Hexrays): """Propagate taint information from Text/Graph view to Pseudocode view.""" @@ -78,7 +79,7 @@ def init(self): def autocolorize_callback(data): ea = data['ea'] - cfunc = ida.cfunc_from_ea(ea) + cfunc = hexrays.find_cfunc(ea) if cfunc is None: return self.run_over_cfunc(cfunc) diff --git a/plugins/bap/utils/_comment_handler.py b/plugins/bap/utils/_comment_handler.py new file mode 100644 index 0000000..7effe7e --- /dev/null +++ b/plugins/bap/utils/_comment_handler.py @@ -0,0 +1,26 @@ + + +class CommentHandlers(object): + def __init__(self): + print('creating comment handlers') + self.handlers = [] + self.comments = {} + + def handler(self): + def wrapped(func): + self.register(func) + return func + return wrapped + + def register_handler(self, func): + for handler in self.handlers: + print(handler.__name__) + self.handlers.append(func) + + def add(self, addr, key, value): + if (addr, key) in self.comments: + self.comments[(addr, key)].append(value) + else: + self.comments[(addr, key)] = [value] + for handler in self.handlers: + handler(addr, key, value) diff --git a/plugins/bap/utils/_ctyperewriter.py b/plugins/bap/utils/_ctyperewriter.py new file mode 100644 index 0000000..14d7496 --- /dev/null +++ b/plugins/bap/utils/_ctyperewriter.py @@ -0,0 +1,25 @@ +import re + +_REWRITERS = ( + (r'(struct|enum|union) ([^{} ]*);', r'\1 \2; typedef \1 \2 \2;'), + (r'unsigned __int(8|16|32|64)', r'uint\1_t'), + (r'(signed )?__int(8|16|32|64)', r'int\2_t'), + (r'__(cdecl|noreturn)', r'__attribute__((\1))'), + ('r^%', r'__'), + (r'_QWORD', r'int64_t'), + (r'_DWORD', r'int32_t'), + (r'_WORD', r'int16_t'), + (r'_BYTE', r'int8_t'), +) + + +class Rewriter(object): + def __init__(self): + self.rewriters = [] + for (patt, subst) in _REWRITERS: + self.rewriters.append((re.compile(patt), subst)) + + def translate(self, expr): + for (regex, subst) in self.rewriters: + expr = regex.subs(subst, expr) + return expr diff --git a/plugins/bap/utils/_service.py b/plugins/bap/utils/_service.py new file mode 100644 index 0000000..4a208cf --- /dev/null +++ b/plugins/bap/utils/_service.py @@ -0,0 +1,57 @@ +import string +import idc + + +class Service(object): + def __init__(self): + self.services = {} + + def provider(self, name): + def wrapped(func): + self.register(name, func) + return func + return wrapped + + def register(self, name, func): + if name in self.services: + raise ServiceAlreadyRegistered(name) + if not is_valid_service_name(name): + raise ServiceNameIsNotValid(name) + self.services[name] = func + + def request(self, service, output): + if service not in self.services: + raise ServiceIsNotRegistered(service) + + idc.Wait() + with open(output) as out: + self.services[service](out) + idc.Exit(0) + + +class ServiceError(Exception): + pass + + +class ServiceRegistrationError(ServiceError): + pass + + +class ServiceAlreadyRegistered(ServiceRegistrationError): + def __init__(self, name): + self.name = name + + +class ServiceNameIsNotValid(ServiceRegistrationError): + def __init__(self, name): + self.name = name + + +class ServiceIsNotRegistered(ServiceError): + def __init__(self, name): + self.name = name + + +def is_valid_service_name(name): + valid_syms = string.ascii_letters + '-_' + string.digits + return set(name).issubset(valid_syms) diff --git a/plugins/bap/utils/abstract_ida_plugins.py b/plugins/bap/utils/abstract_ida_plugins.py index 072594d..df92216 100644 --- a/plugins/bap/utils/abstract_ida_plugins.py +++ b/plugins/bap/utils/abstract_ida_plugins.py @@ -36,121 +36,3 @@ def term(self): def run(self, arg): """Do nothing.""" pass - - -class SimpleLine_Modifier_Hexrays(idaapi.plugin_t): - """ - Abstract Base Plugin Class to modify simplelines in Pseudocode View. - - This plugin should be subclassed and the following members should be - implemented before being usable: - - simpleline_modify(cls, cfunc, sl) - - Accepts a simpleline_t (sl) and modifies it - - cfunc and class cls are provided for use if necessary - - comment - - string to describe your plugin - - help - - string for any help information - - wanted_name - - string for what you want your plugin to be named - - Methods that might be useful while implementing above methods: - - get_ea_list(cls, cfunc, sl) - - Note: You will need to add a PLUGIN_ENTRY() function, to your plugin code, - that returns an object of your plugin, which uses this Class as a super - class. - """ - - @classmethod - def _simpleline_modify(cls, cfunc, sl): - raise NotImplementedError("Please implement this method") - - @classmethod - def get_ea_list(cls, cfunc, sl): - """Get a list of EAs that are in a simpleline_t.""" - def ea_from_addr_tag(addr_tag): - return cfunc.treeitems.at(addr_tag).ea - - def is_addr_code(s): - return (s[0] == idaapi.COLOR_ON and - s[1] == chr(idaapi.COLOR_ADDR)) - - anchor = idaapi.ctree_anchor_t() - line = sl.line[:] # Copy - ea_list = [] - - while len(line) > 0: - skipcode_index = idaapi.tag_skipcode(line) - if skipcode_index == 0: # No code found - line = line[1:] # Skip one character ahead - else: - if is_addr_code(line): - addr_tag = int(line[2:skipcode_index], 16) - anchor.value = addr_tag - if ( - anchor.is_citem_anchor() and - not anchor.is_blkcmt_anchor() - ): - line_ea = ea_from_addr_tag(addr_tag) - if line_ea != idaapi.BADADDR: - ea_list.append(line_ea) - line = line[skipcode_index:] # Skip the colorcodes - - return ea_list - - @classmethod - def run_over_cfunc(cls, cfunc): - """Run the plugin over the given cfunc.""" - simplelinevec = cfunc.get_pseudocode() - - for simpleline in simplelinevec: - cls._simpleline_modify(cfunc, simpleline) - - flags = idaapi.PLUGIN_PROC - wanted_hotkey = "" - - def init(self): - """ - Ensure plugin's line modification function is called whenever needed. - - If Hex-Rays is not installed, or is not initialized yet, then plugin - will not load. To ensure that the plugin loads after Hex-Rays, please - name your plugin's .py file with a name that starts lexicographically - after "hexx86f" - """ - try: - if idaapi.init_hexrays_plugin(): - def hexrays_event_callback(event, *args): - if event == idaapi.hxe_refresh_pseudocode: - # We use this event instead of hxe_text_ready because - # MacOSX doesn't seem to work well with it - # TODO: Look into this - vu, = args - self.run_over_cfunc(vu.cfunc) - return 0 - - idaapi.install_hexrays_callback(hexrays_event_callback) - - else: - return idaapi.PLUGIN_SKIP - - except AttributeError: - print "init_hexrays_plugin() not found. Skipping Hex-Rays plugin." - - return idaapi.PLUGIN_KEEP - - def term(self): - """Terminate Plugin.""" - try: - idaapi.term_hexrays_plugin() - except AttributeError: - pass - - def run(self, arg): - """ - Run Plugin. - - Ignored since callbacks are installed. - """ - pass diff --git a/plugins/bap/utils/bap_comment.py b/plugins/bap/utils/bap_comment.py index d9fb131..c667869 100644 --- a/plugins/bap/utils/bap_comment.py +++ b/plugins/bap/utils/bap_comment.py @@ -155,6 +155,13 @@ def push(result, key, values): return result +def is_valid(comm): + try: + return comm.startswith('BAP:') and parse(comm) + except SyntaxError: + return False + + def dumps(comm): """ Dump dictionary into a comment string. diff --git a/plugins/bap/utils/hexrays.py b/plugins/bap/utils/hexrays.py new file mode 100644 index 0000000..e369ef5 --- /dev/null +++ b/plugins/bap/utils/hexrays.py @@ -0,0 +1,104 @@ +from __future__ import print_function + +from copy import copy + +import idaapi +import idc + + +def tag_addrcode(s): + return (s[0] == idaapi.COLOR_ON and + s[1] == chr(idaapi.COLOR_ADDR)) + + +class PseudocodeLineWidget(object): + + def __init__(self, parent, widget): + self.parent = parent + self.widget = widget + + def extract_addresses(self): + '''A set of addresses associated with the line''' + anchor = idaapi.ctree_anchor_t() + line = copy(self.widget.line) + addresses = set() + + while len(line) > 0: + skipcode_index = idaapi.tag_skipcode(line) + if skipcode_index == 0: # No code found + line = line[1:] # Skip one character ahead + else: + if tag_addrcode(line): + addr_tag = int(line[2:skipcode_index], 16) + anchor.value = addr_tag + if anchor.is_citem_anchor() \ + and not anchor.is_blkcmt_anchor(): + address = self.parent.treeitems.at(addr_tag).ea + if address != idaapi.BADADDR: + addresses.append(address) + line = line[skipcode_index:] # Skip the colorcodes + return addresses + + +class PseudocodeVisitor(idaapi.plugin_t): + """ + Abstract Base Plugin Class to modify simplelines in Pseudocode View. + + Methods that might be useful while implementing above methods: + - get_ea_list(self, cfunc, sl) + + Note: You will need to add a PLUGIN_ENTRY() function, to your plugin code, + that returns an object of your plugin, which uses this Class as a super + class. + """ + + flags = idaapi.PLUGIN_PROC + wanted_hotkey = "" + + def visit_line(self, line): + pass + + def visit_func(self, func): + """Run the plugin over the given cfunc.""" + for line in func.get_pseudocode(): + self.visit_line(PseudocodeLineWidget(func, line)) + + def init(self): + """ + Ensure plugin's line modification function is called whenever needed. + + If Hex-Rays is not installed, or is not initialized yet, then plugin + will not load. To ensure that the plugin loads after Hex-Rays, please + name your plugin's .py file with a name that starts lexicographically + after "hexx86f" + """ + try: + if idaapi.init_hexrays_plugin(): + def hexrays_event_callback(event, *args): + if event == idaapi.hxe_refresh_pseudocode: + # We use this event instead of hxe_text_ready because + # MacOSX doesn't seem to work well with it + # TODO: Look into this + vu, = args + self.visit_func(vu.cfunc) + return 0 + idaapi.install_hexrays_callback(hexrays_event_callback) + else: + return idaapi.PLUGIN_SKIP + except AttributeError: + idc.Warning('''init_hexrays_plugin() not found. + Skipping Hex-Rays plugin.''') + return idaapi.PLUGIN_KEEP + + def term(self): + pass + + def run(self, arg): + pass + + +def find_cfunc(ea): + """Get cfuncptr_t from EA.""" + func = idaapi.get_func(ea) + if func: + return idaapi.decompile(func) diff --git a/plugins/bap/utils/ida.py b/plugins/bap/utils/ida.py index 5c45d01..6480dae 100644 --- a/plugins/bap/utils/ida.py +++ b/plugins/bap/utils/ida.py @@ -1,75 +1,45 @@ """Utilities that interact with IDA.""" import idaapi import idc -from bap.utils import bap_comment - - -def add_to_comment(ea, key, value): - """Add key:value to comm string at EA.""" - cmt = idaapi.get_cmt(ea, 0) - comm = {} - if cmt: - comm = bap_comment.parse(cmt) - if comm is None: - comm = {} - if value == '()': - comm.setdefault(key, []) - else: - if key in comm: - comm[key].append(value) - else: - comm[key] = [value] - idaapi.set_cmt(ea, bap_comment.dumps(comm), 0) - - -def cfunc_from_ea(ea): - """Get cfuncptr_t from EA.""" - func = idaapi.get_func(ea) - if func is None: - return None - cfunc = idaapi.decompile(func) - return cfunc - - -def all_valid_ea(): - """Return all valid EA as a Python generator.""" - from idautils import Segments - from idc import SegStart, SegEnd - for s in Segments(): - ea = SegStart(s) - while ea < SegEnd(s): - yield ea - ea = idaapi.nextaddr(ea) +import idautils +from ._service import Service +from ._comment_handler import CommentHandlers +from ._ctyperewriter import Rewriter -def dump_loader_info(output_filename): - """Dump information for BAP's loader into output_filename.""" - from idautils import Segments - import idc - idc.Wait() +service = Service() +comment = CommentHandlers() +rewriter = Rewriter() - with open(output_filename, 'w+') as out: - info = idaapi.get_inf_structure() - size = "r32" if info.is_32bit else "r64" - out.write("(%s %s (" % (info.get_proc_name()[1], size)) - for seg in Segments(): - out.write("\n(%s %s %d (0x%X %d))" % ( - idaapi.get_segm_name(seg), - "code" if idaapi.segtype(seg) == idaapi.SEG_CODE else "data", - idaapi.get_fileregion_offset(seg), - seg, idaapi.getseg(seg).size())) - out.write("))\n") +def addresses(): + """Generate all mapped addresses.""" + for s in idc.Segments(): + ea = idc.SegStart(s) + while ea < idc.SegEnd(s): + yield ea + ea = idaapi.nextaddr(ea) -def dump_symbol_info(out): - """Dump information for BAP's symbolizer into the out file object.""" - from idautils import Segments, Functions - from idc import ( - SegStart, SegEnd, GetFunctionAttr, - FUNCATTR_START, FUNCATTR_END - ) +@service.provider('loader') +def output_segments(out): + """Dump binary segmentation.""" + info = idaapi.get_inf_structure() + size = "r32" if info.is_32bit else "r64" + out.write('(', info.get_proc_name()[1], ' ', size, ' (') + for seg in idautils.Segments(): + out.write("\n({} {} {:d} ({:#x} {d}))".format( + idaapi.get_segm_name(seg), + "code" if idaapi.segtype(seg) == idaapi.SEG_CODE else "data", + idaapi.get_fileregion_offset(seg), + seg, idaapi.getseg(seg).size())) + out.write("))\n") + + +@service.provider('symbols') +def output_symbols(out): + """Dump symbols.""" try: from idaapi import get_func_name2 as get_func_name # Since get_func_name is deprecated (at least from IDA 6.9) @@ -97,96 +67,79 @@ def func_name_propagate_thunk(ea): # Fallback to non-propagated name for weird times that IDA gives # a 0 length name, or finds a longer import name - idaapi.autoWait() - - for ea in Segments(): - fs = Functions(SegStart(ea), SegEnd(ea)) + for ea in idc.Segments(): + fs = idc.Functions(idc.SegStart(ea), idc.SegEnd(ea)) for f in fs: out.write('("%s" 0x%x 0x%x)\n' % ( func_name_propagate_thunk(f), - GetFunctionAttr(f, FUNCATTR_START), - GetFunctionAttr(f, FUNCATTR_END))) - - -def dump_c_header(out): - """Dump type information as a C header.""" - def local_type_info(): - class my_sink(idaapi.text_sink_t): - def __init__(self): - try: - idaapi.text_sink_t.__init__(self) - except AttributeError: - pass # Older IDA versions keep the text_sink_t abstract - self.text = [] - - def _print(self, thing): - self.text.append(thing) - return 0 - - sink = my_sink() - - idaapi.print_decls(sink, idaapi.cvar.idati, [], - idaapi.PDF_INCL_DEPS | idaapi.PDF_DEF_FWD) - return sink.text - - def function_sigs(): - import idautils - f_types = [] - for ea in idautils.Functions(): - ft = idaapi.print_type(ea, True) - if ft is not None: - f_types.append(ft + ';') - return list(set(f_types)) # Set, since sometimes, IDA gives repeats - - def replacer(regex, replacement): - import re - r = re.compile(regex) - return lambda s: r.sub(replacement, s) - - pp_decls = replacer(r'(struct|enum|union) ([^{} ]*);', - r'\1 \2; typedef \1 \2 \2;') - pp_unsigned = replacer(r'unsigned __int(8|16|32|64)', - r'uint\1_t') - pp_signed = replacer(r'(signed )?__int(8|16|32|64)', - r'int\2_t') - pp_annotations = replacer(r'__(cdecl|noreturn)', r'__attribute__((\1))') - pp_wd = lambda s: ( - replacer(r'_QWORD', r'int64_t')( - replacer(r'_DWORD', r'int32_t')( - replacer(r'_WORD', r'int16_t')( - replacer(r'_BYTE', r'int8_t')(s))))) - - def preprocess(line): - line = pp_decls(line) - line = pp_unsigned(line) # Must happen before signed - line = pp_signed(line) - line = pp_annotations(line) - line = pp_wd(line) - return line - - for line in local_type_info() + function_sigs(): - line = preprocess(line) - out.write(line + '\n') - - -def dump_brancher_info(output_filename): - """Dump information for BAP's brancher into output_filename.""" - from idautils import CodeRefsFrom - - idc.Wait() - - def dest(ea, flow): # flow denotes whether normal flow is also taken - return set(CodeRefsFrom(ea, flow)) - - def pp(l): - return ' '.join('0x%x' % e for e in l) - - with open(output_filename, 'w+') as out: - for ea in all_valid_ea(): - branch_dests = dest(ea, False) - if len(branch_dests) > 0: - out.write('(0x%x (%s) (%s))\n' % ( - ea, - pp(dest(ea, True) - branch_dests), - pp(branch_dests) - )) + idc.GetFunctionAttr(f, idc.FUNCATTR_START), + idc.GetFunctionAttr(f, idc.FUNCATTR_END))) + + +@service.provider('types') +def output_types(out): + """Dump type information.""" + for line in local_types() + prototypes(): + out.write(rewriter.translate(line) + '\n') + + +@service.provider('brancher') +def output_branches(out): + """Dump static successors for each instruction """ + for addr in addresses(): + succs = Succs(addr) + if succs.jmps: + out.write('{}\n'.format(succs.dumps)) + + +class Printer(idaapi.text_sink_t): + def __init__(self): + try: + idaapi.text_sink_t.__init__(self) + except AttributeError: + pass # Older IDA versions keep the text_sink_t abstract + self.lines = [] + + def _print(self, thing): + self.lines.append(thing) + return 0 + + +def local_types(): + printer = Printer() + idaapi.print_decls(printer, idaapi.cvar.idati, [], + idaapi.PDF_INCL_DEPS | idaapi.PDF_DEF_FWD) + return printer.lines + + +def prototypes(): + types = set() + for ea in idautils.Functions(): + proto = idaapi.print_type(ea, True) + if proto: + types.append(proto + ';') + return list(types) + + +def Succs(object): + def __init__(self, addr): + self.addr = addr + self.dests = set(idautils.CodeRefsFrom(addr, True)) + self.jmps = set(idautils.CodeRefsFrom(addr, False)) + falls = self.succs - self.dests + self.fall = falls[0] if falls else None + + def dumps(self): + return ''.join([ + '({:#x} '.format(self.addr), + sexps(self.dests), + ' {:#x})'.format(self.fall) if self.fall else ' )' + ]) + + +def sexps(addrs): + sexp = ['('] + for addr in addrs: + sexp.append('{:#x}'.format(addr)) + sexp.append(')') + return ' '.join(sexp) diff --git a/plugins/bap/utils/run.py b/plugins/bap/utils/run.py index ba4591d..ca980a7 100644 --- a/plugins/bap/utils/run.py +++ b/plugins/bap/utils/run.py @@ -187,7 +187,7 @@ def run(self): def _setup_symbols(self): "pass symbol information from IDA to BAP" with self.tmpfile("sym") as out: - ida.dump_symbol_info(out) + ida.output_symbols(out) self.args += [ "--read-symbols-from", out.name, "--symbolizer=file", @@ -202,7 +202,7 @@ def _setup_headers(self, bap): # Will leave it as it is until issue #588 is # resolved in the upstream with self.tmpfile("h") as out: - ida.dump_c_header(out) + ida.output_types(out) subprocess.call(bap, [ '--api-add', 'c:"{0}"'.format(out.name), ]) diff --git a/tests/conftest.py b/tests/conftest.py index 59044a4..fc64f38 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,13 @@ import sys import subprocess -from time import sleep, time as current_time +from time import sleep import pytest sys.modules['idaapi'] = __import__('mockidaapi') sys.modules['idc'] = __import__('mockidc') +sys.modules['idautils'] = __import__('mockidautils') @pytest.fixture @@ -20,7 +21,7 @@ def patch(attrs, ns='idaapi'): @pytest.fixture def addresses(monkeypatch): addresses = (0xDEADBEAF, 0xDEADBEEF) - monkeypatch.setattr('bap.utils.ida.all_valid_ea', lambda: addresses) + monkeypatch.setattr('bap.utils.ida.addresses', lambda: addresses) return addresses @@ -34,7 +35,16 @@ def set_cmt(ea, val, off): 'get_cmt': lambda ea, off: cmts.get(ea), 'set_cmt': set_cmt }) - return cmts + yield cmts + + +@pytest.fixture(scope='session') +def load(): + def load(module): + plugin = module.PLUGIN_ENTRY() + plugin.init() + return plugin + return load @pytest.fixture(params=['yes', 'no', 'cancel']) @@ -256,5 +266,5 @@ def run_bap(args): 'SetStatus': ida.set_status, }) popenpatch(run_bap) - monkeypatch.setattr('bap.utils.ida.dump_symbol_info', lambda out: None) + monkeypatch.setattr('bap.utils.ida.output_symbols', lambda out: None) return (bap, ida) diff --git a/tests/mockidaapi.py b/tests/mockidaapi.py index f2929bb..f8066f2 100644 --- a/tests/mockidaapi.py +++ b/tests/mockidaapi.py @@ -4,8 +4,10 @@ ASKBTN_NO = NotImplemented ASKBTN_CANCEL = NotImplemented PLUGIN_DRAW = NotImplemented +PLUGIN_HIDE = NotImplemented PLUGIN_KEEP = NotImplemented -class plugin_t(): NotImplemented +class plugin_t(object): NotImplemented +class text_sink_t(object): NotImplemented def idadir(sub): NotImplemented def get_cmt(ea, off): NotImplemented def set_cmt(ea, off): NotImplemented diff --git a/tests/mockidautils.py b/tests/mockidautils.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/mockidautils.py @@ -0,0 +1 @@ + diff --git a/tests/test_bap_comment.py b/tests/test_bap_comment.py index 80252c1..cc31ef3 100644 --- a/tests/test_bap_comment.py +++ b/tests/test_bap_comment.py @@ -1,22 +1,30 @@ -from bap.utils.bap_comment import parse, dumps +from bap.utils.bap_comment import parse, dumps, is_valid def test_parse(): - assert(parse('hello') is None) - assert(parse('BAP: hello') == {'hello': []}) - assert(parse('BAP: hello,world') == {'hello': [], 'world': []}) - assert(parse('BAP: hello=cruel,world') == {'hello': ['cruel', 'world']}) - assert(parse('BAP: hello="hello, world"') == {'hello': ['hello, world']}) - assert(parse('BAP: hello=cruel,world goodbye=real,life') == - {'hello': ['cruel', 'world'], 'goodbye': ['real', 'life']}) - assert(parse('BAP: hello="f\'"') == {'hello': ["f'"]}) + assert parse('hello') is None + assert parse('BAP: hello') == {'hello': []} + assert parse('BAP: hello,world') == {'hello': [], 'world': []} + assert parse('BAP: hello=cruel,world') == {'hello': ['cruel', 'world']} + assert parse('BAP: hello="hello, world"') == {'hello': ['hello, world']} + assert parse('BAP: hello=cruel,world goodbye=real,life') == { + 'hello': ['cruel', 'world'], + 'goodbye': ['real', 'life'] + } + assert parse('BAP: hello="f\'"') == {'hello': ["f'"]} def test_dumps(): - assert('BAP:' in dumps({'hello': []})) - assert(dumps({'hello': ['cruel', 'world'], 'nice': [], 'thing': []}) == - 'BAP: nice,thing hello=cruel,world') - assert(dumps({'hello': ["world\'"]}) == 'BAP: hello="world\'"') + assert 'BAP:' in dumps({'hello': []}) + assert dumps({'hello': ['cruel', 'world'], 'nice': [], 'thing': []}) == \ + 'BAP: nice,thing hello=cruel,world' + assert dumps({'hello': ["world\'"]}) == 'BAP: hello="world\'"' + + +def test_is_valid(): + assert is_valid('BAP: hello') + assert is_valid('BAP: hello,world') + assert not is_valid('some comment') def test_roundup(): @@ -27,4 +35,4 @@ def test_roundup(): 'c': ['many things'], 'd': ['strange \\ things'], } - assert(parse(dumps(parse(dumps(comm)))) == comm) + assert parse(dumps(parse(dumps(comm)))) == comm diff --git a/tests/test_ida.py b/tests/test_ida.py index 8681aec..e055ba6 100644 --- a/tests/test_ida.py +++ b/tests/test_ida.py @@ -1,16 +1,25 @@ -def test_comments(addresses, comments, choice): - from bap.utils.ida import add_to_comment - from bap.plugins.bap_clear_comments import PLUGIN_ENTRY - for key in addresses: - add_to_comment(key, 'foo', 'bar') - assert comments[key] == 'BAP: foo=bar' - add_to_comment(key, 'foo', 'baz') - assert comments[key] == 'BAP: foo=bar,baz' - add_to_comment(key, 'bar', '()') - assert comments[key] == 'BAP: bar foo=bar,baz' - plugin = PLUGIN_ENTRY() - plugin.init() - plugin.run(0) +def test_comments(addresses, comments, choice, load): + from bap.utils import ida + from bap.plugins import bap_clear_comments + from bap.plugins import bap_comments + + ida.comment.handlers = [] + ida.comment.comments.clear() + + load(bap_comments) + clear = load(bap_clear_comments) + + assert len(ida.comment.handlers) == 1 + + for addr in addresses: + ida.comment.add(addr, 'foo', 'bar') + assert comments[addr] == 'BAP: foo=bar' + ida.comment.add(addr, 'foo', 'baz') + assert comments[addr] == 'BAP: foo=bar,baz' + ida.comment.add(addr, 'bar', '()') + assert comments[addr] == 'BAP: bar foo=bar,baz' + + clear.run(0) bap_cmts = [c for c in comments.values() if 'BAP:' in c] expected = { 'yes': 0, From 5daa792632e785ad3499fd51a5048ea05a605498 Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Wed, 21 Dec 2016 17:20:03 -0500 Subject: [PATCH 16/20] multiple fixes so far so good, work with IDA, but still have issues. --- plugins/bap/plugins/bap_bir_attr.py | 40 ++++---- plugins/bap/plugins/bap_comments.py | 16 +-- plugins/bap/plugins/pseudocode_bap_comment.py | 25 +++-- plugins/bap/plugins/pseudocode_bap_taint.py | 99 ++++++------------- plugins/bap/utils/_comment_handler.py | 1 - plugins/bap/utils/hexrays.py | 2 +- plugins/bap/utils/ida.py | 14 ++- plugins/bap/utils/run.py | 79 +++++++-------- 8 files changed, 122 insertions(+), 154 deletions(-) diff --git a/plugins/bap/plugins/bap_bir_attr.py b/plugins/bap/plugins/bap_bir_attr.py index 98d05f8..e6abb46 100644 --- a/plugins/bap/plugins/bap_bir_attr.py +++ b/plugins/bap/plugins/bap_bir_attr.py @@ -13,21 +13,24 @@ Comments in the Text/Graph Views are added using a key-value storage with the format (BAP (k1 v1) (k2 v2) ...) """ -import idautils +import idaapi +import idc + from bap.utils.run import BapIda + class BapScripter(BapIda): def __init__(self, user_args, attrs): - super(BapScripter,self).__init__() - if attrs != []: + super(BapScripter, self).__init__() + if attrs: self.action = 'extracting ' + ','.join(attrs) else: self.action = 'running bap ' + user_args self.script = self.tmpfile('py') self.args += user_args.split(' ') - self.args.append('--emit-ida-script') self.args += [ + '--emit-ida-script', '--emit-ida-script-file', self.script.name ] self.args += [ @@ -37,10 +40,11 @@ def __init__(self, user_args, attrs): # perfectly random numbers -ARGS_HISTORY=324312 -ATTR_HISTORY=234345 +ARGS_HISTORY = 324312 +ATTR_HISTORY = 234345 + -class BAP_BIR_Attr(idaapi.plugin_t): +class BapBirAttr(idaapi.plugin_t): """ Plugin to get BIR attributes from arbitrary BAP executions. @@ -56,8 +60,8 @@ class BAP_BIR_Attr(idaapi.plugin_t): recipes = {} - def _do_callbacks(self,ea): - for callback in BAP_BIR_Attr._callbacks: + def _do_callbacks(self, ea): + for callback in self._callbacks: callback({'ea': ea}) def run(self, arg): @@ -68,27 +72,28 @@ def run(self, arg): address at the location pointed to by the cursor. """ - args_msg = "Arguments that will be passed to `bap'" + args_msg = "Arguments that will be passed to `bap'" args = idaapi.askstr(ARGS_HISTORY, '--passes=', args_msg) if args is None: return - attr_msg = "A comma separated list of attributes,\n" + attr_msg = "A comma separated list of attributes,\n" attr_msg += "that should be propagated to comments" - attr_def = BAP_BIR_Attr.recipes.get(args, '') + attr_def = self.recipes.get(args, '') attr = idaapi.askstr(ATTR_HISTORY, attr_def, attr_msg) if attr is None: return # store a choice of attributes for the given set of arguments - BAP_BIR_Attr.recipes[args] = attr + # TODO: store recipes in IDA's database + self.recipes[args] = attr ea = idc.ScreenEA() attrs = [] if attr != '': attrs = attr.split(',') analysis = BapScripter(args, attrs) - analysis.on_finish(lambda bap: self.load_script(bap,ea)) + analysis.on_finish(lambda bap: self.load_script(bap, ea)) analysis.run() def load_script(self, bap, ea): @@ -100,8 +105,6 @@ def load_script(self, bap, ea): idaapi.refresh_idaview_anyway() idc.SetStatus(idc.IDA_STATUS_READY) - - def init(self): """Initialize Plugin.""" return idaapi.PLUGIN_KEEP @@ -110,7 +113,6 @@ def term(self): """Terminate Plugin.""" pass - @classmethod def install_callback(cls, callback_fn): """ @@ -122,9 +124,9 @@ def install_callback(cls, callback_fn): 'ea': The value of EA at point where user propagated taint from. """ idc.Message('a callback is installed\n') - cls._callbacks[ptr_or_reg].append(callback_fn) + cls._callbacks.append(callback_fn) def PLUGIN_ENTRY(): """Install BAP_BIR_Attr upon entry.""" - return BAP_BIR_Attr() + return BapBirAttr() diff --git a/plugins/bap/plugins/bap_comments.py b/plugins/bap/plugins/bap_comments.py index 30044a3..76f0e57 100644 --- a/plugins/bap/plugins/bap_comments.py +++ b/plugins/bap/plugins/bap_comments.py @@ -61,18 +61,10 @@ def term(self): def update(self, ea, key, value): """Add key:value to comm string at EA.""" cmt = idaapi.get_cmt(ea, 0) - comm = {} - if cmt: - comm = bap_comment.parse(cmt) - if comm is None: - comm = {} - if not value or value == '()': - comm.setdefault(key, []) - else: - if key in comm: - comm[key].append(value) - else: - comm[key] = [value] + comm = cmt and bap_comment.parse(cmt) or {} + values = comm.setdefault(key, []) + if value and value != '()' and value not in values: + values.append(value) idaapi.set_cmt(ea, bap_comment.dumps(comm), 0) diff --git a/plugins/bap/plugins/pseudocode_bap_comment.py b/plugins/bap/plugins/pseudocode_bap_comment.py index adc9013..eb87df5 100644 --- a/plugins/bap/plugins/pseudocode_bap_comment.py +++ b/plugins/bap/plugins/pseudocode_bap_comment.py @@ -11,6 +11,14 @@ COLOR_END = '\x02\x0c\x02\x0c' +def union(lhs, rhs): + for (key, rvalues) in rhs.items(): + lvalues = lhs.setdefault(key, []) + for value in rvalues: + if value not in lvalues: + lvalues.append(value) + + class PseudocodeBapComment(hexrays.PseudocodeVisitor): """Propagate comments from Text/Graph view to Pseudocode view.""" flags = idaapi.PLUGIN_HIDE @@ -18,13 +26,16 @@ class PseudocodeBapComment(hexrays.PseudocodeVisitor): help = "Propagate BAP comments to pseudocode view" wanted_name = "BAP: " - def visit_line(self, widget): - for address in widget.extract_addresses(): - comm = idc.Comment(address) - if comm and bap_comment.is_valid(comm): - widget.line += COLOR_START - widget.line += comm - widget.line += COLOR_END + def visit_line(self, line): + comm = {} + for address in line.extract_addresses(): + idacomm = idc.Comment(address) + newcomm = idacomm and bap_comment.parse(idacomm) or {} + union(comm, newcomm) + if comm: + line.widget.line += COLOR_START + line.widget.line += bap_comment.dumps(comm) + line.widget.line += COLOR_END def PLUGIN_ENTRY(): diff --git a/plugins/bap/plugins/pseudocode_bap_taint.py b/plugins/bap/plugins/pseudocode_bap_taint.py index bfdecf2..8cc63c3 100644 --- a/plugins/bap/plugins/pseudocode_bap_taint.py +++ b/plugins/bap/plugins/pseudocode_bap_taint.py @@ -7,10 +7,10 @@ import idc import idaapi -from bap.utils import abstract_ida_plugins, ida, hexrays +from bap.utils import hexrays from bap.plugins.bap_taint import BapTaint -bap_color = { +colors = { 'black': 0x000000, 'red': 0xCCCCFF, 'green': 0x99FF99, @@ -23,79 +23,42 @@ } -class Pseudocode_BAP_Taint(abstract_ida_plugins.SimpleLine_Modifier_Hexrays): +def next_color(current_color, ea): + coloring_order = [ + colors[c] for c in [ + 'gray', + 'white', + 'red', + 'yellow', + ] + ] + BGR_MASK = 0xffffff + ea_color = idaapi.get_item_color(ea) + if ea_color & BGR_MASK not in coloring_order: + return current_color + assert(current_color & BGR_MASK in coloring_order) + ea_idx = coloring_order.index(ea_color & BGR_MASK) + current_idx = coloring_order.index(current_color & BGR_MASK) + if ea_idx >= current_idx: + return ea_color + else: + return current_color + + +class PseudocodeBapTaint(hexrays.PseudocodeVisitor): """Propagate taint information from Text/Graph view to Pseudocode view.""" - flags=idaapi.PLUGIN_HIDE + flags = idaapi.PLUGIN_HIDE comment = "BAP Taint Plugin for Pseudocode View" help = "BAP Taint Plugin for Pseudocode View" wanted_name = "BAP Taint Pseudocode" - @classmethod - def _simpleline_modify(cls, cfunc, sl): - cls._color_line(sl, bap_color['gray']) - # Ready to be painted over by other colors - for ea in cls.get_ea_list(cfunc, sl): - new_color = cls._get_new_color(sl.bgcolor, ea) - cls._color_line(sl, new_color) - - @staticmethod - def _color_line(sl, color): - sl.bgcolor = color - - @staticmethod - def _get_new_color(current_color, ea): - coloring_order = [ - bap_color[c] for c in [ - 'gray', - 'white', - 'red', - 'yellow', - ] - ] - - BGR_MASK = 0xffffff - - ea_color = idaapi.get_item_color(ea) - - if ea_color & BGR_MASK not in coloring_order: - # Since BAP didn't color it, we can't infer anything - return current_color - - assert(current_color & BGR_MASK in coloring_order) - - ea_idx = coloring_order.index(ea_color & BGR_MASK) - current_idx = coloring_order.index(current_color & BGR_MASK) - - if ea_idx >= current_idx: - return ea_color - else: - return current_color - - def init(self): - """Initialize Plugin.""" - try: - if idaapi.init_hexrays_plugin(): - - def autocolorize_callback(data): - ea = data['ea'] - cfunc = hexrays.find_cfunc(ea) - if cfunc is None: - return - self.run_over_cfunc(cfunc) - - # idaapi.load_plugin('bap_taint') - BapTaint.install_callback(autocolorize_callback) - else: - return idaapi.PLUGIN_SKIP - - except AttributeError: - idc.Warning("init_hexrays_plugin() not found. Skipping Hex-Rays plugin.") - - return abstract_ida_plugins.SimpleLine_Modifier_Hexrays.init(self) - # Call superclass init() + def visit_line(self, line): + line.widget.bgcolor = colors['gray'] + for addr in line.extract_addresses(): + line.widget.bgcolor = next_color(line.widget.bgcolor, addr) def PLUGIN_ENTRY(): """Install Pseudocode_BAP_Taint upon entry.""" - return Pseudocode_BAP_Taint() + return PseudocodeBapTaint() diff --git a/plugins/bap/utils/_comment_handler.py b/plugins/bap/utils/_comment_handler.py index 7effe7e..c0e1075 100644 --- a/plugins/bap/utils/_comment_handler.py +++ b/plugins/bap/utils/_comment_handler.py @@ -2,7 +2,6 @@ class CommentHandlers(object): def __init__(self): - print('creating comment handlers') self.handlers = [] self.comments = {} diff --git a/plugins/bap/utils/hexrays.py b/plugins/bap/utils/hexrays.py index e369ef5..70a1b46 100644 --- a/plugins/bap/utils/hexrays.py +++ b/plugins/bap/utils/hexrays.py @@ -35,7 +35,7 @@ def extract_addresses(self): and not anchor.is_blkcmt_anchor(): address = self.parent.treeitems.at(addr_tag).ea if address != idaapi.BADADDR: - addresses.append(address) + addresses.add(address) line = line[skipcode_index:] # Skip the colorcodes return addresses diff --git a/plugins/bap/utils/ida.py b/plugins/bap/utils/ida.py index 6480dae..482eed8 100644 --- a/plugins/bap/utils/ida.py +++ b/plugins/bap/utils/ida.py @@ -15,9 +15,9 @@ def addresses(): """Generate all mapped addresses.""" - for s in idc.Segments(): - ea = idc.SegStart(s) - while ea < idc.SegEnd(s): + for s in idautils.Segments(): + ea = idautils.SegStart(s) + while ea < idautils.SegEnd(s): yield ea ea = idaapi.nextaddr(ea) @@ -67,8 +67,8 @@ def func_name_propagate_thunk(ea): # Fallback to non-propagated name for weird times that IDA gives # a 0 length name, or finds a longer import name - for ea in idc.Segments(): - fs = idc.Functions(idc.SegStart(ea), idc.SegEnd(ea)) + for ea in idautils.Segments(): + fs = idautils.Functions(idc.SegStart(ea), idc.SegEnd(ea)) for f in fs: out.write('("%s" 0x%x 0x%x)\n' % ( func_name_propagate_thunk(f), @@ -92,6 +92,10 @@ def output_branches(out): out.write('{}\n'.format(succs.dumps)) +def set_color(addr, color): + idc.SetColor(addr, idc.CIC_ITEM, color) + + class Printer(idaapi.text_sink_t): def __init__(self): try: diff --git a/plugins/bap/utils/run.py b/plugins/bap/utils/run.py index ca980a7..e93b1fe 100644 --- a/plugins/bap/utils/run.py +++ b/plugins/bap/utils/run.py @@ -104,7 +104,6 @@ def tmpfile(self, suffix, *args, **kwargs): "creates a new temporary files in the self.tmpdir" if self.tmpdir is None: self.tmpdir = tempfile.mkdtemp(prefix="bap") - tmp = tempfile.NamedTemporaryFile( delete=False, prefix='bap-ida', @@ -175,12 +174,10 @@ def run(self): "Previous instances of BAP didn't finish yet.\ Do you really want to start a new one?". format(len(BapIda.instances))) - if answer == idaapi.ASKBTN_YES: self._do_run() else: self._do_run() - idc.Message("BAP> total number of running instances: {0}\n". format(len(BapIda.instances))) @@ -312,63 +309,59 @@ def __str__(self): return 'Unable to detect bap executable ' -BAP_FINDERS = [] +class BapFinder(object): + def __init__(self): + self.finders = [] + def register(self, func): + self.finders.append(func) -def check_and_configure_bap(): - """ - Check if bap_executable_path is set in the config; ask user if necessary. - - Automagically also tries a bunch of strategies to find `bap` if it can, - and uses this to populate the default path in the popup, to make the - user's life easier. :) + def finder(self, func): + self.register(func) + return func - Also, this specifically enables the BAP API option in the config if it is - unspecified. - """ - if config.get('bap_executable_path') is not None: - return - - bap_path = '' - - for find in BAP_FINDERS: - path = find() - if path: - bap_path = path + def find(self): + path = None + for find in self.finders: + path = find() break + return path - # always ask a user to confirm the path that was found using heuristics - bap_path = ask_user(bap_path) - if bap_path and len(bap_path) > 0: - config.set('bap_executable_path', bap_path) +bap = BapFinder() -def preadline(cmd): - try: - res = subprocess.check_output(cmd, universal_newlines=True) - return res.strip() - except (OSError, subprocess.CalledProcessError): - return None +def check_and_configure_bap(): + """Ensures that bap location is known.""" + if not config.get('bap_executable_path'): + path = ask_user(bap.find()) + if path and len(path) > 0: + config.set('bap_executable_path', path) -def system_path(): +@bap.finder +def system(): return preadline(['which', 'bap']) -def opam_path(): +@bap.finder +def opam(): try: cmd = ['opam', 'config', 'var', 'bap:bin'] res = preadline(cmd).strip() - return os.path.join(res, 'bap') + if 'undefined' not in res: + return os.path.join(res, 'bap') + else: + return None except: return None -def ask_user(default_path): - def confirm(msg): - return idaapi.askyn_c(idaapi.ASKBTN_YES, msg) == idaapi.ASKBTN_YES +def confirm(msg): + return idaapi.askyn_c(idaapi.ASKBTN_YES, msg) == idaapi.ASKBTN_YES + +def ask_user(default_path): while True: bap_path = idaapi.askfile_c(False, default_path, 'Path to bap') if bap_path is None: @@ -385,5 +378,9 @@ def confirm(msg): return bap_path -BAP_FINDERS.append(system_path) -BAP_FINDERS.append(opam_path) +def preadline(cmd): + try: + res = subprocess.check_output(cmd, universal_newlines=True) + return res.strip() + except (OSError, subprocess.CalledProcessError): + return None From db35dc8afb5d6a8574e458239a37e1ebd0b9f187 Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Thu, 22 Dec 2016 13:20:09 -0500 Subject: [PATCH 17/20] made integration more robust: 1. comments are now better escaped, although I'm still getting occasional exceptions with the quotes. 2. If bap fails with error, the result is now displayed correctly. --- plugins/bap/plugins/bap_bir_attr.py | 2 +- plugins/bap/plugins/bap_clear_comments.py | 4 ++-- plugins/bap/plugins/bap_comments.py | 2 +- plugins/bap/plugins/bap_view.py | 26 ++++++++++++----------- plugins/bap/utils/bap_comment.py | 11 +++++++--- plugins/bap/utils/ida.py | 4 ++-- plugins/bap/utils/run.py | 20 +++++++---------- tests/test_bap_comment.py | 6 ++++++ 8 files changed, 42 insertions(+), 33 deletions(-) diff --git a/plugins/bap/plugins/bap_bir_attr.py b/plugins/bap/plugins/bap_bir_attr.py index e6abb46..bef2bc7 100644 --- a/plugins/bap/plugins/bap_bir_attr.py +++ b/plugins/bap/plugins/bap_bir_attr.py @@ -50,7 +50,7 @@ class BapBirAttr(idaapi.plugin_t): Also supports installation of callbacks using install_callback() """ - flags = 0 + flags = idaapi.PLUGIN_DRAW comment = "Run BAP " help = "Runs BAP and extracts data from the output" wanted_name = "BAP: Run" diff --git a/plugins/bap/plugins/bap_clear_comments.py b/plugins/bap/plugins/bap_clear_comments.py index 3d90ef3..630e83c 100644 --- a/plugins/bap/plugins/bap_clear_comments.py +++ b/plugins/bap/plugins/bap_clear_comments.py @@ -16,12 +16,12 @@ def clear_bap_comments(self): """Ask user for confirmation and then clear (BAP ..) comments.""" if idaapi.askyn_c(ASKBTN_YES, - "Delete all (BAP ..) comments?") != ASKBTN_YES: + "Delete all `BAP: ..` comments?") != ASKBTN_YES: return for ea in ida.addresses(): # TODO: store actually commented addresses comm = idaapi.get_cmt(ea, 0) - if bap_comment.parse(comm): + if comm and comm.startswith('BAP:'): idaapi.set_cmt(ea, '', 0) def init(self): diff --git a/plugins/bap/plugins/bap_comments.py b/plugins/bap/plugins/bap_comments.py index 76f0e57..37c94d9 100644 --- a/plugins/bap/plugins/bap_comments.py +++ b/plugins/bap/plugins/bap_comments.py @@ -59,7 +59,7 @@ def term(self): pass def update(self, ea, key, value): - """Add key:value to comm string at EA.""" + """Add key=values to comm string at EA.""" cmt = idaapi.get_cmt(ea, 0) comm = cmt and bap_comment.parse(cmt) or {} values = comm.setdefault(key, []) diff --git a/plugins/bap/plugins/bap_view.py b/plugins/bap/plugins/bap_view.py index a3c46c3..4eeaa5b 100644 --- a/plugins/bap/plugins/bap_view.py +++ b/plugins/bap/plugins/bap_view.py @@ -4,10 +4,11 @@ from bap.utils.run import BapIda import re -import idaapi # pylint: disable=import-error +import idaapi # pylint: disable=import-error + class BapViews(idaapi.Choose2): - #pylint: disable=invalid-name,missing-docstring,no-self-use + # pylint: disable=invalid-name,missing-docstring,no-self-use def __init__(self, views): idaapi.Choose2.__init__(self, 'Choose BAP view', [ ['PID', 4], @@ -31,9 +32,10 @@ def OnGetLine(self, n): def OnGetSize(self): return len(self.views) + class View(idaapi.simplecustviewer_t): - #pylint: disable=invalid-name,missing-docstring,no-self-use - #pylint: disable=super-on-old-class,no-member + # pylint: disable=invalid-name,missing-docstring,no-self-use + # pylint: disable=super-on-old-class,no-member def __init__(self, caption, instance, on_close=None): super(View, self).__init__() self.Create(caption) @@ -69,7 +71,6 @@ class BapView(idaapi.plugin_t): def __init__(self): self.views = {} - def create_view(self, bap): "creates a new view" pid = bap.proc.pid @@ -78,7 +79,7 @@ def create_view(self, bap): view.instance = bap curr = idaapi.get_current_tform() self.views[pid] = view - view.Show() #pylint: disable=no-member + view.Show() # pylint: disable=no-member idaapi.switchto_tform(curr, True) def delete_view(self, pid): @@ -94,15 +95,16 @@ def update_view(self, bap): def finished(self, bap): "final update" self.update_view(bap) - view = self.views[bap.proc.pid] - if bap.proc.returncode > 0: - view.Show() #pylint: disable=no-member + if bap.proc.pid in self.views: # because a user could close the view + if bap.proc.returncode > 0: + self.views[bap.proc.pid].Show() # pylint: disable=no-member def init(self): """Initialize BAP view to load whenever hotkey is pressed.""" BapIda.observers['instance_created'].append(self.create_view) BapIda.observers['instance_updated'].append(self.update_view) BapIda.observers['instance_finished'].append(self.finished) + BapIda.observers['instance_failed'].append(self.finished) return idaapi.PLUGIN_KEEP def term(self): @@ -113,16 +115,16 @@ def term(self): def show_view(self): "Switch to one of the BAP views" chooser = BapViews(self.views) - choice = chooser.Show(modal=True) #pylint: disable=no-member + choice = chooser.Show(modal=True) # pylint: disable=no-member if choice >= 0: view = self.views[self.views.keys()[choice]] view.Show() - - def run(self, arg): #pylint: disable=unused-argument + def run(self, arg): # pylint: disable=unused-argument "invokes the plugin" self.show_view() + def recolorize(line): """fix ansi colors""" ansi_escape = re.compile(r'\x1b[^m]*m([^\x1b]*)\x1b[^m]*m') diff --git a/plugins/bap/utils/bap_comment.py b/plugins/bap/utils/bap_comment.py index c667869..e6207b7 100644 --- a/plugins/bap/utils/bap_comment.py +++ b/plugins/bap/utils/bap_comment.py @@ -163,8 +163,7 @@ def is_valid(comm): def dumps(comm): - """ - Dump dictionary into a comment string. + """Dump dictionary into a comment string. The representation is parseable with the parse function. """ @@ -195,7 +194,13 @@ def quote(token): '"hello, world"' """ if set(token) - set(WORDCHARS): - return '"{0}"'.format(token) + if "'" not in token: + return "'{}'".format(token) + elif '"' not in token: + return '"{}"'.format(token) + else: # we ran out of quotes, so we need + return "'{}'".format(''.join('\\'+c if c == "'" else c + for c in token)) else: return token diff --git a/plugins/bap/utils/ida.py b/plugins/bap/utils/ida.py index 482eed8..3f6267b 100644 --- a/plugins/bap/utils/ida.py +++ b/plugins/bap/utils/ida.py @@ -16,8 +16,8 @@ def addresses(): """Generate all mapped addresses.""" for s in idautils.Segments(): - ea = idautils.SegStart(s) - while ea < idautils.SegEnd(s): + ea = idc.SegStart(s) + while ea < idc.SegEnd(s): yield ea ea = idaapi.nextaddr(ea) diff --git a/plugins/bap/utils/run.py b/plugins/bap/utils/run.py index e93b1fe..c9b9947 100644 --- a/plugins/bap/utils/run.py +++ b/plugins/bap/utils/run.py @@ -49,13 +49,16 @@ def __init__(self, bap, input_file): - `fds` a list of opened filedescriptors to be closed on the exit """ - self.tmpdir = None + self.tmpdir = tempfile.mkdtemp(prefix="bap") self.args = [bap, input_file] self.proc = None self.fds = [] self.out = self.tmpfile("out") self.action = "running bap" self.closed = False + self.env = {'BAP_LOG_DIR': self.tmpdir} + if self.DEBUG: + self.env['BAP_DEBUG'] = 'yes' def run(self): "starts BAP process" @@ -65,13 +68,11 @@ def run(self): self.args, stdout=self.out, stderr=subprocess.STDOUT, - env={ - 'BAP_LOG_DIR': self.tmpdir - }) + env=self.env) def finished(self): "true if the process is no longer running" - return self.proc is not None and self.proc.poll() is not None + return self.proc and self.proc.poll() is not None def close(self): "terminate the process if needed and cleanup" @@ -254,6 +255,8 @@ def close(self): BapIda.instances.remove(self) def update(self): + if all(bap.finished() for bap in BapIda.instances): + idc.SetStatus(idc.IDA_STATUS_READY) if self.finished(): if self.proc.returncode == 0: self.run_handlers('instance_finished') @@ -272,13 +275,6 @@ def update(self): return -1 else: self.run_handlers('instance_updated') - thinking = False - for bap in BapIda.instances: - if bap.finished(): - idc.SetStatus(idc.IDA_STATUS_THINKING) - thinking = True - if not thinking: - idc.SetStatus(idc.IDA_STATUS_READY) return self.poll_interval_ms def cancel(self): diff --git a/tests/test_bap_comment.py b/tests/test_bap_comment.py index cc31ef3..e100119 100644 --- a/tests/test_bap_comment.py +++ b/tests/test_bap_comment.py @@ -36,3 +36,9 @@ def test_roundup(): 'd': ['strange \\ things'], } assert parse(dumps(parse(dumps(comm)))) == comm + + +def test_quotation(): + data = 'BAP: chars=\'{"a", "b", "c"}\'' + assert parse(data) == {'chars': ['{"a", "b", "c"}']} + assert parse(data) == parse(dumps(parse(data))) From 67f5df5594c9acd40ec5131c002a450b3674a406 Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Thu, 22 Dec 2016 14:15:11 -0500 Subject: [PATCH 18/20] we are not using sexp for storing comments anymore --- plugins/bap/utils/sexpr.py | 92 -------------------------------------- 1 file changed, 92 deletions(-) delete mode 100644 plugins/bap/utils/sexpr.py diff --git a/plugins/bap/utils/sexpr.py b/plugins/bap/utils/sexpr.py deleted file mode 100644 index e1d3b4d..0000000 --- a/plugins/bap/utils/sexpr.py +++ /dev/null @@ -1,92 +0,0 @@ -"""S-Expression utilities.""" - - -def to_list(s): - """Convert S-Expression to List.""" - assert(is_valid(s)) - sexp = [[]] - word = '' - in_str = False - for c in s: - if c == '(' and not in_str: - sexp.append([]) - elif c == ')' and not in_str: - if word: - sexp[-1].append(word) - word = '' - temp = sexp.pop() - sexp[-1].append(temp) - elif c in (' ', '\n', '\t') and not in_str: - if word: - sexp[-1].append(word) - word = '' - elif c == '\"': - in_str = not in_str - else: - word += c - if word: # Final word, if it remains - sexp[-1].append(word) - return sexp[0] - - -def from_list(l): - """Convert List to S-Expression.""" - if isinstance(l, str): - for special_char in (' ', '\n', '\t', '(', ')', '\"'): - if special_char in l: - return '\"' + l + '\"' - return l - return '(' + ' '.join(from_list(e) for e in l) + ')' - - -def is_valid(s): - """Return True if s is a valid S-Expression.""" - in_str = False - escaped = False - bb = 0 - for c in s: - if not escaped and in_str and c == '\\': - escaped = True - else: - if not escaped and c == '\"': - in_str = not in_str - elif c == '(' and not in_str: - bb += 1 - elif c == ')' and not in_str: - bb -= 1 - if bb < 0: - return False - escaped=False - return bb == 0 - - -def truncate(s): - """Truncate s to a valid S-Expression.""" - in_str = False - bb = 0 - for i, c in enumerate(s): - if c == '(' and not in_str: - bb += 1 - elif c == ')' and not in_str: - bb -= 1 - if bb == 0: - return s[:i+1] - elif c == '\"': - in_str = not in_str - raise ValueError('Insufficient close brackets in ' + repr(s)) - - -def complete(s): - """Add enough close brackets to s to make it a valid S-Expression.""" - in_str = False - bb = 0 - for i, c in enumerate(s): - if c == '(' and not in_str: - bb += 1 - elif c == ')' and not in_str: - bb -= 1 - if bb < 0: - raise ValueError('Insufficient open brackets in ' + repr(s)) - elif c == '\"': - in_str = not in_str - return s + ')' * bb From 3382cd243f62742d85a77dddbfb8361188125c75 Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Thu, 22 Dec 2016 15:08:44 -0500 Subject: [PATCH 19/20] added a view that shows all BAP Attributes --- plugins/bap/plugins/bap_comments.py | 75 ++++++++++++++-------------- plugins/bap/plugins/bap_functions.py | 15 ++---- plugins/bap/plugins/bap_view.py | 3 +- 3 files changed, 43 insertions(+), 50 deletions(-) diff --git a/plugins/bap/plugins/bap_comments.py b/plugins/bap/plugins/bap_comments.py index 37c94d9..b78e572 100644 --- a/plugins/bap/plugins/bap_comments.py +++ b/plugins/bap/plugins/bap_comments.py @@ -1,59 +1,60 @@ import idaapi import idc -from bap.utils import bap_comment, ida - - -class CommentIterator(object): - def __init__(self, storage): - self.storage = storage - self.index = idc.GetFirstIndex(idc.AR_STR, storage) - - def __iter__(self): - return self - - def next(self): - value = idc.GetArrayElement(idc.AR_STR, self.storage) - if value == 0: - raise StopIteration() - else: - self.index = idc.GetNextIndex(idc.AR_STR, self.storage, self.index) - return value +from bap.utils import bap_comment, ida -class CommentStorage(object): +class Attributes(idaapi.Choose2): + def __init__(self, comms): + idaapi.Choose2.__init__(self, 'Select an attribute', [ + ['name', 8], + ['addr', 8], + ['data', 64] + ]) + self.comms = [ + [name, '{:#x}'.format(addr), ' '.join(data)] + for (name, addr, data) in comms + ] - def __init__(self): - name = 'bap-comments' - existing = idc.GetArrayId(name) - if existing < 0: - self.storage = idc.CreateArray(name) - else: - self.storage = existing + def OnClose(self): + pass - def __iter__(self): - return CommentIterator(self.storage) + def OnGetSize(self): + return len(self.comms) - def __len__(self): - n = 0 - for elt in self: - n += 1 - return n + def OnGetLine(self, n): + return self.comms[n] class BapComment(idaapi.plugin_t): - flags = idaapi.PLUGIN_HIDE + flags = 0 help = 'propagate comments to IDA Views' comment = '' - wanted_name = 'BAP: Comment code' - wanted_hotkey = '' + wanted_name = 'BAP: View BAP Attributes' + wanted_hotkey = 'Shift-B' + + def __init__(self): + self.comms = {} def init(self): ida.comment.register_handler(self.update) return idaapi.PLUGIN_KEEP def run(self, arg): - pass + comms = {} + for addr in ida.addresses(): + comm = idaapi.get_cmt(addr, 0) + if comm: + parsed = bap_comment.parse(comm) + if parsed: + for (name, data) in parsed.items(): + comms[(addr, name)] = data + comms = [(name, addr, data) + for ((addr, name), data) in comms.items()] + attrs = Attributes(comms) + choice = attrs.Show(modal=True) + if choice >= 0: + idc.Jump(comms[choice][1]) def term(self): pass diff --git a/plugins/bap/plugins/bap_functions.py b/plugins/bap/plugins/bap_functions.py index f25c45e..ae512c3 100644 --- a/plugins/bap/plugins/bap_functions.py +++ b/plugins/bap/plugins/bap_functions.py @@ -7,18 +7,15 @@ Keybindings: Shift-P : Run BAP and mark code as functions in IDA """ -import idautils import idaapi import idc -from bap.utils import config - from bap.utils.run import BapIda + class FunctionFinder(BapIda): def __init__(self): - idc.Message('in the FunctionFinder constructor\n') - super(FunctionFinder,self).__init__() + super(FunctionFinder, self).__init__() self.action = 'looking for function starts' self.syms = self.tmpfile('syms', mode='r') self.args += [ @@ -26,6 +23,7 @@ def __init__(self): '--dump', 'symbols:{0}'.format(self.syms.name) ] + class BAP_Functions(idaapi.plugin_t): """Plugin to get functions from BAP and mark them in IDA.""" @@ -35,18 +33,13 @@ class BAP_Functions(idaapi.plugin_t): wanted_name = "BAP: Discover functions" wanted_hotkey = "Shift-P" - def mark_functions(self): """Run BAP, get functions, and mark them in IDA.""" - idc.Message('creating BAP instance\n') analysis = FunctionFinder() - idc.Message('installing handler\n') analysis.on_finish(lambda x: self.add_starts(x)) - idc.Message('running the analysis\n') analysis.run() def add_starts(self, bap): - idc.Message('Analysis has finished\n') idaapi.refresh_idaview_anyway() for line in bap.syms: line = line.strip() @@ -54,9 +47,7 @@ def add_starts(self, bap): continue addr = int(line, 16) end_addr = idaapi.BADADDR - idc.Message('Adding function at {0}\n'.format(addr)) idaapi.add_func(addr, end_addr) - idc.Refresh() def init(self): diff --git a/plugins/bap/plugins/bap_view.py b/plugins/bap/plugins/bap_view.py index 4eeaa5b..385acd6 100644 --- a/plugins/bap/plugins/bap_view.py +++ b/plugins/bap/plugins/bap_view.py @@ -130,6 +130,7 @@ def recolorize(line): ansi_escape = re.compile(r'\x1b[^m]*m([^\x1b]*)\x1b[^m]*m') return ansi_escape.sub('\1\x22\\1\2\x22', line) -def PLUGIN_ENTRY(): #pylint: disable=invalid-name + +def PLUGIN_ENTRY(): # pylint: disable=invalid-name """Install BAP_View upon entry.""" return BapView() From 343e42637e41c18b241a4a074d87337cc1dc1177 Mon Sep 17 00:00:00 2001 From: Ivan Gotovchits Date: Thu, 22 Dec 2016 16:07:30 -0500 Subject: [PATCH 20/20] version 0.2.0 This is major update to the bap-ida-python package, it brings lots of new features, here is the excerpt from the CHANGES.md: * call BAP asynchronously (without blocking IDA) * run several instances of BAP in parallel * special attribute view (instead of `Alt-T` search) * neater comment syntax (attr=value instead of sexp) * task manager for primitive job control * plugins are now callable from the menu (try `Ctrl-3`) * each instance has its own view * view selector can switch between views * stderr and stdout are properly dumped into the view * cross-platform implementation (Docker, Windows should work) * more robust type emition * new generic ida service integration (for calls to IDA from BAP) * added unit tests * Travis-CI integration * code refactoring: more pythonic, PEP8 compilant, pylint-happy The most neat features are: 1. Run multiple instances of BAP without blocking IDA 2. Lookup extracted attributes with the new attribute view 3. Run plugins from the menu (no need to memorize all these shortcuts, just use `Ctrl-3` to see them all) 4. More readable and robust comments (though still with issues) From the software engineering perspective, the codebase was heavily rewritten. The code is now more pythonic (subjective of course), PEP8 compilant, (some modules are even good to pylint), and, most importantly, we now have tests. A big effort was spent on mocking the IDA, and lots of bugs were fixed during the process. The coverage is still very low, though. This version also brings a new generic interface for the services, that are provide by IDA to BAP (rooter, brancher, etc). As well as exposing a new interface for the emit-ida-script plugin. These changes are breaking, so bap.1.0.0 will not work with bap-ida-python 0.2.0. The new interface exposes a singleton instance `ida.service` that will accept the service name, and the destination file. --- CHANGES.md | 21 ++++++++ README.md | 98 ++++++++++++++++++++++++++++++----- plugins/bap/utils/_service.py | 2 +- plugins/bap/utils/ida.py | 4 +- tests/mockidaapi.py | 1 + 5 files changed, 110 insertions(+), 16 deletions(-) create mode 100644 CHANGES.md diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..f4d1317 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,21 @@ +0.2.0 +----- +* call BAP asynchronously (without blocking IDA) +* run several instances of BAP in parallel +* special attribute view (instead of `Alt-T` search) +* neater comment syntax (attr=value instead of sexp) +* task manager for primitive job control +* plugins are now callable from the menu (try `Ctrl-3`) +* each instance has its own view +* view selector can switch between views +* stderr and stdout are properly dumped into the view +* cross-platform implementation (Docker, Windows should work) +* more robust type emition +* new generic ida service integration (for calls to IDA from BAP) +* added unit tests +* Travis-CI integration +* code refactoring: more pythonic, PEP8 compilant, pylint-happy + +0.1.0 +----- +* initial release diff --git a/README.md b/README.md index 56f2a31..d340c92 100644 --- a/README.md +++ b/README.md @@ -2,18 +2,35 @@ BAP IDA Python ============== This package provides the necessary IDAPython scripts required for -interoperatibility between BAP and IDA Pro. It also provides many useful feature additions to IDA, by leveraging power from BAP. +interoperatibility between BAP and IDA Pro. It also provides many +useful feature additions to IDA, by leveraging power from BAP. Features -------- +BAP-IDA integration package installs several plugins into IDA +distribution. Some plugins works automatically, and do not require +user intervention, while others are invoked with keybindings, or via +the `Edit->Plugins` menu, that can be popped at with the `Ctrl-3` +bindging. + + ### Function information augmentation -By just hitting the `Shift+P` key, IDA will call BAP which will use its own analysis (and all the information sources that it knows of) to obtain all the locations where there are functions. This information is then propagated to IDA and used to create functions there automatically. This is especially useful in scenarios where there are a lot of indirect calls etc and BAP (using its different plugins) is able to detect functions in the code which IDA is unable to do so. +By just hitting the `Shift+P` key, IDA will call BAP which will use +its own analysis (and all the information sources that it knows of) to +obtain all the locations where there are functions. This information +is then propagated to IDA and used to create functions there +automatically. This is especially useful in scenarios where there are +a lot of indirect calls etc and BAP (using its different plugins) is +able to detect functions in the code which IDA is unable to do so. ### Taint Propagation -By choosing a taint source and hitting either `Ctrl+A` (for tainting register) or `Ctrl+Shift+A` (for tainting pointer), one can easily see how taint propagates through the code, in both disassembly and decompilation views. +By choosing a taint source and hitting either `Ctrl+A` (for tainting +an immediate value) or `Ctrl+Shift+A` (for data pointed by a value), +one can easily see how taint propagates through the code, in both +disassembly and decompilation views. #### In Text/Graph View ![taint](docs/taint.png) @@ -23,11 +40,22 @@ By choosing a taint source and hitting either `Ctrl+A` (for tainting register) o ### BIR Attribute Tagging, with arbitrary BAP plugins -BAP has the ability to tag a lot of possible attributes to instructions. These BIR attributes can be tagged automatically as comments in IDA, by running arbitrary plugins in BAP. Just hit `Ctrl+S`. +BAP has the ability to tag a lot of possible attributes to +instructions. These BIR attributes can be tagged automatically as +comments in IDA, by running arbitrary plugins in BAP. Just hit +`Ctrl+S`. + +Here's an example of output for Saluki showing that a certain malloc +is unchecked (pointing to a potential vulnerability). -Here's an example of output for Saluki showing that a certain malloc is unchecked (pointing to a potential vulnerability). +Clearing all BAP comments (without affecting your own personal +comments in IDA) can be done by pressing `Ctrl+Shift+S`. -Clearing all BAP comments (without affecting your own personal comments in IDA) can be done by pressing `Ctrl+Shift+S`. +To view all current attributes in the single window hit `Shift-B`. +You can sort attributes by clicking the columns or you can search +through them using IDA's extensive search facilities (hit the `Help` +bottom on the list to get more information). You can jump directly +to the attribute location by selecting it. #### In Text/Graph View ![bir-attr-saluki](docs/bir-attr-saluki.png) @@ -35,31 +63,75 @@ Clearing all BAP comments (without affecting your own personal comments in IDA) #### In Pseudocode View ![bir-attr-saluki-decompiler](docs/bir-attr-saluki-decompiler.png) -### BAP View +### BAP Task Manager and Viewer -Sometimes, you just wish to see the BAP output of the command you just ran to generate BIR attributes (or for the taints), and you can do this in IDA by hitting `Ctrl+Alt+Shift+S` to see the command the BAP ran, along with its output. Do note that this also shows bir output from bap. +Every instance of BAP will have a corresponding view, that will +accumulate all data written by BAP. The BAP Viewer (`Ctrl-Alt-F5`) +provides an easy way to switch between multiple BAP Views. + +Since you can run multiple instances of BAP asynchronously, it is +useful to have an ability to view the state of currently running +processes, and, even, to terminate those who take too much time or +memory. The BAP Task Manager (accessible via the `Ctrl-Alt-Shift-F5` +keybinding, or via the `Ctrl-3` plugin menu) provides such +functionality. ![bap-view](docs/bap-view.png) ### Symbol and Type Information -Whenever possible, `bap-ida-python` passes along the latest symbol and type information from IDA (including changes you might have made manually), so as to aid better and more accurate analysis in BAP. For example, let's say you recognize that a function is a malloc in a stripped binary, by just using IDA's rename feature (Keybinding: `N`), you can inform BAP of this change during the next run of, say, saluki, without needing to do anything extra. It works automagically! +Whenever possible, `bap-ida-python` passes along the latest symbol and +type information from IDA (including changes you might have made +manually), so as to aid better and more accurate analysis in BAP. For +example, let's say you recognize that a function is a malloc in a +stripped binary, by just using IDA's rename feature (Keybinding: `N`), +you can inform BAP of this change during the next run of, say, saluki, +without needing to do anything extra. It works automagically! Installation ------------ -Copy all of the files and directories from the `plugins` directory into `$IDADIR/plugins`. +Copy all of the files and directories from the `plugins` directory +into `$IDADIR/plugins`. -The first run of IDA after that will prompt you to provide the path to BAP (along with a default if IDA is able to automatically detect BAP). If you wish to edit the path to BAP manually later, you can edit the file `$IDADIR/cfg/bap.cfg`. +The first run of IDA after that will prompt you to provide the path to +BAP (along with a default if IDA is able to automatically detect +BAP). If you wish to edit the path to BAP manually later, you can edit +the file `$IDADIR/cfg/bap.cfg`. #### Opam? -It is usually much easier to install through opam if you have already followed all the installation steps in the [bap repository](https://github.com/BinaryAnalysisPlatform/bap). Just run: +It is usually much easier to install through opam if you have already +followed all the installation steps in the +[bap repository](https://github.com/BinaryAnalysisPlatform/bap). Just +run: ``` opam install bap-ida-python ``` +Debugging +--------- + +The integration package is still in alpha stage, so there are a few +bugs lurking in the codebase. If you have any issues, then, please, +enable the debug mode, by typing the following command in the IDA's +python console: + +```python +BapIda.DEBUG=True +``` + +This will increase the verbosity level, so that you can see what commands +were actually issued to the bap backend. In the debug mode, the temporary +files will not be removed, so they can be archived and sent to us, for the +ease of debugging. + + #### IDA Demo? -You can also use parts of the functionality (i.e. most of everything except for the decompiler outputs, and batch processing from bap) with IDA Free/Demo. However, you would need to install IDAPython. See [here](docs/IDAPython_on_IDADemo.md) for what one of our users reported to work. \ No newline at end of file +You can also use parts of the functionality (i.e. most of everything +except for the decompiler outputs, and batch processing from bap) with +IDA Free/Demo. However, you would need to install IDAPython. See +[here](docs/IDAPython_on_IDADemo.md) for what one of our users +reported to work. diff --git a/plugins/bap/utils/_service.py b/plugins/bap/utils/_service.py index 4a208cf..2035f7e 100644 --- a/plugins/bap/utils/_service.py +++ b/plugins/bap/utils/_service.py @@ -24,7 +24,7 @@ def request(self, service, output): raise ServiceIsNotRegistered(service) idc.Wait() - with open(output) as out: + with open(output, 'w') as out: self.services[service](out) idc.Exit(0) diff --git a/plugins/bap/utils/ida.py b/plugins/bap/utils/ida.py index 3f6267b..f1dbc94 100644 --- a/plugins/bap/utils/ida.py +++ b/plugins/bap/utils/ida.py @@ -136,8 +136,8 @@ def __init__(self, addr): def dumps(self): return ''.join([ '({:#x} '.format(self.addr), - sexps(self.dests), - ' {:#x})'.format(self.fall) if self.fall else ' )' + ' ({:#x}) '.format(self.fall) if self.fall else '()', + '{})'.format(sexps(self.dests)) ]) diff --git a/tests/mockidaapi.py b/tests/mockidaapi.py index f8066f2..2492e18 100644 --- a/tests/mockidaapi.py +++ b/tests/mockidaapi.py @@ -8,6 +8,7 @@ PLUGIN_KEEP = NotImplemented class plugin_t(object): NotImplemented class text_sink_t(object): NotImplemented +class Choose2(object): NotImplemented def idadir(sub): NotImplemented def get_cmt(ea, off): NotImplemented def set_cmt(ea, off): NotImplemented