diff --git a/util/runtest.py b/util/runtest.py index 2543f2e986..c3a1f1c19a 100644 --- a/util/runtest.py +++ b/util/runtest.py @@ -32,6 +32,7 @@ import md5 import json import yaml +import xml.etree.ElementTree as ET # # Platform detection @@ -160,6 +161,8 @@ def blue(text): return ansi_color(text, '0;37;44') def yellow(text): return ansi_color(text, '0;33;40') +def grey(text): + return ansi_color(text, '6;37;40') # Parse lines. Split a text input into lines using LF as the line separator. # Assume last line is terminated with a LF and ignore an "empty line" that @@ -200,6 +203,72 @@ def clipline(x): def remove_cr(data): return data.replace('\r', '') +# +# Valgrind result processing +# + +def parse_massif_result(f, res): + # Allocated bytes. + re_heap_b = re.compile(r'^mem_heap_B=(\d+)$') + + # Allocation overhead. Matters for desktop environments, for efficient + # zero overhead pool allocators this is not usually a concern (the waste + # in a pool allocator behaves very differently than a libc allocator). + re_heap_extra_b = re.compile(r'^mem_heap_extra_B=(\d+)$') + + # Stacks. + re_stacks_b = re.compile(r'^mem_stacks_B=(\d+)$') + + peak_heap = 0 + peak_stack = 0 + + for line in f: + line = line.strip() + #print(line) + m1 = re_heap_b.match(line) + m2 = re_heap_extra_b.match(line) + m3 = re_stacks_b.match(line) + + heap = None + if m1 is not None: + heap = int(m1.group(1)) + stack = None + if m3 is not None: + stack = int(m3.group(1)) + + if heap is not None: + peak_heap = max(peak_heap, heap) + if stack is not None: + peak_stack = max(peak_stack, stack) + + res['massif_peak_heap_bytes'] = peak_heap + res['massif_peak_stack_bytes'] = peak_stack + +def parse_memcheck_result(f, res): + try: + tree = ET.parse(f) + except ET.ParseError: + res['errors'].append('memcheck-parse-failed') + return + root = tree.getroot() + if root.tag != 'valgrindoutput': + raise Exception('invalid valgrind xml format') + + def parse_error(node): + err = {} + for child in node.findall('kind'): + err['kind'] = child.text + for child in node.findall('xwhat'): + for child2 in child.findall('text'): + err['text'] = child2.text + + res['errors'].append(err['kind']) + # XXX: make res['errors'] structured rather than text list? + # 'err' is now ignored. + + for child in root.findall('error'): + parse_error(child) + # # Test execution and result interpretation helpers. # @@ -384,13 +453,13 @@ def parse_known_issue(data): def find_known_issues(): for dirname in [ os.path.join(os.path.dirname(testcase_filename), '..', 'knownissues'), - os.path.join(os.path.dirname(script_path), '..', 'tests', 'knownissues'), - os.path.join(os.path.dirname(entry_cwd), 'tests', 'knownissues') + os.path.join(script_path, '..', 'tests', 'knownissues'), + os.path.join(entry_cwd, 'tests', 'knownissues') ]: #print('Find known issues, try: %r' % dirname) if os.path.isdir(dirname): return dirname - raise Exception('failed to locate testcases') + raise Exception('failed to locate known issues') # Check known issues against output data. def check_known_issues(dirname, output): @@ -423,20 +492,35 @@ def execute_ecmascript_testcase(res, data, name): test_fn = os.path.abspath(os.path.join(tempdir, name)) write_file(test_fn, data) + valgrind_output = None + cmd = [] try: start_time = time.time() try: if opts.valgrind: res['valgrind'] = True - cmd += [ 'valgrind', path_to_platform(opts.duk) ] + res['valgrind_tool'] = opts.valgrind_tool + cmd += [ 'valgrind' ] + cmd += [ '--tool=' + opts.valgrind_tool ] + + valgrind_output = os.path.abspath(os.path.join(tempdir, 'valgrind.out')) + if opts.valgrind_tool == 'massif': + cmd += [ '--massif-out-file=' + path_to_platform(valgrind_output) ] + #cmd += [ '--peak-inaccuracy=0.0' ] + #cmd += [ '--stacks=yes' ] + elif opts.valgrind_tool == 'memcheck': + cmd += [ '--xml=yes', '--xml-file=' + path_to_platform(valgrind_output) ] + else: + raise Exception('invalid valgrind tool %r' % opts.valgrind_tool) + cmd += [ path_to_platform(os.path.abspath(opts.duk)) ] else: - cmd += [ opts.duk ] - cmd += [ path_to_platform(test_fn) ] + cmd += [ os.path.abspath(opts.duk) ] + cmd += [ path_to_platform(os.path.abspath(test_fn)) ] res['command'] = cmd #print('Executing: %r' % cmd) - proc = subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc = subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.path.abspath(tempdir)) timeout_sec = opts.timeout def kill_proc(p): @@ -458,6 +542,16 @@ def kill_proc(p): if opts.valgrind: res['valgrind_output'] = ret[1] res['stderr'] = '' # otherwise interpreted as an error + if valgrind_output is not None and os.path.exists(valgrind_output): + with open(valgrind_output, 'rb') as f: + res['valgrind_output'] += f.read() + with open(valgrind_output, 'rb') as f: + if opts.valgrind_tool == 'massif': + parse_massif_result(f, res) + elif opts.valgrind_tool == 'memcheck': + parse_memcheck_result(f, res) + else: + res['errors'].append('no-valgrind-output') except: exc_type, exc_value, exc_traceback = sys.exc_info() print('Command execution failed for %r:\n%s' % (cmd, traceback.format_exc(exc_traceback))) @@ -477,7 +571,7 @@ def execute_api_testcase(data): def interpret_test_result(doc, expect): meta = doc['metadata'] - errors = [] + errors = doc['errors'] known_meta = check_known_issues(opts.known_issues, doc['stdout']) @@ -506,6 +600,8 @@ def interpret_test_result(doc, expect): success = False doc['knownissue'] = known_meta.get('summary', 'no summary') doc['knownissue_meta'] = known_meta + if len(errors) > 0: + success = False doc['success'] = success doc['errors'] = errors @@ -519,41 +615,70 @@ def interpret_test_result(doc, expect): def print_summary(doc): meta = doc['metadata'] - parts = [] - test_time = '[%.1f sec]' % float(doc['duration']) + def fmt_time(x): + if x >= 60: + return '%.1f m' % (float(x) / 60.0) + else: + return '%.1f s' % float(x) + + def fmt_size(x): + if x < 1024 * 1024: + return '%.2f k' % (float(x) / 1024.0) + else: + return '%.2f M' % (float(x) / (1024.0 * 1024.0)) + parts = [] + issues = [] + test_result = '???' + test_name = doc['testcase_name'].ljust(50) print_diff = True # print diff if it is nonzero + test_time = fmt_time(doc['duration']) + test_time = '[%s]' % (test_time.rjust(6)) + if doc['skipped']: - parts += [ 'SKIPPED', doc['testcase_name'] ] + test_result = 'SKIPPED' elif doc['success']: if doc['timeout']: - parts += [ red('TIMEOUT'), doc['testcase_name'] ] + test_result = red('TIMEOUT') else: - parts += [ green('SUCCESS'), doc['testcase_name'] ] + test_result = green('SUCCESS') else: if doc['timeout']: - parts += [ red('TIMEOUT'), doc['testcase_name'] ] + test_result = red('TIMEOUT') elif doc['knownissue'] != '': - parts += [ blue('KNOWN '), doc['testcase_name'] ] + test_result = blue('KNOWN ') print_diff = False else: - parts += [ red('FAILURE'), doc['testcase_name'] ] + test_result = red('FAILURE') - parts += [ '[%d diff lines]' % count_lines(doc['diff_expect']) ] + issues += [ '[%d diff lines]' % count_lines(doc['diff_expect']) ] if doc['knownissue'] != '': - parts += [ '[known: ' + doc['knownissue'] + ']' ] + issues += [ '[known: ' + doc['knownissue'] + ']' ] if len(doc['errors']) > 0: - parts += [ '[errors: ' + ','.join(doc['errors']) + ']' ] + issues += [ '[errors: ' + ','.join(doc['errors']) + ']' ] + + parts += [ test_result, test_name ] - if doc['duration'] >= 30.0: + if doc['duration'] >= 60.0: parts += [ blue(test_time) ] elif doc['duration'] >= 5.0: parts += [ yellow(test_time) ] else: parts += [ test_time ] + if doc.has_key('massif_peak_heap_bytes'): + tmp = [] + tmp += [ '%s heap' % fmt_size(doc['massif_peak_heap_bytes']) ] + #tmp += [ '%s stack' % fmt_size(doc['massif_peak_stack_bytes']) ] + parts += [ '[%s]' % (', '.join(tmp).rjust(14)) ] + + if doc.has_key('valgrind_tool'): + parts += [ grey('[%s]' % doc['valgrind_tool']) ] + + parts += issues + print(' '.join(parts)) if doc['stderr'] != '' and not meta.get('intended_uncaught', False): @@ -604,6 +729,9 @@ def main(): parser.add_option('--duk', dest='duk', default=None, help='Path to "duk" command, default is autodetect') parser.add_option('--timeout', dest='timeout', type='int', default=15*60, help='Test execution timeout (seconds), default 15min') parser.add_option('--valgrind', dest='valgrind', action='store_true', default=False, help='Run test inside valgrind') + parser.add_option('--valgrind-tool', dest='valgrind_tool', default=None, help='Valgrind tool to use (implies --valgrind)') + parser.add_option('--memcheck', dest='memcheck', default=False, action='store_true', help='Shorthand for --valgrind-tool memcheck') + parser.add_option('--massif', dest='massif', default=False, action='store_true', help='Shorthand for --valgrind-tool massif') parser.add_option('--prepare-only', dest='prepare_only', action='store_true', default=False, help='Only prepare a testcase without running it') parser.add_option('--clip-lines', dest='clip_lines', type='int', default=10, help='Number of lines for stderr/diff summaries') parser.add_option('--clip-columns', dest='clip_columns', type='int', default=80, help='Number of columns for stderr/diff summaries') @@ -623,6 +751,16 @@ def main(): if opts.known_issues is None: opts.known_issues = find_known_issues() #print('Autodetect known issues directory: %r' % opts.known_issues) + if opts.memcheck: + opts.valgrind = True + opts.valgrind_tool = 'memcheck' + if opts.massif: + opts.valgrind = True + opts.valgrind_tool = 'massif' + if opts.valgrind_tool is not None: + opts.valgrind = True + if opts.valgrind and opts.valgrind_tool is None: + opts.valgrind_tool = 'memcheck' # Create a temporary directory for anything test related, automatic # atexit deletion. Plumbed through globals to minimize argument passing.