diff --git a/.gitignore b/.gitignore index f1bd3b8e..a6ec95ef 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .userconfig .var.profile *.swp +*.pyc Adreno* Makefile abi/ diff --git a/get-about-memory.py b/get-about-memory.py deleted file mode 100755 index d72b7f7c..00000000 --- a/get-about-memory.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python - -'''Get a dump of about:memory from all the processes running on your device. - -You can then view these dumps using Firefox on your desktop. - -We also include the output of b2g-procrank and b2g-ps. - -''' - -from __future__ import print_function -from __future__ import division - -import sys -import re -import os -import subprocess -import textwrap -import argparse -import json -from gzip import GzipFile -from time import sleep - -if sys.version_info < (2,7): - print('This script requires Python 2.7.') - sys.exit(1) - -def shell(cmd, cwd=None): - proc = subprocess.Popen(cmd, shell=True, cwd=cwd, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (out, err) = proc.communicate() - if proc.returncode: - print("Command %s failed with error code %d" % (cmd, proc.returncode), file=sys.stderr) - if err: - print(err, file=sys.stderr) - raise subprocess.CalledProcessError(proc.returncode, cmd, err) - return out - -def get_pids(): - """Get the pids of all gecko processes running on the device. - - Returns a tuple (master_pid, child_pids), where child_pids is a list. - - """ - procs = shell("adb shell ps").split('\n') - master_pid = None - child_pids = [] - for line in procs: - if re.search(r'/b2g\s*$', line): - if master_pid: - raise Exception("Two copies of b2g process found?") - master_pid = int(line.split()[1]) - if re.search(r'/plugin-container\s*$', line): - child_pids.append(int(line.split()[1])) - - if not master_pid: - raise Exception("b2g does not appear to be running on the device.") - - return (master_pid, child_pids) - -def list_files(): - return set(['/data/local/tmp/' + f.strip() for f in - shell("adb shell ls '/data/local/tmp'").split('\n') - if f.strip().startswith('memory-report-')]) - -def send_signal(args, pid): - # killer is a program we put on the device which is like kill(1), except it - # accepts signals above 31. It also understands "SIGRTn" to mean - # SIGRTMIN + n. - # - # SIGRT0 dumps memory reports, and SIGRT1 first minimizes memory usage and - # then dumps the reports. - signal = 'SIGRT0' if not args.minimize_memory_usage else 'SIGRT1' - shell("adb shell killer %s %d" % (signal, pid)) - -def choose_output_dir(args): - if args.output_directory: - return args.output_directory - - for i in range(0, 1024): - try: - dir = 'about-memory-%d' % i - os.mkdir(dir) - return dir - except: - pass - raise Exception("Couldn't create about-memory output directory.") - -def wait_for_all_files(num_expected_files, old_files): - wait_interval = .25 - max_wait = 30 - - warn_time = 5 - warned = False - - for i in range(0, int(max_wait / wait_interval)): - new_files = list_files() - old_files - - # For some reason, print() doesn't work with the \r hack. - sys.stdout.write('\rGot %d/%d files.' % (len(new_files), num_expected_files)) - sys.stdout.flush() - - if not warned and len(new_files) == 0 and i * wait_interval >= warn_time: - warned = True - sys.stdout.write('\r') - print(textwrap.fill(textwrap.dedent("""\ - The device may be asleep and not responding to our signal. - Try pressing a button on the device to wake it up.\n\n"""))) - - if len(new_files) == num_expected_files: - print('') - return - - sleep(wait_interval) - - print("We've waited %ds but the only about:memory dumps we see are" % max_wait) - print('\n'.join([' ' + f for f in new_files])) - print('We expected %d but see only %d files. Giving up...' % - (num_expected_files, len(new_files))) - raise Exception("Missing some about:memory dumps.") - -def get_files(args, master_pid, child_pids, old_files): - """Get the memory reporter dumps from the device and return the directory - we saved them to. - - """ - num_expected_files = 1 + len(child_pids) - - wait_for_all_files(num_expected_files, old_files) - new_files = list_files() - old_files - dir = choose_output_dir(args) - for f in new_files: - shell('adb pull %s' % f, cwd=dir) - pass - print("Pulled files into %s." % dir) - merge_files(dir, [os.path.basename(f) for f in new_files]) - return dir - -def merge_files(dir, files): - """Merge the given memory reporter dump files into one giant file.""" - dumps = [json.load(GzipFile(os.path.join(dir, f))) for f in files] - - merged_dump = dumps[0] - for dump in dumps[1:]: - # All of the properties other than 'reports' must be identical in all - # dumps, otherwise we can't merge them. - if set(dump.keys()) != set(merged_dump.keys()): - print("Can't merge dumps because they don't have the " - "same set of properties.") - return - for prop in merged_dump: - if prop != 'reports' and dump[prop] != merged_dump[prop]: - print("Can't merge dumps because they don't have the " - "same value for property '%s'" % prop) - - merged_dump['reports'] += dump['reports'] - - json.dump(merged_dump, - GzipFile(os.path.join(dir, 'merged-reports.gz'), 'w'), - indent=2) - -def remove_new_files(old_files): - # Hopefully this command line won't get too long for ADB. - shell('adb shell rm %s' % ' '.join(["'%s'" % f for f in list_files() - old_files])) - -def get_procrank_etc(dir): - shell('adb shell procrank > procrank', cwd=dir) - shell('adb shell b2g-ps > b2g-ps', cwd=dir) - shell('adb shell b2g-procrank > b2g-procrank', cwd=dir) - -def get_dumps(args): - (master_pid, child_pids) = get_pids() - old_files = list_files() - send_signal(args, master_pid) - dir = get_files(args, master_pid, child_pids, old_files) - if args.remove_from_device: - remove_new_files(old_files) - get_procrank_etc(dir) - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description=textwrap.dedent('''\ - This script pulls about:memory reports from a device. You can then - open these reports in desktop Firefox by visiting about:memory.''')) - - parser.add_argument('--minimize', '-m', dest='minimize_memory_usage', - action='store_true', default=False, - help='Minimize memory usage before collecting the memory reports.') - - parser.add_argument('--directory', '-d', dest='output_directory', - action='store', metavar='DIR', - help=textwrap.dedent('''\ - The directory to store the reports in. By default, we'll store the - reports in the directory about-memory-N, for some N.''')) - - parser.add_argument('--remove', '-r', dest='remove_from_device', - action='store_true', default=False, - help='Delete the reports from the device after pulling them.') - - args = parser.parse_args() - get_dumps(args) diff --git a/tools/get_about_memory.py b/tools/get_about_memory.py new file mode 100755 index 00000000..ae255c4b --- /dev/null +++ b/tools/get_about_memory.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python + +'''Get a dump of about:memory from all the processes running on your device. + +You can then view these dumps using a recent Firefox nightly on your desktop by +opening about:memory and using the button at the bottom of the page to load the +memory-reports file that this script creates. + +This script also saves the output of b2g-procrank and a few other diagnostic +programs. + +''' + +from __future__ import print_function + +import sys +if sys.version_info < (2,7): + # We need Python 2.7 because we import argparse. + print('This script requires Python 2.7.') + sys.exit(1) + +import os +import textwrap +import argparse +import json +from gzip import GzipFile + +import include.device_utils as utils + +def merge_files(dir, files): + '''Merge the given memory reporter dump files into one giant file.''' + dumps = [json.load(GzipFile(os.path.join(dir, f))) for f in files] + + merged_dump = dumps[0] + for dump in dumps[1:]: + # All of the properties other than 'reports' must be identical in all + # dumps, otherwise we can't merge them. + if set(dump.keys()) != set(merged_dump.keys()): + print("Can't merge dumps because they don't have the " + "same set of properties.") + return + for prop in merged_dump: + if prop != 'reports' and dump[prop] != merged_dump[prop]: + print("Can't merge dumps because they don't have the " + "same value for property '%s'" % prop) + + merged_dump['reports'] += dump['reports'] + + merged_reports_path = os.path.join (dir, 'memory-reports') + json.dump(merged_dump, + open(merged_reports_path, 'w'), + indent=2) + return merged_reports_path + +def get_dumps(args): + if args.output_directory: + out_dir = utils.create_specific_output_dir(args.output_directory) + else: + out_dir = utils.create_new_output_dir('about-memory-') + + # Do this function inside a try/catch which will delete out_dir if the + # function throws and out_dir is empty. + def do_work(): + signal = 'SIGRT0' if not args.minimize_memory_usage else 'SIGRT1' + new_files = utils.send_signal_and_pull_files( + signal=signal, + outfiles_prefixes=['memory-report-'], + remove_outfiles_from_device=not args.leave_on_device, + out_dir=out_dir) + + merged_reports_path = merge_files(out_dir, new_files) + utils.pull_procrank_etc(out_dir) + + if not args.keep_individual_reports: + for f in new_files: + os.remove(os.path.join(out_dir, f)) + + print() + print(textwrap.fill(textwrap.dedent('''\ + To view this report, open Firefox on your desktop, load + about:memory, click "read reports from a file" at the bottom, and + open %s''' % + os.path.abspath(merged_reports_path)))) + + utils.run_and_delete_dir_on_exception(do_work, out_dir) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + parser.add_argument('--minimize', '-m', dest='minimize_memory_usage', + action='store_true', default=False, + help='Minimize memory usage before collecting the memory reports.') + + parser.add_argument('--directory', '-d', dest='output_directory', + action='store', metavar='DIR', + help=textwrap.dedent('''\ + The directory to store the reports in. By default, we'll store the + reports in the directory about-memory-N, for some N.''')) + + parser.add_argument('--leave-on-device', '-l', dest='leave_on_device', + action='store_true', default=False, + help='Leave the reports on the device after pulling them.') + + parser.add_argument('--keep-individual-reports', + dest='keep_individual_reports', + action='store_true', default=False, + help=textwrap.dedent('''\ + Don't delete the individual memory reports which we merge to create + the memory-reports file. You shouldn't need to pass this parameter + except for debugging.''')) + + args = parser.parse_args() + get_dumps(args) diff --git a/tools/get_gc_cc_log.py b/tools/get_gc_cc_log.py new file mode 100755 index 00000000..588dc6e6 --- /dev/null +++ b/tools/get_gc_cc_log.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +'''This script pulls GC and CC logs from all B2G processes on a device. These +logs are primarily used by leak-checking tools. + +This script also saves the output of b2g-procrank and a few other diagnostic +programs. + +''' + +from __future__ import print_function + +import sys +if sys.version_info < (2,7): + # We need Python 2.7 because we import argparse. + print('This script requires Python 2.7.') + sys.exit(1) + +import os +import sys +import re +import argparse +import textwrap +import subprocess + +import include.device_utils as utils + +def compress_logs(log_filenames, out_dir): + # Compress with xz if we can; otherwise, use gzip. + try: + utils.shell('xz -V', show_errors=False) + compression_prog='xz' + except subprocess.CalledProcessError: + compression_prog='gzip' + + # Compress in parallel. While we're at it, we also strip off the + # long identifier from the filenames, if we can. (The filename is + # something like gc-log.PID.IDENTIFIER.log, where the identifier is + # something like the number of seconds since the epoch when the log was + # triggered.) + compression_procs = [] + for f in log_filenames: + # Rename the log file if we can. + match = re.match(r'^([a-zA-Z-]+\.[0-9]+)\.[0-9]+.log$', f) + if match: + if not os.path.exists(os.path.join(out_dir, match.group(1))): + new_name = match.group(1) + '.log' + os.rename(os.path.join(out_dir, f), + os.path.join(out_dir, new_name)) + f = new_name + + # Start compressing. + compression_procs.append((f, subprocess.Popen([compression_prog, f], + cwd=out_dir))) + # Wait for all the compression processes to finish. + for (filename, proc) in compression_procs: + proc.wait() + if proc.returncode: + print('Compression of %s failed!' % filename) + raise subprocess.CalledProcessError(proc.returncode, + [compression_prog, filename], + None) +def get_logs(args): + if args.output_directory: + out_dir = utils.create_specific_output_dir(args.output_directory) + else: + out_dir = utils.create_new_output_dir('gc-cc-logs-') + + def do_work(): + log_filenames = utils.send_signal_and_pull_files( + signal='SIGRT2', + outfiles_prefixes=['cc-edges.', 'gc-edges.'], + remove_outfiles_from_device=not args.leave_on_device, + out_dir=out_dir) + + compress_logs(log_filenames, out_dir) + utils.pull_procrank_etc(out_dir) + + utils.run_and_delete_dir_on_exception(do_work, out_dir) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + parser.add_argument('--directory', '-d', dest='output_directory', + action='store', metavar='DIR', + help=textwrap.dedent('''\ + The directory to store the logs in. By default, we'll store the + reports in the directory gc-cc-logs-N, for some N.''')) + + parser.add_argument('--leave-on-device', '-l', dest='leave_on_device', + action='store_true', default=False, + help=textwrap.dedent('''\ + Leave the logs on the device after pulling them. (Note: These logs + can take up tens of megabytes and are stored uncompressed on the + device!)''')) + + args = parser.parse_args() + get_logs(args) diff --git a/tools/include/__init__.py b/tools/include/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/include/device_utils.py b/tools/include/device_utils.py new file mode 100644 index 00000000..0585e35a --- /dev/null +++ b/tools/include/device_utils.py @@ -0,0 +1,239 @@ +'''Utilities for interacting with a remote device.''' + +from __future__ import print_function +from __future__ import division + +import os +import sys +import re +import subprocess +from time import sleep + +def remote_shell(cmd): + '''Run the given command on on the device and return stdout.''' + return shell("adb shell '%s'" % cmd) + +def shell(cmd, cwd=None, show_errors=True): + '''Run the given command as a shell script on the host machine. + + If cwd is specified, we run the command from that directory; otherwise, we + run the command from the current working directory. + + ''' + proc = subprocess.Popen(cmd, shell=True, cwd=cwd, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (out, err) = proc.communicate() + if proc.returncode: + if show_errors: + print('Command %s failed with error code %d' % + (cmd, proc.returncode), file=sys.stderr) + if err: + print(err, file=sys.stderr) + raise subprocess.CalledProcessError(proc.returncode, cmd, err) + return out + +def create_specific_output_dir(out_dir): + '''Create the given directory if it doesn't exist. + + Throw an exception if a non-directory file exists with the same name. + + ''' + if os.path.exists(out_dir): + if os.path.isdir(out_dir): + # Directory already exists; we're all good. + return + else: + raise Exception(textwrap.dedent('''\ + Can't use %s as output directory; something that's not a + directory already exists with that name.''' % out_dir)) + os.mkdir(out_dir) + +def create_new_output_dir(out_dir_prefix): + '''Create a new directory whose name begins with out_dir_prefix.''' + for i in range(0, 1024): + try: + dir = '%s%d' % (out_dir_prefix, i) + os.mkdir(dir) + return dir + except: + pass + raise Exception("Couldn't create output directory.") + +def get_remote_b2g_pids(): + '''Get the pids of all gecko processes running on the device. + + Returns a tuple (master_pid, child_pids), where child_pids is a list. + + ''' + procs = remote_shell('ps').split('\n') + master_pid = None + child_pids = [] + for line in procs: + if re.search(r'/b2g\s*$', line): + if master_pid: + raise Exception('Two copies of b2g process found?') + master_pid = int(line.split()[1]) + if re.search(r'/plugin-container\s*$', line): + child_pids.append(int(line.split()[1])) + + if not master_pid: + raise Exception('b2g does not appear to be running on the device.') + + return (master_pid, child_pids) + +def pull_procrank_etc(out_dir): + '''Get the output of procrank and a few other diagnostic programs and save + it into out_dir. + + ''' + shell('adb shell procrank > procrank', cwd=out_dir) + shell('adb shell b2g-ps > b2g-ps', cwd=out_dir) + shell('adb shell b2g-procrank > b2g-procrank', cwd=out_dir) + +def run_and_delete_dir_on_exception(fun, dir): + '''Run the given function and, if it throws an exception, delete the given + directory, if it's empty, before re-throwing the exception. + + You might want to wrap your call to send_signal_and_pull_files in this + function.''' + try: + fun() + except: + # os.rmdir will throw if the directory is non-empty, and a simple + # 'raise' will re-throw the exception from os.rmdir (if that throws), + # so we need to explicitly save the exception info here. See + # http://nedbatchelder.com/blog/200711/rethrowing_exceptions_in_python.html + exception_info = sys.exc_info() + + try: + # Throws if the directory is not empty. + os.rmdir(dir) + except OSError: + pass + + # Raise the original exception. + raise exception_info[1], None, exception_info[2] + +def send_signal_and_pull_files(signal, + outfiles_prefixes, + remove_outfiles_from_device, + out_dir): + '''Send a signal to the main B2G process and pull files created as a + result. + + We send the given signal (which may be either a number of a string of the + form 'SIGRTn', which we interpret as the signal SIGRTMIN + n) and pull the + files generated into out_dir on the host machine. We only pull files + which were created after the signal was sent. + + When we're done, we remove the files from the device if + remote_outfiles_from_device is true. + + outfiles_prefixes must be a list containing the beginnings of the files we + expect to be created as a result of the signal. For example, if we expect + to see files named 'foo-XXX' and 'bar-YYY', we'd set outfiles_prefixes to + ['foo-', 'bar-']. + + We expect to pull len(outfiles_prefixes) * (# b2g processes) files from the + device. + + ''' + (master_pid, child_pids) = get_remote_b2g_pids() + old_files = _list_remote_temp_files(outfiles_prefixes) + _send_remote_signal(signal, master_pid) + + num_expected_files = len(outfiles_prefixes) * (1 + len(child_pids)) + _wait_for_remote_files(outfiles_prefixes, num_expected_files, old_files) + new_files = _pull_remote_files(outfiles_prefixes, old_files, out_dir) + if remove_outfiles_from_device: + _remove_files_from_device(outfiles_prefixes, old_files) + return [os.path.basename(f) for f in new_files] + +# You probably don't need to call the functions below from outside this module, +# but hey, maybe you do. + +def _send_remote_signal(signal, pid): + '''Send a signal to a process on the device. + + signal can be either an integer or a string of the form 'SIGRTn' where n is + an integer. We interpret SIGRTn to mean the signal SIGRTMIN + n. + + ''' + # killer is a program we put on the device which is like kill(1), except it + # accepts signals above 31. It also understands "SIGRTn" per above. + remote_shell("killer %s %d" % (signal, pid)) + +def _list_remote_temp_files(prefixes): + '''Return a set of absolute filenames in the device's temp directory which + start with one of the given prefixes.''' + return set(['/data/local/tmp/' + f.strip() for f in + remote_shell('ls /data/local/tmp').split('\n') + if any([f.strip().startswith(prefix) for prefix in prefixes])]) + +def _wait_for_remote_files(outfiles_prefixes, num_expected_files, old_files): + '''Wait for files to appear on the remote device. + + We wait until we see num_expected_files whose names begin with one of the + elements of outfiles_prefixes and which aren't in old_files appear in the + device's temp directory. If we don't see these files after a timeout + expires, we throw an exception. + + ''' + wait_interval = .25 + max_wait = 30 + + warn_time = 5 + warned = False + + for i in range(0, int(max_wait / wait_interval)): + new_files = _list_remote_temp_files(outfiles_prefixes) - old_files + + # For some reason, print() doesn't work with the \r hack. + sys.stdout.write('\rGot %d/%d files.' % + (len(new_files), num_expected_files)) + sys.stdout.flush() + + if not warned and len(new_files) == 0 and i * wait_interval >= warn_time: + warned = True + sys.stdout.write('\r') + print(textwrap.fill(textwrap.dedent("""\ + The device may be asleep and not responding to our signal. + Try pressing a button on the device to wake it up.\n\n"""))) + + if len(new_files) == num_expected_files: + print('') + return + + sleep(wait_interval) + + print("We've waited %ds but the only relevant files we see are" % max_wait) + print('\n'.join([' ' + f for f in new_files])) + print('We expected %d but see only %d files. Giving up...' % + (num_expected_files, len(new_files))) + raise Exception("Unable to pull some files.") + +def _pull_remote_files(outfiles_prefixes, old_files, out_dir): + '''Pull files from the remote device's temp directory into out_dir. + + We pull each file in the temp directory whose name begins with one of the + elements of outfiles_prefixes and which isn't listed in old_files. + + ''' + new_files = _list_remote_temp_files(outfiles_prefixes) - old_files + for f in new_files: + shell('adb pull %s' % f, cwd=out_dir) + pass + print("Pulled files into %s." % out_dir) + return new_files + +def _remove_files_from_device(outfiles_prefixes, old_files): + '''Remove files from the remote device's temp directory. + + We remove all files starting with one of the elements of outfiles_prefixes + which aren't listed in old_files. + + ''' + files_to_remove = _list_remote_temp_files(outfiles_prefixes) - old_files + + # Hopefully this command line won't get too long for ADB. + remote_shell('rm %s' % ' '.join([str(f) for f in files_to_remove]))