diff --git a/README.rst b/README.rst index 3a4f3d2..b084b02 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,26 @@ -============================== -Profimp - python import tracer -============================== +=============================== +Profimp - python imports tracer +=============================== +Profimp allows you to trace imports of your code. +This lib should be used to simplify optimization of imports in your code. +At least you will find what consumes the most part of time and do the +right decisions. + +Syntax: + +.. code-block:: + + profimp [import_module_line] + +Samples: + +.. code-block:: + + profimp "import re" + + or + + profimp "from somemoudle import something" diff --git a/doc/source/index.rst b/doc/source/index.rst index 05a453a..abbde87 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -14,8 +14,27 @@ under the License. -Profimp - python import tracer -============================== +Profimp - python imports tracer +=============================== +Profimp allows you to trace imports of your code. +This lib should be used to simplify optimization of imports in your code. +At least you will find what consumes the most part of time and do the +right decisions. +Syntax: + +.. code-block:: + + profimp [import_module_line] + +Samples: + +.. code-block:: + + profimp "import re" + + or + + profimp "from somemoudle import something" diff --git a/profimp/main.py b/profimp/main.py index 21dba48..54fec5d 100644 --- a/profimp/main.py +++ b/profimp/main.py @@ -13,9 +13,54 @@ # License for the specific language governing permissions and limitations # under the License. +from __future__ import print_function + +import sys + +from profimp import reports +from profimp import tracer + + +HELP_MESSAGE = """ + Profimp allows you to trace imports of your code. + + This lib should be used to simplify optimization of imports in your code. + At least you will find what consumes the most part of time and do the + right decisions. + + Syntax: + profimp [import_module_line] + + Samples: + profimp "import re" + + or + + profimp "from somemoudle import something" +""" + + +def print_help(): + print(HELP_MESSAGE) + + +def trace_module(import_line): + root_pt = tracer.init_stack() + with tracer.patch_import(): + exec(import_line) + return root_pt + def main(): - return "Hello world" + if len(sys.argv) == 1: + print_help() + elif len(sys.argv) == 2: + report = reports.to_json(trace_module(sys.argv[1])) + sys.stdout.write(report) + else: + print_help() + raise SystemExit("Wrong input arguments: %s" % sys.argv) + if __name__ == "__main__": main() diff --git a/tests/unit/test_noop.py b/profimp/reports.py similarity index 78% rename from tests/unit/test_noop.py rename to profimp/reports.py index b12aa1f..11a2fee 100644 --- a/tests/unit/test_noop.py +++ b/profimp/reports.py @@ -13,11 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. -from tests.unit import test +import json -class NoopTestCase(test.TestCase): - """Test case base class for all unit tests.""" - - def test_noop(self): - self.assertEqual(4, 2 + 2) +def to_json(results): + return json.dumps(results.to_dict(), indent=2) diff --git a/profimp/tracer.py b/profimp/tracer.py new file mode 100644 index 0000000..4676121 --- /dev/null +++ b/profimp/tracer.py @@ -0,0 +1,98 @@ +# Copyright 2015: Boris Pavlovic +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import contextlib +import time + +from six.moves import builtins + + +class TracePoint(object): + + def __init__(self, import_line, level=0): + self.started_at = 0 + self.finished_at = 0 + self.import_line = import_line + self.level = level + self.children = [] + + def __enter__(self): + self.start() + return self + + def __exit__(self, etype, value, traceback): + self.stop() + + def to_dict(self): + + result = { + "started_at": self.started_at, + "finished_at": self.finished_at, + "duration": (self.finished_at - self.started_at) * 1000, + "import_line": self.import_line, + "level": self.level, + "children": [] + } + + for child in self.children: + result["children"].append(child.to_dict()) + + return result + + def start(self): + self.started_at = time.time() + + def stop(self): + self.finished_at = time.time() + + def add_child(self, child): + self.children.append(child) + child.level = self.level + 1 + + +TRACE_STACK = [] + + +def init_stack(): + global TRACE_STACK + TRACE_STACK = [TracePoint("root", 0)] + return TRACE_STACK[0] + + +@contextlib.contextmanager +def patch_import(): + old_import = builtins.__import__ + builtins.__import__ = _traceit(builtins.__import__) + yield + builtins.__import__ = old_import + + +def _traceit(f): + def w(*args, **kwargs): + import_line = "" + + if len(args) > 3 and args[3]: + import_line = "from %s import %s" % (args[0], ", ".join(args[3])) + else: + import_line = "import %s" % args[0] + + with TracePoint(import_line) as trace_pt: + TRACE_STACK[-1].add_child(trace_pt) + TRACE_STACK.append(trace_pt) + try: + return f(*args, **kwargs) + finally: + TRACE_STACK.pop() + return w diff --git a/tests/hacking/__init__.py b/tests/hacking/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/hacking/checks.py b/tests/hacking/checks.py new file mode 100644 index 0000000..e0e6237 --- /dev/null +++ b/tests/hacking/checks.py @@ -0,0 +1,251 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Guidelines for writing new hacking checks + - Use only for Rally specific tests. OpenStack general tests + should be submitted to the common 'hacking' module. + - Pick numbers in the range N3xx. Find the current test with + the highest allocated number and then pick the next value. + - Keep the test method code in the source file ordered based + on the N3xx value. + - List the new rule in the top level HACKING.rst file + - Add test cases for each new rule to tests/test_hacking.py +""" + +import functools +import re + + +re_assert_true_instance = re.compile( + r"(.)*assertTrue\(isinstance\((\w|\.|\'|\"|\[|\])+, " + r"(\w|\.|\'|\"|\[|\])+\)\)") +re_assert_equal_type = re.compile( + r"(.)*assertEqual\(type\((\w|\.|\'|\"|\[|\])+\), " + r"(\w|\.|\'|\"|\[|\])+\)") +re_assert_equal_end_with_none = re.compile(r"assertEqual\(.*?,\s+None\)$") +re_assert_equal_start_with_none = re.compile(r"assertEqual\(None,") +re_assert_true_false_with_in_or_not_in = re.compile( + r"assert(True|False)\(" + r"(\w|[][.'\"])+( not)? in (\w|[][.'\",])+(, .*)?\)") +re_assert_true_false_with_in_or_not_in_spaces = re.compile( + r"assert(True|False)\((\w|[][.'\"])+( not)? in [\[|'|\"](\w|[][.'\", ])+" + r"[\[|'|\"](, .*)?\)") +re_assert_equal_in_end_with_true_or_false = re.compile( + r"assertEqual\((\w|[][.'\"])+( not)? in (\w|[][.'\", ])+, (True|False)\)") +re_assert_equal_in_start_with_true_or_false = re.compile( + r"assertEqual\((True|False), (\w|[][.'\"])+( not)? in (\w|[][.'\", ])+\)") + + +def skip_ignored_lines(func): + + @functools.wraps(func) + def wrapper(logical_line, filename): + line = logical_line.strip() + if not line or line.startswith("#") or line.endswith("# noqa"): + return + yield next(func(logical_line, filename)) + + return wrapper + + +def _parse_assert_mock_str(line): + point = line.find(".assert_") + + if point != -1: + end_pos = line[point:].find("(") + point + return point, line[point + 1: end_pos], line[: point] + else: + return None, None, None + + +@skip_ignored_lines +def check_assert_methods_from_mock(logical_line, filename): + """Ensure that ``assert_*`` methods from ``mock`` library is used correctly + + N301 - base error number + N302 - related to nonexistent "assert_called" + N303 - related to nonexistent "assert_called_once" + """ + + correct_names = ["assert_any_call", "assert_called_once_with", + "assert_called_with", "assert_has_calls"] + ignored_files = ["./tests/unit/test_hacking.py"] + + if filename.startswith("./tests") and filename not in ignored_files: + pos, method_name, obj_name = _parse_assert_mock_str(logical_line) + + if pos: + if method_name not in correct_names: + error_number = "N301" + msg = ("%(error_number)s:'%(method)s' is not present in `mock`" + " library. %(custom_msg)s For more details, visit " + "http://www.voidspace.org.uk/python/mock/ .") + + if method_name == "assert_called": + error_number = "N302" + custom_msg = ("Maybe, you should try to use " + "'assertTrue(%s.called)' instead." % + obj_name) + elif method_name == "assert_called_once": + # For more details, see a bug in Rally: + # https://bugs.launchpad.net/rally/+bug/1305991 + error_number = "N303" + custom_msg = ("Maybe, you should try to use " + "'assertEqual(1, %(obj_name)s.call_count)' " + "or '%(obj_name)s.assert_called_once_with()'" + " instead." % {"obj_name": obj_name}) + else: + custom_msg = ("Correct 'assert_*' methods: '%s'." + % "', '".join(correct_names)) + + yield (pos, msg % { + "error_number": error_number, + "method": method_name, + "custom_msg": custom_msg}) + + +@skip_ignored_lines +def assert_true_instance(logical_line, filename): + """Check for assertTrue(isinstance(a, b)) sentences + + N320 + """ + if re_assert_true_instance.match(logical_line): + yield (0, "N320 assertTrue(isinstance(a, b)) sentences not allowed, " + "you should use assertIsInstance(a, b) instead.") + + +@skip_ignored_lines +def assert_equal_type(logical_line, filename): + """Check for assertEqual(type(A), B) sentences + + N321 + """ + if re_assert_equal_type.match(logical_line): + yield (0, "N321 assertEqual(type(A), B) sentences not allowed, " + "you should use assertIsInstance(a, b) instead.") + + +@skip_ignored_lines +def assert_equal_none(logical_line, filename): + """Check for assertEqual(A, None) or assertEqual(None, A) sentences + + N322 + """ + res = (re_assert_equal_start_with_none.search(logical_line) or + re_assert_equal_end_with_none.search(logical_line)) + if res: + yield (0, "N322 assertEqual(A, None) or assertEqual(None, A) " + "sentences not allowed, you should use assertIsNone(A) " + "instead.") + + +@skip_ignored_lines +def assert_true_or_false_with_in(logical_line, filename): + """Check assertTrue/False(A in/not in B) with collection contents. + + Check for assertTrue/False(A in B), assertTrue/False(A not in B), + assertTrue/False(A in B, message) or assertTrue/False(A not in B, message) + sentences. + + N323 + """ + + res = (re_assert_true_false_with_in_or_not_in.search(logical_line) or + re_assert_true_false_with_in_or_not_in_spaces.search(logical_line)) + if res: + yield (0, "N323 assertTrue/assertFalse(A in/not in B)sentences not " + "allowed, you should use assertIn(A, B) or assertNotIn(A, B)" + " instead.") + + +@skip_ignored_lines +def assert_equal_in(logical_line, filename): + """Check assertEqual(A in/not in B, True/False) with collection contents + + Check for assertEqual(A in B, True/False), assertEqual(True/False, A in B), + assertEqual(A not in B, True/False) or assertEqual(True/False, A not in B) + sentences. + + N324 + """ + + res = (re_assert_equal_in_end_with_true_or_false.search(logical_line) or + re_assert_equal_in_start_with_true_or_false.search(logical_line)) + if res: + yield (0, "N324: Use assertIn/NotIn(A, B) rather than " + "assertEqual(A in/not in B, True/False) when checking " + "collection contents.") + + +@skip_ignored_lines +def check_quotes(logical_line, filename): + """Check that single quotation marks are not used + + N350 + """ + + in_string = False + in_multiline_string = False + single_quotas_are_used = False + + check_tripple = ( + lambda line, i, char: ( + i + 2 < len(line) and + (char == line[i] == line[i + 1] == line[i + 2]) + ) + ) + + i = 0 + while i < len(logical_line): + char = logical_line[i] + + if in_string: + if char == "\"": + in_string = False + if char == "\\": + i += 1 # ignore next char + + elif in_multiline_string: + if check_tripple(logical_line, i, "\""): + i += 2 # skip next 2 chars + in_multiline_string = False + + elif char == "#": + break + + elif char == "'": + single_quotas_are_used = True + break + + elif char == "\"": + if check_tripple(logical_line, i, "\""): + in_multiline_string = True + i += 3 + continue + in_string = True + + i += 1 + + if single_quotas_are_used: + yield (i, "N350 Remove Single quotes") + + +def factory(register): + register(check_assert_methods_from_mock) + register(assert_true_instance) + register(assert_equal_type) + register(assert_equal_none) + register(assert_true_or_false_with_in) + register(assert_equal_in) + register(check_quotes) diff --git a/tests/unit/test.py b/tests/unit/test.py index d2feb2b..dd63e06 100644 --- a/tests/unit/test.py +++ b/tests/unit/test.py @@ -14,7 +14,6 @@ # under the License. import mock - import testtools diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py new file mode 100644 index 0000000..736de3e --- /dev/null +++ b/tests/unit/test_main.py @@ -0,0 +1,60 @@ +# Copyright 2015: Boris Pavlovic +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from profimp import main +from tests.unit import test + + +class MainTestCase(test.TestCase): + + @mock.patch("profimp.main.print", create=True) + def test_print_help(self, mock_print): + main.print_help() + mock_print.assert_called_once_with(main.HELP_MESSAGE) + + def test_trace_module(self): + root_pt = main.trace_module("import re") + + self.assertEqual(1, len(root_pt.children)) + self.assertEqual("import re", root_pt.children[0].import_line) + + @mock.patch("profimp.main.print_help") + @mock.patch("profimp.main.sys") + def test_main_too_few_args(self, mock_sys, mock_print_help): + mock_sys.argv = ["profimp"] + main.main() + mock_print_help.assert_called_once_with() + + @mock.patch("profimp.main.reports") + @mock.patch("profimp.main.trace_module") + @mock.patch("profimp.main.sys") + def test_main(self, mock_sys, mock_trace_module, mock_reports): + mock_sys.argv = ["profimp", "import re"] + + mock_trace_module.return_value + main.main() + + mock_trace_module.assert_called_once_with("import re") + mock_reports.to_json.assert_called_once_with( + mock_trace_module.return_value) + + @mock.patch("profimp.main.print_help") + @mock.patch("profimp.main.sys") + def test_main_with_too_many_args(self, mock_sys, mock_print_help): + mock_sys.argv = ["profimp", "module_one", "something else"] + self.assertRaises(SystemExit, main.main) + mock_print_help.assert_called_once_with() diff --git a/tests/unit/test_reports.py b/tests/unit/test_reports.py new file mode 100644 index 0000000..e092907 --- /dev/null +++ b/tests/unit/test_reports.py @@ -0,0 +1,32 @@ +# Copyright 2015: Boris Pavlovic +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +import mock + +from profimp import reports +from tests.unit import test + + +class ReportsTestCase(test.TestCase): + + def test_to_json(self): + + results = mock.MagicMock() + results.to_dict.return_value = {"a": 1, "b": 20} + + self.assertEqual(json.dumps(results.to_dict.return_value, indent=2), + reports.to_json(results)) diff --git a/tests/unit/test_tracer.py b/tests/unit/test_tracer.py new file mode 100644 index 0000000..92b864e --- /dev/null +++ b/tests/unit/test_tracer.py @@ -0,0 +1,138 @@ +# Copyright 2015: Boris Pavlovic +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from six.moves import builtins + +from profimp import tracer +from tests.unit import test + + +class TracePointTestCase(test.TestCase): + + def test_init(self): + pt = tracer.TracePoint("some_import_line", level=10) + self.assertEqual(0, pt.started_at) + self.assertEqual(0, pt.finished_at) + self.assertEqual("some_import_line", pt.import_line) + self.assertEqual(10, pt.level) + self.assertEqual([], pt.children) + + @mock.patch("time.time", side_effect=[1, 2]) + def test_context(self, mock_time): + with tracer.TracePoint("some_import_line") as pt: + self.assertEqual(1, pt.started_at) + + self.assertEqual(2, pt.finished_at) + + @mock.patch("time.time", side_effect=[1, 2, 3, 4]) + def test_to_dict(self, mock_time): + pt = tracer.TracePoint("import_line", level=2) + pt_child = tracer.TracePoint("import_child") + pt.add_child(pt_child) + pt.start() + pt_child.start() + pt_child.stop() + pt.stop() + + expected_pt_child_dict = { + "started_at": 2, + "finished_at": 3, + "duration": 1000, + "import_line": "import_child", + "level": 3, + "children": [] + } + + exepected_pt_dict = { + "started_at": 1, + "finished_at": 4, + "duration": 3000, + "import_line": "import_line", + "level": 2, + "children": [expected_pt_child_dict] + } + + self.assertEqual(expected_pt_child_dict, pt_child.to_dict()) + self.assertEqual(exepected_pt_dict, pt.to_dict()) + + @mock.patch("time.time", side_effect=[10]) + def test_start(self, mock_time): + pt = tracer.TracePoint("import_line") + pt.start() + self.assertEqual(10, pt.started_at) + + @mock.patch("time.time", side_effect=[10]) + def test_stop(self, mock_time): + pt = tracer.TracePoint("import_line") + pt.stop() + self.assertEqual(10, pt.finished_at) + + def test_add_child(self): + pt = tracer.TracePoint("import_line") + pt_child1 = tracer.TracePoint("import_line") + pt_child11 = tracer.TracePoint("import_line") + pt_child12 = tracer.TracePoint("import_line") + pt_child2 = tracer.TracePoint("import_line") + + pt.add_child(pt_child1) + pt_child1.add_child(pt_child11) + pt_child1.add_child(pt_child12) + pt.add_child(pt_child2) + + self.assertEqual([pt_child11, pt_child12], pt_child1.children) + self.assertEqual([pt_child1, pt_child2], pt.children) + + self.assertEqual(0, pt.level) + self.assertEqual(1, pt_child1.level) + self.assertEqual(1, pt_child2.level) + self.assertEqual(2, pt_child11.level) + self.assertEqual(2, pt_child12.level) + + +class TraceModuleTestCase(test.TestCase): + + @mock.patch("profimp.tracer.TracePoint") + def test_init_stack(self, mock_trace_pt): + trace_pt = tracer.init_stack() + mock_trace_pt.assert_called_once_with("root", 0) + self.assertEqual(mock_trace_pt.return_value, trace_pt) + self.assertEqual([mock_trace_pt.return_value], tracer.TRACE_STACK) + + @mock.patch("profimp.tracer._traceit") + def test_patch_import(self, mock_traceit): + normal_import = builtins.__import__ + + with tracer.patch_import(): + self.assertEqual(mock_traceit.return_value, builtins.__import__) + self.assertNotEqual(normal_import, builtins.__import__) + + self.assertEqual(normal_import, builtins.__import__) + + def test__traceit(self): + tr_pt = tracer.init_stack() + with tracer.patch_import(): + exec("import re") + + self.assertEqual(1, len(tr_pt.children)) + self.assertEqual("import re", tr_pt.children[0].import_line) + + def test__traceit_from_import(self): + tr_pt = tracer.init_stack() + with tracer.patch_import(): + exec("from os import sys") + + self.assertEqual(1, len(tr_pt.children)) + self.assertEqual("from os import sys", tr_pt.children[0].import_line)