From fc6bcaf22d88017551db2c50efff7420ba3cdf13 Mon Sep 17 00:00:00 2001 From: Isaac Hwang Date: Tue, 21 Dec 2021 07:51:23 -0800 Subject: [PATCH] Move the source DB builder into pyls since pyre does not need it anymore Summary: Original discussion on this diff - D31055253 (https://github.com/facebook/pyre-check/commit/71c05f34c7a080e26cd0edddf422fb89b487c036). grievejia mentioned that the Pyre team doesn't need this dep anymore, so let's just shovel it into PYLS so we can mess with it to our heart's content. Reviewed By: grievejia Differential Revision: D31406584 fbshipit-source-id: 6e4b2a1bc937c3aa253f5005141b6f23d7d324ff --- client/source_database_buck_builder.py | 241 ----------------- .../source_database_buck_builder_test.py | 255 ------------------ 2 files changed, 496 deletions(-) delete mode 100644 client/source_database_buck_builder.py delete mode 100644 client/tests/source_database_buck_builder_test.py diff --git a/client/source_database_buck_builder.py b/client/source_database_buck_builder.py deleted file mode 100644 index ca07a9daaf5..00000000000 --- a/client/source_database_buck_builder.py +++ /dev/null @@ -1,241 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import itertools -import json -import logging -import re -import shutil -import subprocess -import sys -import tempfile -from itertools import chain -from pathlib import Path -from typing import Dict, List, Optional - -from typing_extensions import TypedDict - - -LOG: logging.Logger = logging.getLogger(__name__) - - -class SourceDatabase(TypedDict): - sources: Dict[str, str] - dependencies: Dict[str, str] - - -def _buck( - arguments: List[str], - isolation_prefix: Optional[str], - buck_root: Path, -) -> str: - isolation_prefix_arguments = ( - ["--isolation_prefix", isolation_prefix] - if isolation_prefix is not None and len(isolation_prefix) > 0 - else [] - ) - command = ( - ["buck"] - + isolation_prefix_arguments - + arguments - + ["--config", "client.id=pyre"] - ) - LOG.debug("Running `%s`", " ".join(command)) - return subprocess.check_output( - command, stderr=subprocess.PIPE, cwd=str(buck_root) - ).decode("utf-8") - - -def _get_buck_query_arguments( - specifications: List[str], mode: Optional[str] -) -> List[str]: - mode_sublist = ["@mode/" + mode] if mode is not None else [] - return [ - "query", - "--json", - *mode_sublist, - 'kind("python_binary|python_library|python_test", %s)' - # Don't check generated rules. - + " - attrfilter(labels, generated, %s)" - # `python_unittest()` sources are separated into a macro-generated - # library, so make sure we include those. - + " + attrfilter(labels, unittest-library, %s)" - # Provide an opt-out label so that rules can avoid type-checking (e.g. - # some libraries wrap generated sources which are expensive to build - # and therefore typecheck). - + " - attrfilter(labels, no_pyre, %s)", - *specifications, - ] - - -def _normalize_specification(specification: str) -> str: - return specification if "//" in specification else "//" + specification - - -def _ignore_target(target: str) -> bool: - suffixes_for_ignored_targets = ("-mypy_ini", "-testmodules-lib") - return target.endswith(suffixes_for_ignored_targets) - - -def _load_json_ignoring_extra_data(source: str) -> Dict[str, str]: - try: - return json.loads(source) - except json.JSONDecodeError as exception: - LOG.debug(f"JSON output: {source}") - LOG.warning("Failed to parse JSON. Retrying by ignoring extra data...") - - match = re.search(r"Extra data: line ([0-9]+) column", exception.args[0]) - if match is None: - raise exception - - line_number = int(match.group(1)) - source_without_extra_data = "\n".join(source.splitlines()[: line_number - 1]) - return json.loads(source_without_extra_data) - - -def _query_targets( - target_specifications: List[str], - mode: Optional[str], - isolation_prefix: Optional[str], - buck_root: Path, -) -> List[str]: - normalized_target_specifications = [ - _normalize_specification(specification) - for specification in target_specifications - ] - query_arguments = _get_buck_query_arguments(normalized_target_specifications, mode) - LOG.info("Running `buck query`...") - specification_targets_dictionary = _load_json_ignoring_extra_data( - _buck(query_arguments, isolation_prefix, buck_root) - ) - targets = list(chain(*specification_targets_dictionary.values())) - return [target for target in targets if not _ignore_target(target)] - - -def _get_buck_build_arguments(mode: Optional[str], targets: List[str]) -> List[str]: - # NOTE(agallagher): We could potentially use flags like - # `-c fbcode.py_version=3 -c fbcode.platform=platform007` to force everything - # onto a consistent set of platforms, but this has a cost of invalidating the - # parser cache, which may not be worth it. - mode_sublist = ["@mode/" + mode] if mode is not None else [] - return [ - *mode_sublist, - "--show-full-json-output", - *(f"{target}#source-db" for target in targets), - ] - - -def _build_targets( - targets: List[str], - mode: Optional[str], - isolation_prefix: Optional[str], - buck_root: Path, -) -> Dict[str, str]: - build_arguments = _get_buck_build_arguments(mode, targets) - LOG.info("Running `buck build`...") - with tempfile.NamedTemporaryFile( - "w+", prefix="pyre_buck_build_arguments" - ) as arguments_file: - build_args_contents = "\n".join(build_arguments) - arguments_file.write(build_args_contents) - arguments_file.flush() # Ensure the contents get to file - output = _buck( - ["build", f"@{arguments_file.name}"], isolation_prefix, buck_root - ) - return _load_json_ignoring_extra_data(output) - - -def _load_source_databases( - target_path_dictionary: Dict[str, str] -) -> Dict[str, SourceDatabase]: - return { - target: json.loads(Path(path).read_text()) - for target, path in target_path_dictionary.items() - } - - -def _merge_source_databases(databases: Dict[str, SourceDatabase]) -> Dict[str, str]: - link_map = {} - for _target, database in sorted(databases.items(), key=lambda pair: pair[0]): - for destination, source in itertools.chain( - database["sources"].items(), database["dependencies"].items() - ): - # Ignore non-Python sources. - if Path(destination).suffix not in (".py", ".pyi"): - continue - - # These auto-generated modules are duplicated between test/binary - # rules and so conflict when merging. In practice, they're probably - # not important for type checking, so just ignore them. - if destination in [ - "__manifest__.py", - "__test_modules__.py", - "__test_main__.py", - ]: - continue - - # If we've already created the link, skip. - link_map.setdefault(destination, source) - return link_map - - -def _build_link_tree( - link_map: Dict[str, str], output_directory: Path, buck_root: Path -) -> None: - """ - Create a symlink tree where we merge the transitive dependency modules for all - Python rules. - """ - shutil.rmtree(output_directory, ignore_errors=True) - output_directory.mkdir(parents=True) - for destination, source in link_map.items(): - source_path = buck_root / source - assert source_path.exists(), source_path - destination_path = output_directory / destination - destination_path.parent.mkdir(parents=True, exist_ok=True) - destination_path.symlink_to(source_path) - - -def build( - target_specifications: List[str], - output_directory: Path, - buck_root: Path, - mode: Optional[str], - isolation_prefix: Optional[str], -) -> None: - targets = _query_targets(target_specifications, mode, isolation_prefix, buck_root) - target_path_dictionary = _build_targets(targets, mode, isolation_prefix, buck_root) - source_databases = _load_source_databases(target_path_dictionary) - link_map = _merge_source_databases(source_databases) - _build_link_tree(link_map, output_directory, buck_root) - - -def main(argv: List[str]) -> None: - parser = argparse.ArgumentParser() - parser.add_argument("-J") - parser.add_argument("--debug", action="store_true") - parser.add_argument("--mode") - parser.add_argument("--project_name") - parser.add_argument("--isolation_prefix") - parser.add_argument("--output_directory", required=True, type=Path) - parser.add_argument("--buck_root", dest="buck_root", required=True, type=Path) - parser.add_argument("target_specifications", nargs="*") - arguments = parser.parse_args(argv[1:]) - - build( - arguments.target_specifications, - arguments.output_directory, - arguments.buck_root, - arguments.mode, - arguments.isolation_prefix, - ) - - -if __name__ == "__main__": - logging.basicConfig( - level=logging.INFO, format="[%(asctime)s] [%(levelname)s] %(message)s" - ) - sys.exit(main(sys.argv)) diff --git a/client/tests/source_database_buck_builder_test.py b/client/tests/source_database_buck_builder_test.py deleted file mode 100644 index deccf6ec4df..00000000000 --- a/client/tests/source_database_buck_builder_test.py +++ /dev/null @@ -1,255 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import json -import shutil -import unittest -from pathlib import Path -from unittest.mock import MagicMock, call, patch - -from .. import source_database_buck_builder - - -class SourceDatabaseBuckBuilderTest(unittest.TestCase): - def setUp(self) -> None: - self._query_arguments = [ - "query", - "--json", - 'kind("python_binary|python_library|python_test", %s) ' - "- attrfilter(labels, generated, %s) " - "+ attrfilter(labels, unittest-library, %s) " - "- attrfilter(labels, no_pyre, %s)", - "//foo/bar/...", - "//bar:baz", - ] - - def test_get_buck_query_arguments(self) -> None: - arguments = source_database_buck_builder._get_buck_query_arguments( - specifications=["//foo/bar/...", "//bar:baz"], mode=None - ) - self.assertEqual(arguments, self._query_arguments) - - def test_get_buck_query_arguments__with_mode(self) -> None: - arguments = source_database_buck_builder._get_buck_query_arguments( - specifications=["//foo/bar/...", "//bar:baz"], mode="foo" - ) - self.assertEqual( - arguments, - [ - "query", - "--json", - "@mode/foo", - 'kind("python_binary|python_library|python_test", %s) ' - "- attrfilter(labels, generated, %s) " - "+ attrfilter(labels, unittest-library, %s) " - "- attrfilter(labels, no_pyre, %s)", - "//foo/bar/...", - "//bar:baz", - ], - ) - - # pyre-fixme[56]: Pyre was not able to infer the type of argument - # `tools.pyre.client.source_database_buck_builder` to decorator factory - # `unittest.mock.patch.object`. - @patch.object(source_database_buck_builder, "_buck") - def test_query_targets(self, buck: MagicMock) -> None: - query_output = { - "//foo/bar/...": ["//foo/bar:baz", "//foo/bar:tests-library"], - "//bar:baz": [ - "//bar:baz", - "//bar:tests-mypy_ini", - "//bar:tests-library-testmodules-lib", - ], - } - buck.return_value = json.dumps(query_output) - - self.assertEqual( - source_database_buck_builder._query_targets( - ["//foo/bar/...", "//bar:baz"], - isolation_prefix=None, - mode=None, - buck_root=Path(""), - ), - ["//foo/bar:baz", "//foo/bar:tests-library", "//bar:baz"], - ) - - def test_buck_build_arguments(self) -> None: - self.assertEqual( - source_database_buck_builder._get_buck_build_arguments( - mode="opt", targets=["//foo/bar:baz", "//foo/bar:tests-library"] - ), - [ - "@mode/opt", - "--show-full-json-output", - "//foo/bar:baz#source-db", - "//foo/bar:tests-library#source-db", - ], - ) - - # pyre-fixme[56]: Argument `json` to decorator factory - # `unittest.mock.patch.object` could not be resolved in a global scope. - @patch.object(json, "loads") - @patch.object(Path, "read_text") - def test_load_source_databases( - self, read_text: MagicMock, loads: MagicMock - ) -> None: - expected_database = { - "sources": {"bar.py": "some/other/bar.py"}, - "dependencies": {"foo.py": "some/foo.py"}, - } - loads.return_value = expected_database - source_databases = source_database_buck_builder._load_source_databases( - {"//foo:bar#source-db": "/some/bar#source-db/db.json"} - ) - self.assertEqual(source_databases, {"//foo:bar#source-db": expected_database}) - - def test_merge_source_databases(self) -> None: - actual = source_database_buck_builder._merge_source_databases( - { - "hello": { - "sources": { - "foo.py": "foo.py", - "duplicate.py": "duplicate_in_hello.py", - }, - "dependencies": { - "bar.pyi": "buck-out/bar.pyi", - "bar.cpp": "bar.cpp", - }, - }, - "foo": { - "sources": {}, - "dependencies": { - "foo2.pyi": "buck-out/foo2.pyi", - "bar2.cpp": "bar2.cpp", - "duplicate.py": "duplicate_in_foo.py", - "__manifest__.py": "__manifest__.py", - "__test_modules__.py": "__test_modules__.py", - "__test_main__.py": "__test_main__.py", - }, - }, - } - ) - self.assertEqual( - actual, - { - "foo.py": "foo.py", - "duplicate.py": "duplicate_in_foo.py", - "bar.pyi": "buck-out/bar.pyi", - "foo2.pyi": "buck-out/foo2.pyi", - }, - ) - - # pyre-fixme[56]: Argument `shutil` to decorator factory - # `unittest.mock.patch.object` could not be resolved in a global scope. - @patch.object(shutil, "rmtree") - @patch.object(Path, "exists") - @patch.object(Path, "mkdir") - @patch.object(Path, "symlink_to") - def test_build_link_tree( - self, - symlink_to: MagicMock, - make_directory: MagicMock, - exists: MagicMock, - remove_tree: MagicMock, - ) -> None: - source_database_buck_builder._build_link_tree( - {"foo.py": "foo.py", "bar/baz.pyi": "buck-out/bar.pyi"}, - Path("foo_directory"), - Path("/root"), - ) - self.assertEqual( - make_directory.call_args_list, - [ - call(parents=True), - call(parents=True, exist_ok=True), - call(parents=True, exist_ok=True), - ], - ) - self.assertEqual( - symlink_to.call_args_list, - [call(Path("/root/foo.py")), call(Path("/root/buck-out/bar.pyi"))], - ) - - @patch.object(source_database_buck_builder, "_build_link_tree") - @patch.object(source_database_buck_builder, "_load_source_databases") - @patch.object(source_database_buck_builder, "_build_targets") - # pyre-fixme[56]: Argument - # `tools.pyre.tools.buck_project_builder.source_database_buck_builder` to - # decorator factory `unittest.mock.patch.object` could not be resolved in a global - # scope. - @patch.object(source_database_buck_builder, "_query_targets") - def test_build( - self, - query_targets: MagicMock, - build_targets: MagicMock, - load_source_databases: MagicMock, - build_link_tree: MagicMock, - ) -> None: - load_source_databases.return_value = { - "hello": {"sources": {"foo.py": "foo.py"}, "dependencies": {}}, - "foo": {"sources": {}, "dependencies": {"bar.pyi": "buck-out/bar.pyi"}}, - } - source_database_buck_builder.build( - ["//foo/bar/..."], - output_directory=Path("output_directory"), - buck_root=Path("buck_root"), - isolation_prefix=None, - mode=None, - ) - query_targets.assert_called_once() - build_targets.assert_called_once() - build_link_tree.assert_called_once_with( - {"foo.py": "foo.py", "bar.pyi": "buck-out/bar.pyi"}, - Path("output_directory"), - Path("buck_root"), - ) - - def test_normalize_specification(self) -> None: - self.assertEqual( - source_database_buck_builder._normalize_specification("foo/bar:baz"), - "//foo/bar:baz", - ) - self.assertEqual( - source_database_buck_builder._normalize_specification( - "some_root//foo/bar:baz" - ), - "some_root//foo/bar:baz", - ) - - def test_load_json__no_extra_data(self) -> None: - self.assertEqual( - source_database_buck_builder._load_json_ignoring_extra_data( - """ - { - "a": "b", - "a2": "b2" - } - """ - ), - {"a": "b", "a2": "b2"}, - ) - - def test_load_json__extra_data(self) -> None: - self.assertEqual( - source_database_buck_builder._load_json_ignoring_extra_data( - """ - { - "a": "b", - "a2": "b2" - } - Some error message. - Some error message. - """ - ), - {"a": "b", "a2": "b2"}, - ) - - def test_load_json__exception(self) -> None: - with self.assertRaises(json.JSONDecodeError): - source_database_buck_builder._load_json_ignoring_extra_data( - """ - Malformed JSON. - """ - )