From 01d6094b17a8cf18ff11e3fba8f9dee0d40b06f6 Mon Sep 17 00:00:00 2001 From: Alex Hornby Date: Fri, 6 Dec 2024 01:30:06 -0800 Subject: [PATCH] speed up file copy on windows Summary: X-link: https://github.com/facebookincubator/fizz/pull/155 X-link: https://github.com/facebookincubator/zstrong/pull/1085 getdeps windows file copy is very slow, speed it up Reviewed By: bigfootjon Differential Revision: D66830544 fbshipit-source-id: 43213e258c71ae706bb059600619e276e7491a90 --- build/fbcode_builder/getdeps/builder.py | 5 ++- build/fbcode_builder/getdeps/cargo.py | 3 +- build/fbcode_builder/getdeps/copytree.py | 47 ++++++++++++++++-------- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 6bd5f52..83916fd 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -18,6 +18,7 @@ from shlex import quote as shellquote from typing import Optional +from .copytree import simple_copytree from .dyndeps import create_dyn_dep_munger from .envfuncs import add_path_entry, Env, path_search from .fetcher import copy_if_different @@ -1329,7 +1330,7 @@ def build(self, reconfigure: bool) -> None: os.makedirs(dest_parent) if os.path.isdir(full_src): if not os.path.exists(full_dest): - shutil.copytree(full_src, full_dest) + simple_copytree(full_src, full_dest) else: shutil.copyfile(full_src, full_dest) shutil.copymode(full_src, full_dest) @@ -1341,7 +1342,7 @@ def build(self, reconfigure: bool) -> None: os.chmod(full_dest, st.st_mode | stat.S_IXUSR) else: if not os.path.exists(self.inst_dir): - shutil.copytree(self.src_dir, self.inst_dir) + simple_copytree(self.src_dir, self.inst_dir) class SqliteBuilder(BuilderBase): diff --git a/build/fbcode_builder/getdeps/cargo.py b/build/fbcode_builder/getdeps/cargo.py index 0e0e0dd..5bb2ada 100644 --- a/build/fbcode_builder/getdeps/cargo.py +++ b/build/fbcode_builder/getdeps/cargo.py @@ -13,6 +13,7 @@ import typing from .builder import BuilderBase +from .copytree import simple_copytree if typing.TYPE_CHECKING: from .buildopts import BuildOptions @@ -79,7 +80,7 @@ def recreate_dir(self, src, dst) -> None: os.remove(dst) else: shutil.rmtree(dst) - shutil.copytree(src, dst) + simple_copytree(src, dst) def cargo_config_file(self): build_source_dir = self.build_dir diff --git a/build/fbcode_builder/getdeps/copytree.py b/build/fbcode_builder/getdeps/copytree.py index 2297bd3..6815f74 100644 --- a/build/fbcode_builder/getdeps/copytree.py +++ b/build/fbcode_builder/getdeps/copytree.py @@ -10,6 +10,7 @@ import subprocess from .platform import is_windows +from .runcmd import run_cmd PREFETCHED_DIRS = set() @@ -65,18 +66,34 @@ def prefetch_dir_if_eden(dirpath) -> None: PREFETCHED_DIRS.add(dirpath) -# pyre-fixme[9]: ignore has type `bool`; used as `None`. -def copytree(src_dir, dest_dir, ignore: bool = None): - """Recursively copy the src_dir to the dest_dir, filtering - out entries using the ignore lambda. The behavior of the - ignore lambda must match that described by `shutil.copytree`. - This `copytree` function knows how to prefetch data when - running in an eden repo. - TODO: I'd like to either extend this or add a variant that - uses watchman to mirror src_dir into dest_dir. - """ - prefetch_dir_if_eden(src_dir) - # pyre-fixme[6]: For 3rd param expected - # `Union[typing.Callable[[Union[PathLike[str], str], List[str]], Iterable[str]], - # typing.Callable[[str, List[str]], Iterable[str]], None]` but got `bool`. - return shutil.copytree(src_dir, dest_dir, ignore=ignore) +def simple_copytree(src_dir, dest_dir, symlinks=False): + """A simple version of shutil.copytree() that can delegate to native tools if faster""" + if is_windows(): + os.makedirs(dest_dir, exist_ok=True) + cmd = [ + "robocopy.exe", + src_dir, + dest_dir, + # copy directories, including empty ones + "/E", + # Ignore Extra files in destination + "/XX", + # enable parallel copy + "/MT", + # be quiet + "/NFL", + "/NDL", + "/NJH", + "/NJS", + "/NP", + ] + if symlinks: + cmd.append("/SL") + # robocopy exits with code 1 if it copied ok, hence allow_fail + # https://learn.microsoft.com/en-us/troubleshoot/windows-server/backup-and-storage/return-codes-used-robocopy-utility + exit_code = run_cmd(cmd, allow_fail=True) + if exit_code > 1: + raise subprocess.CalledProcessError(exit_code, cmd) + return dest_dir + else: + return shutil.copytree(src_dir, dest_dir, symlinks=symlinks)