From cf4b567cc68267855767025bab9e10a3e2af64b3 Mon Sep 17 00:00:00 2001 From: torzdf <36920800+torzdf@users.noreply.github.com> Date: Sun, 5 Sep 2021 01:25:12 +0000 Subject: [PATCH] Tensorflow 2.6 Support (#1182) * lib.cli.launcher - Bump max tf version to 2.6 * Remove pathlib requirement * Update requirements files * Update setup.py * bugfix - GUI: Supress errors when attempting to load previews in extract * GUI: Suppress ptxas error messages for Windows --- INSTALL.md | 1 - _requirements_base.txt | 25 +++++++--------- docs/sphinx_requirements.txt | 19 ++++++------ lib/cli/launcher.py | 18 +++++------ lib/gui/utils.py | 8 +++++ lib/gui/wrapper.py | 3 ++ lib/model/backup_restore.py | 2 +- lib/utils.py | 15 ++++------ plugins/train/trainer/_base.py | 6 ++-- requirements_cpu.txt | 2 +- requirements_nvidia.txt | 2 +- scripts/convert.py | 4 +-- scripts/extract.py | 4 +-- scripts/fsmedia.py | 3 +- scripts/train.py | 18 +++++------ setup.py | 55 +++++++++++++++++++++------------- tools/manual/detected_faces.py | 2 +- tools/mask/mask.py | 4 +-- 18 files changed, 104 insertions(+), 87 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 2a94d4f246..02fe3068cf 100755 --- a/INSTALL.md +++ b/INSTALL.md @@ -257,7 +257,6 @@ WARNING Tensorflow has no official prebuild for CUDA 9.1 currently. Are System Dependencies met? [y/N] y INFO Installing Missing Python Packages... INFO Installing tensorflow-gpu -INFO Installing pathlib==1.0.1 ...... INFO Installing tqdm INFO Installing matplotlib diff --git a/_requirements_base.txt b/_requirements_base.txt index d356106e43..d4662e8ec7 100644 --- a/_requirements_base.txt +++ b/_requirements_base.txt @@ -1,18 +1,15 @@ -tqdm>=4.42 -psutil>=5.7.0 -pathlib==1.0.1 +tqdm>=4.62 +psutil>=5.8.0 numpy>=1.18.0,<1.20.0 -opencv-python>=4.1.2.0 -pillow>=7.0.0 -scikit-learn>=0.22.0 -fastcluster==1.1.26 +opencv-python>=4.5.3.0 +pillow>=8.3.1 +scikit-learn>=0.24.2 +fastcluster>=1.1.26 # matplotlib 3.3.1 breaks custom toolbar in graph popup -matplotlib>=3.0.3,<3.3.0 -imageio>=2.8.0 -imageio-ffmpeg>=0.4.2 +matplotlib>=3.2.0,<3.3.0 +imageio>=2.9.0 +imageio-ffmpeg>=0.4.5 ffmpy==0.2.3 -# Revert back to nvidia-ml-py3 when windows/system32 patch is implemented -git+https://github.com/deepfakes/nvidia-ml-py3.git -#nvidia-ml-py3 -pywin32>=227 ; sys_platform == "win32" +nvidia-ml-py>=11.470.66 +pywin32>=228 ; sys_platform == "win32" pynvx==1.0.0 ; sys_platform == "darwin" diff --git a/docs/sphinx_requirements.txt b/docs/sphinx_requirements.txt index 1a220a47f5..2c076eeb7f 100755 --- a/docs/sphinx_requirements.txt +++ b/docs/sphinx_requirements.txt @@ -1,20 +1,19 @@ # NB Do not install from this requirements file # It is for documentation purposes only -tqdm==4.42 -psutil==5.7.0 -pathlib==1.0.1 +tqdm==4.62 +psutil==5.8.0 numpy==1.18.0 -opencv-python==4.1.2.30 -pillow==7.0.0 -scikit-learn==0.22.0 +opencv-python==4.5.3.0 +pillow==8.3.1 +scikit-learn==0.24.2 fastcluster==1.1.26 -matplotlib>3.0.3,<3.3.0 -imageio==2.8.0 -imageio-ffmpeg==0.4.2 +matplotlib>3.2.0,<3.3.0 +imageio==2.9.0 +imageio-ffmpeg==0.4.5 ffmpy==0.2.3 nvidia-ml-py3 -pywin32==227 ; sys_platform == "win32" +pywin32==228 ; sys_platform == "win32" pynvx==1.0.0 ; sys_platform == "darwin" plaidml-keras==0.7.0 tensorflow==2.2.0 diff --git a/lib/cli/launcher.py b/lib/cli/launcher.py index 70aa1a2ed1..09e6f63bc5 100644 --- a/lib/cli/launcher.py +++ b/lib/cli/launcher.py @@ -53,10 +53,10 @@ def _test_for_tf_version(self): Raises ------ FaceswapError - If Tensorflow is not found, or is not between versions 2.2 and 2.4 + If Tensorflow is not found, or is not between versions 2.2 and 2.6 """ min_ver = 2.2 - max_ver = 2.4 + max_ver = 2.6 try: # Ensure tensorflow doesn't pin all threads to one core when using Math Kernel Library os.environ["TF_MIN_GPU_MULTIPROCESSOR_COUNT"] = "4" @@ -65,15 +65,16 @@ def _test_for_tf_version(self): except ImportError as err: if "DLL load failed while importing" in str(err): msg = ( - "A DLL library file failed to load. Make sure that you have Microsoft Visual " + f"A DLL library file failed to load. Make sure that you have Microsoft Visual " "C++ Redistributable (2015, 2017, 2019) installed for your machine from: " - "https://support.microsoft.com/en-gb/help/2977003") + "https://support.microsoft.com/en-gb/help/2977003. Original error: " + f"{str(err)}") else: msg = ( - "There was an error importing Tensorflow. This is most likely because you do " + f"There was an error importing Tensorflow. This is most likely because you do " "not have TensorFlow installed, or you are trying to run tensorflow-gpu on a " "system without an Nvidia graphics card. Original import " - "error: {}".format(str(err))) + f"error: {str(err)}") self._handle_import_error(msg) tf_ver = float(".".join(tf.__version__.split(".")[:2])) # pylint:disable=no-member @@ -126,9 +127,8 @@ def _test_tkinter(): If tkinter cannot be imported """ try: - # pylint: disable=unused-variable import tkinter # noqa pylint: disable=unused-import,import-outside-toplevel - except ImportError: + except ImportError as err: logger.error("It looks like TkInter isn't installed for your OS, so the GUI has been " "disabled. To enable the GUI please install the TkInter application. You " "can try:") @@ -139,7 +139,7 @@ def _test_tkinter(): logger.info("Arch: sudo pacman -S tk") logger.info("CentOS/Redhat: sudo yum install tkinter") logger.info("Fedora: sudo dnf install python3-tkinter") - raise FaceswapError("TkInter not found") + raise FaceswapError("TkInter not found") from err @staticmethod def _check_display(): diff --git a/lib/gui/utils.py b/lib/gui/utils.py index 3107fa084d..868ba22854 100644 --- a/lib/gui/utils.py +++ b/lib/gui/utils.py @@ -612,6 +612,14 @@ def _load_images_to_cache(self, image_files, frame_dims, thumbnail_size): fname, str(err)) dropped_files.append(fname) continue + except Exception as err: # pylint:disable=broad-except + # Swallow any issues with opening an image rather than spamming console + # Can happen when trying to read partially saved images + logger.debug("Error opening preview file: '%s'. Original error: %s", + fname, str(err)) + dropped_files.append(fname) + continue + width, height = img.size scaling = thumbnail_size / max(width, height) logger.debug("image width: %s, height: %s, scaling: %s", width, height, scaling) diff --git a/lib/gui/wrapper.py b/lib/gui/wrapper.py index 8ecea281c3..d258685c47 100644 --- a/lib/gui/wrapper.py +++ b/lib/gui/wrapper.py @@ -245,6 +245,9 @@ def read_stderr(self): if self.command == "train" and output.startswith("Reading training images"): print(output.strip(), file=sys.stdout) continue + if os.name == "nt" and "Call to CreateProcess failed. Error code: 2" in output: + # Suppress ptxas errors on Tensorflow for Windows + logger.debug("Suppressed call to subprocess error: '%s'", output) print(output.strip(), file=sys.stderr) logger.debug("Terminated stderr reader") diff --git a/lib/model/backup_restore.py b/lib/model/backup_restore.py index 6026228663..0408040b85 100644 --- a/lib/model/backup_restore.py +++ b/lib/model/backup_restore.py @@ -105,7 +105,7 @@ def snapshot_models(self, iterations): logger.debug("Removing previously existing snapshot folder: '%s'", snapshot_dir) rmtree(snapshot_dir) - dst = str(get_folder(snapshot_dir)) + dst = get_folder(snapshot_dir) for filename in os.listdir(self.model_dir): if not self._check_valid(filename, for_restore=False): logger.debug("Not snapshotting file: '%s'", filename) diff --git a/lib/utils.py b/lib/utils.py index dd5ee2b17d..3e81cbad7b 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -10,7 +10,6 @@ import warnings import zipfile -from pathlib import Path from re import finditer from multiprocessing import current_process from socket import timeout as socket_timeout, error as socket_error @@ -146,19 +145,18 @@ def get_folder(path, make_folder=True): Returns ------- - :class:`pathlib.Path` or `None` + str or `None` The path to the requested folder. If `make_folder` is set to ``False`` and the requested path does not exist, then ``None`` is returned """ logger = logging.getLogger(__name__) # pylint:disable=invalid-name logger.debug("Requested path: '%s'", path) - output_dir = Path(path) - if not make_folder and not output_dir.exists(): + if not make_folder and not os.path.isdir(path): logger.debug("%s does not exist", path) return None - output_dir.mkdir(parents=True, exist_ok=True) - logger.debug("Returning: '%s'", output_dir) - return output_dir + os.makedirs(path, exist_ok=True) + logger.debug("Returning: '%s'", path) + return path def get_image_paths(directory, extension=None): @@ -189,8 +187,7 @@ def get_image_paths(directory, extension=None): logger.trace("Scanned Folder Contents: %s", dir_scanned) for chkfile in dir_scanned: - if any([chkfile.name.lower().endswith(ext) - for ext in image_extensions]): + if any(chkfile.name.lower().endswith(ext) for ext in image_extensions): logger.trace("Adding '%s' to image list", chkfile.path) dir_contents.append(chkfile.path) diff --git a/plugins/train/trainer/_base.py b/plugins/train/trainer/_base.py index f664803b4f..ef7e6049ff 100644 --- a/plugins/train/trainer/_base.py +++ b/plugins/train/trainer/_base.py @@ -945,7 +945,7 @@ def _duplicate_headers(cls, headers, columns): The original headers duplicated by the number of columns """ for side, header in headers.items(): - duped = tuple([header for _ in range(columns)]) + duped = tuple(header for _ in range(columns)) headers[side] = np.concatenate(duped, axis=1) logger.debug("side: %s header.shape: %s", side, header.shape) return headers @@ -996,8 +996,8 @@ def _setup(self, input_a=None, input_b=None, output=None): """ logger.debug("Setting up time-lapse") if output is None: - output = str(get_folder(os.path.join(str(self._model.model_dir), - "{}_timelapse".format(self._model.name)))) + output = get_folder(os.path.join(str(self._model.model_dir), + f"{self._model.name}_timelapse")) self._output_file = str(output) logger.debug("Time-lapse output set to '%s'", self._output_file) diff --git a/requirements_cpu.txt b/requirements_cpu.txt index 29096926b7..0de8d8c2c6 100644 --- a/requirements_cpu.txt +++ b/requirements_cpu.txt @@ -1,2 +1,2 @@ -r _requirements_base.txt -tensorflow>=2.2.0,<2.5.0 +tensorflow>=2.2.0,<2.7.0 diff --git a/requirements_nvidia.txt b/requirements_nvidia.txt index ec8b87f880..eaeb261a3f 100644 --- a/requirements_nvidia.txt +++ b/requirements_nvidia.txt @@ -1,2 +1,2 @@ -r _requirements_base.txt -tensorflow-gpu>=2.2.0,<2.5.0 +tensorflow-gpu>=2.2.0,<2.7.0 diff --git a/scripts/convert.py b/scripts/convert.py index 5da15aa6c6..243a5d1124 100644 --- a/scripts/convert.py +++ b/scripts/convert.py @@ -107,7 +107,7 @@ def _validate(self): Ensure that certain cli selections are valid and won't result in an error. Checks: * If frames have been passed in with video output, ensure user supplies reference video. - * If "on-the-fly" and an NN mask is selected, output warning and switch to 'extended' + * If "on-the-fly" and a Neural Network mask is selected, warn and switch to 'extended' * If a mask-type is selected, ensure it exists in the alignments file. * If a predicted mask-type is selected, ensure model has been trained with a mask otherwise attempt to select first available masks, otherwise raise error. @@ -750,7 +750,7 @@ def _load_model(self): logger.debug("Loading Model") model_dir = get_folder(self._args.model_dir, make_folder=False) if not model_dir: - raise FaceswapError("{} does not exist.".format(self._args.model_dir)) + raise FaceswapError(f"{self._args.model_dir} does not exist.") trainer = self._get_model_name(model_dir) model = PluginLoader.get_model(trainer)(model_dir, self._args, predict=True) model.build() diff --git a/scripts/extract.py b/scripts/extract.py index 968937f539..80dffaf601 100644 --- a/scripts/extract.py +++ b/scripts/extract.py @@ -38,8 +38,8 @@ class Extract(): # pylint:disable=too-few-public-methods def __init__(self, arguments): logger.debug("Initializing %s: (args: %s", self.__class__.__name__, arguments) self._args = arguments - self._output_dir = None if self._args.skip_saving_faces else str(get_folder( - self._args.output_dir)) + self._output_dir = None if self._args.skip_saving_faces else get_folder( + self._args.output_dir) logger.info("Output Directory: %s", self._args.output_dir) self._images = ImagesLoader(self._args.input_dir, fast_count=True) diff --git a/scripts/fsmedia.py b/scripts/fsmedia.py index bf9b3bfbf5..e721f47fd5 100644 --- a/scripts/fsmedia.py +++ b/scripts/fsmedia.py @@ -9,7 +9,6 @@ import logging import os import sys -from pathlib import Path import cv2 import numpy as np @@ -604,7 +603,7 @@ def _set_face_filter(f_type, f_args): logger.info("%s: %s", f_type.title(), f_args) filter_files = f_args if isinstance(f_args, list) else [f_args] - filter_files = list(filter(lambda fpath: Path(fpath).exists(), filter_files)) + filter_files = list(filter(lambda fpath: os.path.exists(fpath), filter_files)) if not filter_files: logger.warning("Face %s files were requested, but no files could be found. This " "filter will not be applied.", f_type) diff --git a/scripts/train.py b/scripts/train.py index 226befcdf5..1747f07d37 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -144,29 +144,29 @@ def _set_timelapse(self): "(--timelapse-input-A, --timelapse-input-B and " "--timelapse-output).") - timelapse_output = str(get_folder(self._args.timelapse_output)) + timelapse_output = get_folder(self._args.timelapse_output) for side in ("a", "b"): - folder = getattr(self._args, "timelapse_input_{}".format(side)) + folder = getattr(self._args, f"timelapse_input_{side}") if folder is not None and not os.path.isdir(folder): - raise FaceswapError("The Timelapse path '{}' does not exist".format(folder)) + raise FaceswapError(f"The Timelapse path '{folder}' does not exist") - training_folder = getattr(self._args, "input_{}".format(side)) + training_folder = getattr(self._args, f"input_{side}") if folder == training_folder: continue # Time-lapse folder is training folder filenames = [fname for fname in os.listdir(folder) if os.path.splitext(fname)[-1].lower() in _image_extensions] if not filenames: - raise FaceswapError("The Timelapse path '{}' does not contain any valid " - "images".format(folder)) + raise FaceswapError(f"The Timelapse path '{folder}' does not contain any valid " + "images") # Time-lapse images must appear in the training set, as we need access to alignment and # mask info. Check filenames are there to save failing much later in the process. training_images = [os.path.basename(img) for img in self._images[side]] if not all(img in training_images for img in filenames): - raise FaceswapError("All images in the Timelapse folder '{}' must exist in the " - "training folder '{}'".format(folder, training_folder)) + raise FaceswapError(f"All images in the Timelapse folder '{folder}' must exist in " + f"the training folder '{training_folder}'") kwargs = {"input_a": self._args.timelapse_input_a, "input_b": self._args.timelapse_input_b, @@ -260,7 +260,7 @@ def _load_model(self): The requested model plugin """ logger.debug("Loading Model") - model_dir = str(get_folder(self._args.model_dir)) + model_dir = get_folder(self._args.model_dir) model = PluginLoader.get_model(self._args.trainer)( model_dir, self._args, diff --git a/setup.py b/setup.py index fe36f55d23..6c88bc63d0 100755 --- a/setup.py +++ b/setup.py @@ -15,13 +15,15 @@ from pkg_resources import parse_requirements, Requirement INSTALL_FAILED = False -# Revisions of tensorflow GPU and cuda/cudnn requirements -TENSORFLOW_REQUIREMENTS = {">=2.2.0,<2.4.0": ["10.1", "7.6"]} +# Revisions of tensorflow GPU and cuda/cudnn requirements. These relate specifically to the +# Tensorflow builds available from pypi +TENSORFLOW_REQUIREMENTS = {">=2.2.0,<2.4.0": ["10.1", "7.6"], + ">=2.4.0,<2.5.0": ["11.0", "8.0"], + ">=2.5.0,<2.7.0": ["11.2", "8.1"]} # Mapping of Python packages to their conda names if different from pip or in non-default channel CONDA_MAPPING = { # "opencv-python": ("opencv", "conda-forge"), # Periodic issues with conda-forge opencv "fastcluster": ("fastcluster", "conda-forge"), - "toposort": ("toposort", "conda-forge"), "imageio-ffmpeg": ("imageio-ffmpeg", "conda-forge")} @@ -692,6 +694,7 @@ def install_missing_dep(self): def install_python_packages(self): """ Install required pip packages """ self.output.info("Installing Required Python Packages. This may take some time...") + conda_only = False for pkg, version in self.env.missing_packages: if self.env.is_conda: pkg = CONDA_MAPPING.get(pkg, (pkg, None)) @@ -700,11 +703,26 @@ def install_python_packages(self): if version: pkg = "{}{}".format(pkg, ",".join("".join(spec) for spec in version)) if self.env.is_conda and not pkg.startswith("git"): + if pkg.startswith("tensorflow-gpu"): + # From TF 2.4 onwards, Anaconda Tensorflow becomes a mess. The version of 2.5 + # installed by Anaconda is compiled against an incorrect numpy version which + # breaks Tensorflow. Coupled with this the versions of cudatoolkit and cudnn + # available in the default Anaconda channel are not compatible with the + # official PyPi versions of Tensorflow. With this in mind we will pull in the + # required Cuda/cuDNN from conda-forge, and install Tensorflow with pip + # TODO Revert to Conda if they get their act together + + # Rewrite tensorflow requirement to versions from highest available cuda/cudnn + highest_cuda = sorted(TENSORFLOW_REQUIREMENTS.values())[-1] + compat_tf = next(k for k, v in TENSORFLOW_REQUIREMENTS.items() + if v == highest_cuda) + pkg = f"tensorflow-gpu{compat_tf}" + conda_only = True + verbose = pkg.startswith("tensorflow") or self.env.updater - if self.conda_installer(pkg, verbose=verbose, channel=channel, conda_only=False): + if self.conda_installer(pkg, + verbose=verbose, channel=channel, conda_only=conda_only): continue - if pkg.startswith("tensorflow-gpu"): - self._tensorflow_dependency_install() self.pip_installer(pkg) def install_conda_packages(self): @@ -717,7 +735,6 @@ def install_conda_packages(self): def conda_installer(self, package, channel=None, verbose=False, conda_only=False): """ Install a conda package """ # Packages with special characters need to be enclosed in double quotes - cuda_cudnn = None success = True condaexe = ["conda", "install", "-y"] if not verbose or self.env.updater: @@ -725,26 +742,24 @@ def conda_installer(self, package, channel=None, verbose=False, conda_only=False if channel: condaexe.extend(["-c", channel]) - # Windows TF2.3 doesn't pull in the Cuda toolkit, so we may as well be explicit - # TODO This is not a robust enough check if we have more than 1 tf version - if package.startswith("tensorflow-gpu"): # Add toolkit - # TODO Remove this hack to lower the max supported TF version when TF2.4 can be - # installed by setup.py - package = package.replace("2.5.0", "2.4.0") + if package.startswith("tensorflow-gpu"): + # Here we will install the cuda/cudnn toolkits, currently only available from + # conda-forge, but fail tensorflow itself so that it can be handled by pip. specs = Requirement.parse(package).specs for key, val in TENSORFLOW_REQUIREMENTS.items(): req_specs = Requirement.parse("foobar" + key).specs if all(item in req_specs for item in specs): - cuda_cudnn = val + cuda, cudnn = val break + condaexe.extend(["-c", "conda-forge", f"cudatoolkit={cuda}", f"cudnn={cudnn}"]) + package = "Cuda Toolkit" + success = False - if any(char in package for char in (" ", "<", ">", "*", "|")): - package = f"\"{package}\"" - condaexe.append(package) + if package != "Cuda Toolkit": + if any(char in package for char in (" ", "<", ">", "*", "|")): + package = f"\"{package}\"" + condaexe.append(package) - if cuda_cudnn is not None: - condaexe.extend([f"cudatoolkit={cuda_cudnn[0]}", - f"cudnn={cuda_cudnn[1]}"]) self.output.info("Installing {}".format(package.replace("\"", ""))) shell = self.env.os_version[0] == "Windows" try: diff --git a/tools/manual/detected_faces.py b/tools/manual/detected_faces.py index c6fb1d9763..5b31f70248 100644 --- a/tools/manual/detected_faces.py +++ b/tools/manual/detected_faces.py @@ -394,7 +394,7 @@ def _background_extract(self, output_folder, progress_queue): progress_queue: :class:`queue.Queue` The queue to place incremental counts to for updating the GUI's progress bar """ - _io = dict(saver=ImagesSaver(str(get_folder(output_folder)), as_bytes=True), + _io = dict(saver=ImagesSaver(get_folder(output_folder), as_bytes=True), loader=ImagesLoader(self._input_location, count=self._alignments.frames_count)) for frame_idx, (filename, image) in enumerate(_io["loader"].load()): diff --git a/tools/mask/mask.py b/tools/mask/mask.py index a4d406984d..6dbb2ac197 100644 --- a/tools/mask/mask.py +++ b/tools/mask/mask.py @@ -94,7 +94,7 @@ def _set_saver(self, arguments): sys.exit(0) logger.debug("No output provided. Not creating saver") return None - output_dir = str(get_folder(arguments.output, make_folder=True)) + output_dir = get_folder(arguments.output, make_folder=True) logger.info("Saving preview masks to: '%s'", output_dir) saver = ImagesSaver(output_dir) logger.debug(saver) @@ -404,7 +404,7 @@ def _save(self, frame, idx, detected_face): for mask_type in mask_types: if mask_type not in detected_face.mask: - # If extracting bisenet-fp mask, then skip versions which don't exist + # If extracting bisenet mask, then skip versions which don't exist continue filename = os.path.join(self._saver.location, "{}_{}_{}".format( os.path.splitext(frame)[0],