Tensorflow 2.6 Support (deepfakes#1182)

* lib.cli.launcher - Bump max tf version to 2.6

* Remove pathlib requirement

* Update requirements files

* Update setup.py

* bugfix - GUI: Supress errors when attempting to load previews in extract

* GUI: Suppress ptxas error messages for Windows

INSTALL.md

@@ -257,7 +257,6 @@ WARNING Tensorflow has no official prebuild for CUDA 9.1 currently.
 Are System Dependencies met? [y/N] y
 INFO    Installing Missing Python Packages...
 INFO    Installing tensorflow-gpu
-INFO    Installing pathlib==1.0.1
 ......
 INFO    Installing tqdm
 INFO    Installing matplotlib

_requirements_base.txt

@@ -1,18 +1,15 @@
-tqdm>=4.42
-psutil>=5.7.0
-pathlib==1.0.1
+tqdm>=4.62
+psutil>=5.8.0
 numpy>=1.18.0,<1.20.0
-opencv-python>=4.1.2.0
-pillow>=7.0.0
-scikit-learn>=0.22.0
-fastcluster==1.1.26
+opencv-python>=4.5.3.0
+pillow>=8.3.1
+scikit-learn>=0.24.2
+fastcluster>=1.1.26
 # matplotlib 3.3.1 breaks custom toolbar in graph popup
-matplotlib>=3.0.3,<3.3.0
-imageio>=2.8.0
-imageio-ffmpeg>=0.4.2
+matplotlib>=3.2.0,<3.3.0
+imageio>=2.9.0
+imageio-ffmpeg>=0.4.5
 ffmpy==0.2.3
-# Revert back to nvidia-ml-py3 when windows/system32 patch is implemented
-git+https://github.com/deepfakes/nvidia-ml-py3.git
-#nvidia-ml-py3
-pywin32>=227 ; sys_platform == "win32"
+nvidia-ml-py>=11.470.66
+pywin32>=228 ; sys_platform == "win32"
 pynvx==1.0.0 ; sys_platform == "darwin"

docs/sphinx_requirements.txt

@@ -1,20 +1,19 @@
 # NB Do not install from this requirements file
 # It is for documentation purposes only
 
-tqdm==4.42
-psutil==5.7.0
-pathlib==1.0.1
+tqdm==4.62
+psutil==5.8.0
 numpy==1.18.0
-opencv-python==4.1.2.30
-pillow==7.0.0
-scikit-learn==0.22.0
+opencv-python==4.5.3.0
+pillow==8.3.1
+scikit-learn==0.24.2
 fastcluster==1.1.26
-matplotlib>3.0.3,<3.3.0
-imageio==2.8.0
-imageio-ffmpeg==0.4.2
+matplotlib>3.2.0,<3.3.0
+imageio==2.9.0
+imageio-ffmpeg==0.4.5
 ffmpy==0.2.3
 nvidia-ml-py3
-pywin32==227 ; sys_platform == "win32"
+pywin32==228 ; sys_platform == "win32"
 pynvx==1.0.0 ; sys_platform == "darwin"
 plaidml-keras==0.7.0
 tensorflow==2.2.0

lib/cli/launcher.py

@@ -53,10 +53,10 @@ def _test_for_tf_version(self):
         Raises
         ------
         FaceswapError
-            If Tensorflow is not found, or is not between versions 2.2 and 2.4
+            If Tensorflow is not found, or is not between versions 2.2 and 2.6
         """
         min_ver = 2.2
-        max_ver = 2.4
+        max_ver = 2.6
         try:
             # Ensure tensorflow doesn't pin all threads to one core when using Math Kernel Library
             os.environ["TF_MIN_GPU_MULTIPROCESSOR_COUNT"] = "4"
@@ -65,15 +65,16 @@ def _test_for_tf_version(self):
         except ImportError as err:
             if "DLL load failed while importing" in str(err):
                 msg = (
-                    "A DLL library file failed to load. Make sure that you have Microsoft Visual "
+                    f"A DLL library file failed to load. Make sure that you have Microsoft Visual "
                     "C++ Redistributable (2015, 2017, 2019) installed for your machine from: "
-                    "https://support.microsoft.com/en-gb/help/2977003")
+                    "https://support.microsoft.com/en-gb/help/2977003. Original error: "
+                    f"{str(err)}")
             else:
                 msg = (
-                    "There was an error importing Tensorflow. This is most likely because you do "
+                    f"There was an error importing Tensorflow. This is most likely because you do "
                     "not have TensorFlow installed, or you are trying to run tensorflow-gpu on a "
                     "system without an Nvidia graphics card. Original import "
-                    "error: {}".format(str(err)))
+                    f"error: {str(err)}")
             self._handle_import_error(msg)
 
         tf_ver = float(".".join(tf.__version__.split(".")[:2]))  # pylint:disable=no-member
@@ -126,9 +127,8 @@ def _test_tkinter():
             If tkinter cannot be imported
         """
         try:
-            # pylint: disable=unused-variable
             import tkinter  # noqa pylint: disable=unused-import,import-outside-toplevel
-        except ImportError:
+        except ImportError as err:
             logger.error("It looks like TkInter isn't installed for your OS, so the GUI has been "
                          "disabled. To enable the GUI please install the TkInter application. You "
                          "can try:")
@@ -139,7 +139,7 @@ def _test_tkinter():
             logger.info("Arch: sudo pacman -S tk")
             logger.info("CentOS/Redhat: sudo yum install tkinter")
             logger.info("Fedora: sudo dnf install python3-tkinter")
-            raise FaceswapError("TkInter not found")
+            raise FaceswapError("TkInter not found") from err
 
     @staticmethod
     def _check_display():

lib/gui/utils.py

@@ -612,6 +612,14 @@ def _load_images_to_cache(self, image_files, frame_dims, thumbnail_size):
                              fname, str(err))
                 dropped_files.append(fname)
                 continue
+            except Exception as err:  # pylint:disable=broad-except
+                # Swallow any issues with opening an image rather than spamming console
+                # Can happen when trying to read partially saved images
+                logger.debug("Error opening preview file: '%s'. Original error: %s",
+                             fname, str(err))
+                dropped_files.append(fname)
+                continue
+
             width, height = img.size
             scaling = thumbnail_size / max(width, height)
             logger.debug("image width: %s, height: %s, scaling: %s", width, height, scaling)

lib/gui/wrapper.py

@@ -245,6 +245,9 @@ def read_stderr(self):
                 if self.command == "train" and output.startswith("Reading training images"):
                     print(output.strip(), file=sys.stdout)
                     continue
+                if os.name == "nt" and "Call to CreateProcess failed. Error code: 2" in output:
+                    # Suppress ptxas errors on Tensorflow for Windows
+                    logger.debug("Suppressed call to subprocess error: '%s'", output)
                 print(output.strip(), file=sys.stderr)
         logger.debug("Terminated stderr reader")
 

lib/model/backup_restore.py

@@ -105,7 +105,7 @@ def snapshot_models(self, iterations):
             logger.debug("Removing previously existing snapshot folder: '%s'", snapshot_dir)
             rmtree(snapshot_dir)
 
-        dst = str(get_folder(snapshot_dir))
+        dst = get_folder(snapshot_dir)
         for filename in os.listdir(self.model_dir):
             if not self._check_valid(filename, for_restore=False):
                 logger.debug("Not snapshotting file: '%s'", filename)

lib/utils.py

@@ -10,7 +10,6 @@
 import warnings
 import zipfile
 
-from pathlib import Path
 from re import finditer
 from multiprocessing import current_process
 from socket import timeout as socket_timeout, error as socket_error
@@ -146,19 +145,18 @@ def get_folder(path, make_folder=True):
 
     Returns
     -------
-    :class:`pathlib.Path` or `None`
+    str or `None`
         The path to the requested folder. If `make_folder` is set to ``False`` and the requested
         path does not exist, then ``None`` is returned
     """
     logger = logging.getLogger(__name__)  # pylint:disable=invalid-name
     logger.debug("Requested path: '%s'", path)
-    output_dir = Path(path)
-    if not make_folder and not output_dir.exists():
+    if not make_folder and not os.path.isdir(path):
         logger.debug("%s does not exist", path)
         return None
-    output_dir.mkdir(parents=True, exist_ok=True)
-    logger.debug("Returning: '%s'", output_dir)
-    return output_dir
+    os.makedirs(path, exist_ok=True)
+    logger.debug("Returning: '%s'", path)
+    return path
 
 
 def get_image_paths(directory, extension=None):
@@ -189,8 +187,7 @@ def get_image_paths(directory, extension=None):
     logger.trace("Scanned Folder Contents: %s", dir_scanned)
 
     for chkfile in dir_scanned:
-        if any([chkfile.name.lower().endswith(ext)
-                for ext in image_extensions]):
+        if any(chkfile.name.lower().endswith(ext) for ext in image_extensions):
             logger.trace("Adding '%s' to image list", chkfile.path)
             dir_contents.append(chkfile.path)
 

plugins/train/trainer/_base.py

@@ -945,7 +945,7 @@ def _duplicate_headers(cls, headers, columns):
             The original headers duplicated by the number of columns
         """
         for side, header in headers.items():
-            duped = tuple([header for _ in range(columns)])
+            duped = tuple(header for _ in range(columns))
             headers[side] = np.concatenate(duped, axis=1)
             logger.debug("side: %s header.shape: %s", side, header.shape)
         return headers
@@ -996,8 +996,8 @@ def _setup(self, input_a=None, input_b=None, output=None):
         """
         logger.debug("Setting up time-lapse")
         if output is None:
-            output = str(get_folder(os.path.join(str(self._model.model_dir),
-                                                 "{}_timelapse".format(self._model.name))))
+            output = get_folder(os.path.join(str(self._model.model_dir),
+                                             f"{self._model.name}_timelapse"))
         self._output_file = str(output)
         logger.debug("Time-lapse output set to '%s'", self._output_file)
 

requirements_cpu.txt

@@ -1,2 +1,2 @@
 -r _requirements_base.txt
-tensorflow>=2.2.0,<2.5.0
+tensorflow>=2.2.0,<2.7.0

requirements_nvidia.txt

@@ -1,2 +1,2 @@
 -r _requirements_base.txt
-tensorflow-gpu>=2.2.0,<2.5.0
+tensorflow-gpu>=2.2.0,<2.7.0

scripts/convert.py

@@ -107,7 +107,7 @@ def _validate(self):
         Ensure that certain cli selections are valid and won't result in an error. Checks:
             * If frames have been passed in with video output, ensure user supplies reference
             video.
-            * If "on-the-fly" and an NN mask is selected, output warning and switch to 'extended'
+            * If "on-the-fly" and a Neural Network mask is selected, warn and switch to 'extended'
             * If a mask-type is selected, ensure it exists in the alignments file.
             * If a predicted mask-type is selected, ensure model has been trained with a mask
             otherwise attempt to select first available masks, otherwise raise error.
@@ -750,7 +750,7 @@ def _load_model(self):
         logger.debug("Loading Model")
         model_dir = get_folder(self._args.model_dir, make_folder=False)
         if not model_dir:
-            raise FaceswapError("{} does not exist.".format(self._args.model_dir))
+            raise FaceswapError(f"{self._args.model_dir} does not exist.")
         trainer = self._get_model_name(model_dir)
         model = PluginLoader.get_model(trainer)(model_dir, self._args, predict=True)
         model.build()

scripts/extract.py

@@ -38,8 +38,8 @@ class Extract():  # pylint:disable=too-few-public-methods
     def __init__(self, arguments):
         logger.debug("Initializing %s: (args: %s", self.__class__.__name__, arguments)
         self._args = arguments
-        self._output_dir = None if self._args.skip_saving_faces else str(get_folder(
-            self._args.output_dir))
+        self._output_dir = None if self._args.skip_saving_faces else get_folder(
+            self._args.output_dir)
 
         logger.info("Output Directory: %s", self._args.output_dir)
         self._images = ImagesLoader(self._args.input_dir, fast_count=True)

scripts/fsmedia.py

@@ -9,7 +9,6 @@
 import logging
 import os
 import sys
-from pathlib import Path
 
 import cv2
 import numpy as np
@@ -604,7 +603,7 @@ def _set_face_filter(f_type, f_args):
 
         logger.info("%s: %s", f_type.title(), f_args)
         filter_files = f_args if isinstance(f_args, list) else [f_args]
-        filter_files = list(filter(lambda fpath: Path(fpath).exists(), filter_files))
+        filter_files = list(filter(lambda fpath: os.path.exists(fpath), filter_files))
         if not filter_files:
             logger.warning("Face %s files were requested, but no files could be found. This "
                            "filter will not be applied.", f_type)

scripts/train.py

@@ -144,29 +144,29 @@ def _set_timelapse(self):
                                 "(--timelapse-input-A, --timelapse-input-B and "
                                 "--timelapse-output).")
 
-        timelapse_output = str(get_folder(self._args.timelapse_output))
+        timelapse_output = get_folder(self._args.timelapse_output)
 
         for side in ("a", "b"):
-            folder = getattr(self._args, "timelapse_input_{}".format(side))
+            folder = getattr(self._args, f"timelapse_input_{side}")
             if folder is not None and not os.path.isdir(folder):
-                raise FaceswapError("The Timelapse path '{}' does not exist".format(folder))
+                raise FaceswapError(f"The Timelapse path '{folder}' does not exist")
 
-            training_folder = getattr(self._args, "input_{}".format(side))
+            training_folder = getattr(self._args, f"input_{side}")
             if folder == training_folder:
                 continue  # Time-lapse folder is training folder
 
             filenames = [fname for fname in os.listdir(folder)
                          if os.path.splitext(fname)[-1].lower() in _image_extensions]
             if not filenames:
-                raise FaceswapError("The Timelapse path '{}' does not contain any valid "
-                                    "images".format(folder))
+                raise FaceswapError(f"The Timelapse path '{folder}' does not contain any valid "
+                                    "images")
 
             # Time-lapse images must appear in the training set, as we need access to alignment and
             # mask info. Check filenames are there to save failing much later in the process.
             training_images = [os.path.basename(img) for img in self._images[side]]
             if not all(img in training_images for img in filenames):
-                raise FaceswapError("All images in the Timelapse folder '{}' must exist in the "
-                                    "training folder '{}'".format(folder, training_folder))
+                raise FaceswapError(f"All images in the Timelapse folder '{folder}' must exist in "
+                                    f"the training folder '{training_folder}'")
 
         kwargs = {"input_a": self._args.timelapse_input_a,
                   "input_b": self._args.timelapse_input_b,
@@ -260,7 +260,7 @@ def _load_model(self):
             The requested model plugin
         """
         logger.debug("Loading Model")
-        model_dir = str(get_folder(self._args.model_dir))
+        model_dir = get_folder(self._args.model_dir)
         model = PluginLoader.get_model(self._args.trainer)(
             model_dir,
             self._args,