From ef03be17064a7653fef41f62c7cd789596dfec8a Mon Sep 17 00:00:00 2001 From: torzdf <36920800+torzdf@users.noreply.github.com> Date: Tue, 10 Dec 2019 02:01:20 +0000 Subject: [PATCH] Update Dependencies (#950) * 1st Round update for Python 3.7, TF1.15, Keras2.3 Move Tensorflow logging verbosity prior to first tensorflow import Keras Optimizers and nn_block update lib.logger - Change tf deprecation messages from WARNING to DEBUG Raise Tensorflow Max version check to 1.15 Update requirements and conda check for python 3.7+ Update install scripts, travis and documentation to Python 3.7 * Revert Keras to 2.2.4 --- .install/linux/faceswap_setup_x64.sh | 4 ++-- .install/windows/install.nsi | 2 +- .travis.yml | 2 +- INSTALL.md | 4 ++-- lib/cli.py | 5 +++-- lib/logger.py | 10 ++++++++++ lib/sysinfo.py | 3 ++- lib/vgg_face2_keras.py | 3 +-- requirements.txt | 12 ++++++------ scripts/convert.py | 1 - scripts/extract.py | 1 - scripts/fsmedia.py | 7 +------ scripts/gui.py | 2 -- scripts/train.py | 3 +-- setup.py | 11 ++++++----- tools/alignments.py | 3 --- tools/mask.py | 3 +-- tools/preview.py | 3 +-- 18 files changed, 38 insertions(+), 41 deletions(-) diff --git a/.install/linux/faceswap_setup_x64.sh b/.install/linux/faceswap_setup_x64.sh index 84ebca4ada..cf9894a872 100644 --- a/.install/linux/faceswap_setup_x64.sh +++ b/.install/linux/faceswap_setup_x64.sh @@ -336,10 +336,10 @@ delete_env() { } create_env() { - # Create Python 3.6 env for faceswap + # Create Python 3.7 env for faceswap delete_env info "Creating Conda Virtual Environment..." - yellow ; "$CONDA_EXECUTABLE" create -n "$ENV_NAME" -q python=3.6 -y + yellow ; "$CONDA_EXECUTABLE" create -n "$ENV_NAME" -q python=3.7 -y } diff --git a/.install/windows/install.nsi b/.install/windows/install.nsi index f7ccabfed9..9d4b54c313 100644 --- a/.install/windows/install.nsi +++ b/.install/windows/install.nsi @@ -22,7 +22,7 @@ InstallDir $PROFILE\faceswap # Install cli flags !define flagsConda "/S /RegisterPython=0 /AddToPath=0 /D=$PROFILE\MiniConda3" !define flagsRepo "--depth 1 --no-single-branch ${wwwRepo}" -!define flagsEnv "-y python=3.6" +!define flagsEnv "-y python=3.7" # Folders Var ProgramData diff --git a/.travis.yml b/.travis.yml index dffbe98134..026d8b71b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ language: shell env: global: - - CONDA_PYTHON=3.6 + - CONDA_PYTHON=3.7 - CONDA_BLD_PATH=${HOME}/conda-bld os: diff --git a/INSTALL.md b/INSTALL.md index aeaf0c189d..1f6e16e180 100755 --- a/INSTALL.md +++ b/INSTALL.md @@ -93,8 +93,8 @@ Reboot your PC, so that everything you have just installed gets registered. - Select "Create" at the bottom - In the pop up: - Give it the name: faceswap - - **IMPORTANT**: Select python version 3.6 - - Hit "Create" (NB: This may take a while as it will need to download Python 3.6) + - **IMPORTANT**: Select python version 3.7 + - Hit "Create" (NB: This may take a while as it will need to download Python 3.7) ![Anaconda virtual env setup](https://i.imgur.com/59RHnLs.png) #### Entering your virtual environment diff --git a/lib/cli.py b/lib/cli.py index f8d312bd07..89310b1109 100644 --- a/lib/cli.py +++ b/lib/cli.py @@ -15,7 +15,7 @@ from lib.logger import crash_log, log_setup from lib.model.masks import get_available_masks, get_default_mask -from lib.utils import FaceswapError, get_backend, safe_shutdown +from lib.utils import FaceswapError, get_backend, safe_shutdown, set_system_verbosity from plugins.plugin_loader import PluginLoader logger = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -46,7 +46,7 @@ def import_script(self): def test_for_tf_version(): """ Check that the minimum required Tensorflow version is installed """ min_ver = 1.12 - max_ver = 1.14 + max_ver = 1.15 try: # Ensure tensorflow doesn't pin all threads to one core when using tf-mkl os.environ["KMP_AFFINITY"] = "disabled" @@ -113,6 +113,7 @@ def check_display(): def execute_script(self, arguments): """ Run the script for called command """ + set_system_verbosity(arguments.loglevel) is_gui = hasattr(arguments, "redirect_gui") and arguments.redirect_gui log_setup(arguments.loglevel, arguments.logfile, self.command, is_gui) logger.debug("Executing: %s. PID: %s", self.command, os.getpid()) diff --git a/lib/logger.py b/lib/logger.py index 80db322fcb..b567f3c2cb 100644 --- a/lib/logger.py +++ b/lib/logger.py @@ -42,6 +42,7 @@ class FaceswapFormatter(logging.Formatter): def format(self, record): record.message = record.getMessage() + record = self.rewrite_tf_deprecation(record) # strip newlines if "\n" in record.message or "\r" in record.message: record.message = record.message.replace("\n", "\\n").replace("\r", "\\r") @@ -64,6 +65,15 @@ def format(self, record): msg = msg + self.formatStack(record.stack_info) return msg + @staticmethod + def rewrite_tf_deprecation(record): + """ Change TF deprecation messages from WARNING to DEBUG """ + if record.levelno == 30 and (record.funcName == "_tfmw_add_deprecation_warning" or + record.module == "deprecation"): + record.levelno = 10 + record.levelname = "DEBUG" + return record + class RollingBuffer(collections.deque): """File-like that keeps a certain number of lines of text in memory.""" diff --git a/lib/sysinfo.py b/lib/sysinfo.py index 8b8af626d9..bf3cb3c847 100644 --- a/lib/sysinfo.py +++ b/lib/sysinfo.py @@ -44,7 +44,8 @@ def encoding(self): @property def is_conda(self): """ Boolean for whether in a conda environment """ - return "conda" in sys.version.lower() + return ("conda" in sys.version.lower() or + os.path.exists(os.path.join(sys.prefix, 'conda-meta'))) @property def is_linux(self): diff --git a/lib/vgg_face2_keras.py b/lib/vgg_face2_keras.py index 5dfbeaeac9..c80b1af3a0 100644 --- a/lib/vgg_face2_keras.py +++ b/lib/vgg_face2_keras.py @@ -9,7 +9,7 @@ import cv2 import numpy as np from fastcluster import linkage, linkage_vector -from lib.utils import GetModel, set_system_verbosity, FaceswapError +from lib.utils import GetModel, FaceswapError logger = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -41,7 +41,6 @@ class VGGFace2(): def __init__(self, backend="GPU", loglevel="INFO"): logger.debug("Initializing %s: (backend: %s, loglevel: %s)", self.__class__.__name__, backend, loglevel) - set_system_verbosity(loglevel) backend = backend.upper() git_model_id = 10 model_filename = ["vggface2_resnet50_v2.h5"] diff --git a/requirements.txt b/requirements.txt index 8399dd0153..ebf3c621dc 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,15 @@ tqdm psutil pathlib -numpy==1.16.2 -opencv-python==4.1.1.26 +numpy==1.17.4 +opencv-python==4.1.2.30 scikit-image -Pillow==6.1.0 +Pillow==6.2.1 scikit-learn toposort fastcluster -matplotlib==2.2.2 -imageio==2.5.0 +matplotlib==3.1.1 +imageio==2.6.1 imageio-ffmpeg ffmpy==0.2.2 # Revert back to nvidia-ml-py3 when windows/system32 patch is implemented @@ -18,7 +18,7 @@ git+https://github.com/deepfakes/nvidia-ml-py3.git h5py==2.9.0 Keras==2.2.4 pywin32 ; sys_platform == "win32" -pynvx==0.0.4 ; sys_platform == "darwin" +pynvx==1.0.0 ; sys_platform == "darwin" # tensorflow is included within the docker image. # If you are looking for dependencies for a manual install, diff --git a/scripts/convert.py b/scripts/convert.py index 47cc6dbe59..3affd740d4 100644 --- a/scripts/convert.py +++ b/scripts/convert.py @@ -34,7 +34,6 @@ class Convert(): def __init__(self, arguments): logger.debug("Initializing %s: (args: %s)", self.__class__.__name__, arguments) self.args = arguments - Utils.set_verbosity(self.args.loglevel) self.patch_threads = None self.images = Images(self.args) diff --git a/scripts/extract.py b/scripts/extract.py index bb9c50030c..b09dcfe9cd 100644 --- a/scripts/extract.py +++ b/scripts/extract.py @@ -38,7 +38,6 @@ class Extract(): def __init__(self, arguments): logger.debug("Initializing %s: (args: %s", self.__class__.__name__, arguments) self._args = arguments - Utils.set_verbosity(self._args.loglevel) self._output_dir = str(get_folder(self._args.output_dir)) diff --git a/scripts/fsmedia.py b/scripts/fsmedia.py index 0c5611b6bd..27da35ddb1 100644 --- a/scripts/fsmedia.py +++ b/scripts/fsmedia.py @@ -15,7 +15,7 @@ from lib.alignments import Alignments as AlignmentsBase from lib.face_filter import FaceFilter as FilterFunc from lib.image import count_frames, read_image -from lib.utils import (camel_case_split, get_image_paths, set_system_verbosity, _video_extensions) +from lib.utils import (camel_case_split, get_image_paths, _video_extensions) logger = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -24,11 +24,6 @@ class Utils(): """ Holds utility functions that are required by more than one media object """ - @staticmethod - def set_verbosity(loglevel): - """ Set the system output verbosity """ - set_system_verbosity(loglevel) - @staticmethod def finalize(images_found, num_faces_detected, verify_output): """ Finalize the image processing """ diff --git a/scripts/gui.py b/scripts/gui.py index 7bb28acc8e..1d39a2c38a 100644 --- a/scripts/gui.py +++ b/scripts/gui.py @@ -9,7 +9,6 @@ from lib.gui import (TaskBar, CliOptions, CommandNotebook, ConsoleOut, Session, DisplayNotebook, get_images, initialize_images, initialize_config, LastSession, MainMenuBar, ProcessWrapper, StatusBar) -from lib.utils import set_system_verbosity logger = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -208,7 +207,6 @@ def _confirm_close_on_running_task(self): class Gui(): # pylint: disable=too-few-public-methods """ The GUI process. """ def __init__(self, arguments): - set_system_verbosity(arguments.loglevel) self.root = FaceswapGui(arguments.debug) def process(self): diff --git a/scripts/train.py b/scripts/train.py index ad51d1c331..2679d3dc6a 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -15,7 +15,7 @@ from lib.image import read_image from lib.keypress import KBHit from lib.multithreading import MultiThread -from lib.utils import get_folder, get_image_paths, set_system_verbosity, deprecation_warning +from lib.utils import get_folder, get_image_paths, deprecation_warning from plugins.plugin_loader import PluginLoader logger = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -151,7 +151,6 @@ def process(self): deprecation_warning("`-nac`, ``--no-augment-color``", additional_info="This option will be available within training " "config settings (/config/train.ini).") - set_system_verbosity(self._args.loglevel) thread = self._start_thread() # from lib.queue_manager import queue_manager; queue_manager.debug_monitor(1) diff --git a/setup.py b/setup.py index 53bb3fd37e..ec6483eebe 100755 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ INSTALL_FAILED = False # Revisions of tensorflow-gpu and cuda/cudnn requirements TENSORFLOW_REQUIREMENTS = {"==1.12.0": ["9.0", "7.2"], - ">=1.13.1,<1.15": ["10.0", "7.4"]} # TF 2.0 Not currently supported + ">=1.13.1,<1.16": ["10.0", "7.4"]} # TF 2.0 Not currently supported # Mapping of Python packages to their conda names if different from pypi or in non-default channel CONDA_MAPPING = { # "opencv-python": ("opencv", "conda-forge"), # Periodic issues with conda-forge opencv @@ -74,7 +74,8 @@ def py_version(self): @property def is_conda(self): """ Check whether using Conda """ - return bool("conda" in sys.version.lower()) + return ("conda" in sys.version.lower() or + os.path.exists(os.path.join(sys.prefix, 'conda-meta'))) @property def ld_library_path(self): @@ -220,7 +221,7 @@ def update_tf_dep(self): return if not self.enable_cuda: - self.required_packages.append("tensorflow==1.14.0") + self.required_packages.append("tensorflow==1.15.0") return tf_ver = None @@ -266,9 +267,9 @@ def update_tf_dep(self): def update_tf_dep_conda(self): """ Update Conda TF Dependency """ if not self.enable_cuda: - self.required_packages.append("tensorflow==1.14.0") + self.required_packages.append("tensorflow==1.15.0") else: - self.required_packages.append("tensorflow-gpu==1.14.0") + self.required_packages.append("tensorflow-gpu==1.15.0") def update_amd_dep(self): """ Update amd dependency for AMD cards """ diff --git a/tools/alignments.py b/tools/alignments.py index b5cde21157..b7ef64007e 100644 --- a/tools/alignments.py +++ b/tools/alignments.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 """ Tools for manipulating the alignments seralized file """ import logging - -from lib.utils import set_system_verbosity from .lib_alignments import (AlignmentData, Check, Dfl, Draw, # noqa pylint: disable=unused-import Extract, Fix, Manual, Merge, Rename, RemoveAlignments, Sort, Spatial, UpdateHashes) @@ -15,7 +13,6 @@ class Alignments(): def __init__(self, arguments): logger.debug("Initializing %s: (arguments: '%s'", self.__class__.__name__, arguments) self.args = arguments - set_system_verbosity(self.args.loglevel) self.alignments = self.load_alignments() logger.debug("Initialized %s", self.__class__.__name__) diff --git a/tools/mask.py b/tools/mask.py index 61a657ad85..550cad1858 100644 --- a/tools/mask.py +++ b/tools/mask.py @@ -12,7 +12,7 @@ from lib.image import ImagesLoader, ImagesSaver from lib.multithreading import MultiThread -from lib.utils import set_system_verbosity, get_folder +from lib.utils import get_folder from plugins.extract.pipeline import Extractor, ExtractMedia @@ -33,7 +33,6 @@ class Mask(): """ def __init__(self, arguments): logger.debug("Initializing %s: (arguments: %s", self.__class__.__name__, arguments) - set_system_verbosity(arguments.loglevel) self._update_type = arguments.processing self._input_is_faces = arguments.input_type == "faces" self._mask_type = arguments.masker diff --git a/tools/preview.py b/tools/preview.py index 5c05eb9417..2c25ba75c2 100644 --- a/tools/preview.py +++ b/tools/preview.py @@ -24,7 +24,7 @@ from lib.faces_detect import DetectedFace from lib.model.masks import get_available_masks from lib.multithreading import MultiThread -from lib.utils import FaceswapError, set_system_verbosity +from lib.utils import FaceswapError from lib.queue_manager import queue_manager from scripts.fsmedia import Alignments, Images from scripts.convert import Predict @@ -42,7 +42,6 @@ class Preview(): def __init__(self, arguments): logger.debug("Initializing %s: (arguments: '%s'", self.__class__.__name__, arguments) - set_system_verbosity(arguments.loglevel) self.config_tools = ConfigTools() self.lock = Lock() self.trigger_patch = Event()