diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 339179c7..fd870e87 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ default_language_version: repos: - repo: "https://github.com/asottile/pyupgrade" - rev: "v3.16.0" + rev: "v3.17.0" hooks: - id: "pyupgrade" name: "Enforce Python 3.7+ idioms" @@ -11,7 +11,7 @@ repos: - "--py37-plus" - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.5.4" + rev: "v0.5.7" hooks: - id: ruff args: ["--fix"] @@ -46,7 +46,7 @@ repos: language: python files: \.py$ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.11.0 + rev: v1.11.1 hooks: - id: mypy exclude: (docs|pyfakefs/tests) diff --git a/CHANGES.md b/CHANGES.md index 53ed438f..f50c5aeb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,12 @@ The released versions correspond to PyPI releases. * the default for `FakeFilesystem.shuffle_listdir_results` will change to `True` to reflect the real filesystem behavior +## Unreleased + +### Enhancements + +- refactor the implementation of `additional_skip_names` parameter to make it work with more modules (see [#1023](../../issues/1023)) + ## [Version 5.6.0](https://pypi.python.org/pypi/pyfakefs/5.6.0) (2024-07-12) Adds preliminary Python 3.13 support. diff --git a/pyfakefs/fake_file.py b/pyfakefs/fake_file.py index 200014c9..5be15984 100644 --- a/pyfakefs/fake_file.py +++ b/pyfakefs/fake_file.py @@ -603,8 +603,8 @@ def remove_entry(self, pathname_name: str, recursive: bool = True) -> None: st = traceback.extract_stack(limit=6) if sys.version_info < (3, 10): if ( - st[1].name == "TemporaryFile" - and st[1].line == "_os.unlink(name)" + st[0].name == "TemporaryFile" + and st[0].line == "_os.unlink(name)" ): raise_error = False else: diff --git a/pyfakefs/fake_filesystem_unittest.py b/pyfakefs/fake_filesystem_unittest.py index 1fc9e475..ac53b6c6 100644 --- a/pyfakefs/fake_filesystem_unittest.py +++ b/pyfakefs/fake_filesystem_unittest.py @@ -570,7 +570,7 @@ def __init__( set_uid(1) set_gid(1) - self._skip_names = self.SKIPNAMES.copy() + self.skip_names = self.SKIPNAMES.copy() # save the original open function for use in pytest plugin self.original_open = open self.patch_open_code = patch_open_code @@ -582,7 +582,7 @@ def __init__( cast(ModuleType, m).__name__ if inspect.ismodule(m) else cast(str, m) for m in additional_skip_names ] - self._skip_names.update(skip_names) + self.skip_names.update(skip_names) self._fake_module_classes: Dict[str, Any] = {} self._unfaked_module_classes: Dict[str, Any] = {} @@ -628,8 +628,8 @@ def __init__( if patched_module_names != self.PATCHED_MODULE_NAMES: self.__class__.PATCHED_MODULE_NAMES = patched_module_names clear_cache = True - if self._skip_names != self.ADDITIONAL_SKIP_NAMES: - self.__class__.ADDITIONAL_SKIP_NAMES = self._skip_names + if self.skip_names != self.ADDITIONAL_SKIP_NAMES: + self.__class__.ADDITIONAL_SKIP_NAMES = self.skip_names clear_cache = True if patch_default_args != self.PATCH_DEFAULT_ARGS: self.__class__.PATCH_DEFAULT_ARGS = patch_default_args @@ -875,7 +875,7 @@ def _find_modules(self) -> None: pass continue skipped = module in self.SKIPMODULES or any( - [sn.startswith(module.__name__) for sn in self._skip_names] + [sn.startswith(module.__name__) for sn in self.skip_names] ) module_items = module.__dict__.copy().items() @@ -922,7 +922,7 @@ def _refresh(self) -> None: for name in self._fake_module_classes: self.fake_modules[name] = self._fake_module_classes[name](self.fs) if hasattr(self.fake_modules[name], "skip_names"): - self.fake_modules[name].skip_names = self._skip_names + self.fake_modules[name].skip_names = self.skip_names self.fake_modules[PATH_MODULE] = self.fake_modules["os"].path for name in self._unfaked_module_classes: self.unfaked_modules[name] = self._unfaked_module_classes[name]() diff --git a/pyfakefs/fake_io.py b/pyfakefs/fake_io.py index 116153d9..0c756a5d 100644 --- a/pyfakefs/fake_io.py +++ b/pyfakefs/fake_io.py @@ -33,7 +33,7 @@ from pyfakefs.fake_file import AnyFileWrapper from pyfakefs.fake_open import fake_open -from pyfakefs.helpers import IS_PYPY +from pyfakefs.helpers import IS_PYPY, is_called_from_skipped_module if TYPE_CHECKING: from pyfakefs.fake_filesystem import FakeFilesystem @@ -180,6 +180,16 @@ def lockf( ) -> Any: pass - def __getattr__(self, name): + def __getattribute__(self, name): """Forwards any unfaked calls to the standard fcntl module.""" - return getattr(self._fcntl_module, name) + fs: FakeFilesystem = object.__getattribute__(self, "filesystem") + fnctl_module = object.__getattribute__(self, "_fcntl_module") + if fs.patcher: + if is_called_from_skipped_module( + skip_names=fs.patcher.skip_names, + case_sensitive=fs.is_case_sensitive, + ): + # remove the `self` argument for FakeOsModule methods + return getattr(fnctl_module, name) + + return object.__getattribute__(self, name) diff --git a/pyfakefs/fake_open.py b/pyfakefs/fake_open.py index ef60d5b4..46cb1e99 100644 --- a/pyfakefs/fake_open.py +++ b/pyfakefs/fake_open.py @@ -18,7 +18,6 @@ import io import os import sys -import traceback from stat import ( S_ISDIR, ) @@ -44,6 +43,7 @@ ) from pyfakefs.helpers import ( AnyString, + is_called_from_skipped_module, is_root, PERM_READ, PERM_WRITE, @@ -86,40 +86,9 @@ def fake_open( """Redirect the call to FakeFileOpen. See FakeFileOpen.call() for description. """ - - # workaround for built-in open called from skipped modules (see #552) - # as open is not imported explicitly, we cannot patch it for - # specific modules; instead we check if the caller is a skipped - # module (should work in most cases) - stack = traceback.extract_stack(limit=3) - - # handle the case that we try to call the original `open_code` - # and get here instead (since Python 3.12) - # TODO: use a more generic approach (see PR #1025) - if sys.version_info >= (3, 12): - from_open_code = ( - stack[0].name == "open_code" - and stack[0].line == "return self._io_module.open_code(path)" - ) - else: - from_open_code = False - - module_name = os.path.splitext(stack[0].filename)[0] - module_name = module_name.replace(os.sep, ".") - if sys.version_info >= (3, 13) and module_name.endswith( - ("pathlib._abc", "pathlib._local") - ): - stack = traceback.extract_stack(limit=6) - frame = 2 - # in Python 3.13, pathlib is implemented in 2 sub-modules that may call - # each other, so we have to look further in the stack - while frame >= 0 and module_name.endswith(("pathlib._abc", "pathlib._local")): - module_name = os.path.splitext(stack[frame].filename)[0] - module_name = module_name.replace(os.sep, ".") - frame -= 1 - - if from_open_code or any( - [module_name == sn or module_name.endswith("." + sn) for sn in skip_names] + if is_called_from_skipped_module( + skip_names=skip_names, + case_sensitive=filesystem.is_case_sensitive, ): return io_open( # pytype: disable=wrong-arg-count file, diff --git a/pyfakefs/fake_os.py b/pyfakefs/fake_os.py index c57de0d0..c2da5d02 100644 --- a/pyfakefs/fake_os.py +++ b/pyfakefs/fake_os.py @@ -54,6 +54,7 @@ from pyfakefs.fake_scandir import scandir, walk, ScanDirIter from pyfakefs.helpers import ( FakeStatResult, + is_called_from_skipped_module, is_int_type, is_byte_string, make_string_path, @@ -1409,27 +1410,40 @@ def __getattr__(self, name: str) -> Any: return getattr(self.os_module, name) -if sys.version_info > (3, 10): +def handle_original_call(f: Callable) -> Callable: + """Decorator used for real pathlib Path methods to ensure that + real os functions instead of faked ones are used. + Applied to all non-private methods of `FakeOsModule`.""" - def handle_original_call(f: Callable) -> Callable: - """Decorator used for real pathlib Path methods to ensure that - real os functions instead of faked ones are used. - Applied to all non-private methods of `FakeOsModule`.""" + @functools.wraps(f) + def wrapped(*args, **kwargs): + should_use_original = FakeOsModule.use_original - @functools.wraps(f) - def wrapped(*args, **kwargs): - if FakeOsModule.use_original: - # remove the `self` argument for FakeOsModule methods - if args and isinstance(args[0], FakeOsModule): - args = args[1:] - return getattr(os, f.__name__)(*args, **kwargs) - return f(*args, **kwargs) + if not should_use_original and args: + self = args[0] + fs: FakeFilesystem = self.filesystem + if self.filesystem.patcher: + skip_names = fs.patcher.skip_names + if is_called_from_skipped_module( + skip_names=skip_names, + case_sensitive=fs.is_case_sensitive, + ): + should_use_original = True - return wrapped + if should_use_original: + # remove the `self` argument for FakeOsModule methods + if args and isinstance(args[0], FakeOsModule): + args = args[1:] + return getattr(os, f.__name__)(*args, **kwargs) - for name, fn in inspect.getmembers(FakeOsModule, inspect.isfunction): - if not fn.__name__.startswith("_"): - setattr(FakeOsModule, name, handle_original_call(fn)) + return f(*args, **kwargs) + + return wrapped + + +for name, fn in inspect.getmembers(FakeOsModule, inspect.isfunction): + if not fn.__name__.startswith("_"): + setattr(FakeOsModule, name, handle_original_call(fn)) @contextmanager diff --git a/pyfakefs/fake_path.py b/pyfakefs/fake_path.py index 9c5d786c..013b9e7b 100644 --- a/pyfakefs/fake_path.py +++ b/pyfakefs/fake_path.py @@ -15,6 +15,8 @@ """Faked ``os.path`` module replacement. See ``fake_filesystem`` for usage.""" import errno +import functools +import inspect import os import sys from stat import ( @@ -23,6 +25,7 @@ ) from types import ModuleType from typing import ( + Callable, List, Optional, Union, @@ -36,6 +39,7 @@ ) from pyfakefs.helpers import ( + is_called_from_skipped_module, make_string_path, to_string, matching_string, @@ -553,3 +557,37 @@ def _isdir(self, path: AnyStr) -> bool: def __getattr__(self, name: str) -> Any: """Forwards any non-faked calls to the real nt module.""" return getattr(self.nt_module, name) + + +def handle_original_call(f: Callable) -> Callable: + """Decorator used for real pathlib Path methods to ensure that + real os functions instead of faked ones are used. + Applied to all non-private methods of `FakePathModule`.""" + + @functools.wraps(f) + def wrapped(*args, **kwargs): + if args: + self = args[0] + should_use_original = self.os.use_original + if not should_use_original and self.filesystem.patcher: + skip_names = self.filesystem.patcher.skip_names + if is_called_from_skipped_module( + skip_names=skip_names, + case_sensitive=self.filesystem.is_case_sensitive, + ): + should_use_original = True + + if should_use_original: + # remove the `self` argument for FakePathModule methods + if args and isinstance(args[0], FakePathModule): + args = args[1:] + return getattr(os.path, f.__name__)(*args, **kwargs) + + return f(*args, **kwargs) + + return wrapped + + +for name, fn in inspect.getmembers(FakePathModule, inspect.isfunction): + if not fn.__name__.startswith("_"): + setattr(FakePathModule, name, handle_original_call(fn)) diff --git a/pyfakefs/fake_pathlib.py b/pyfakefs/fake_pathlib.py index f56ee4fb..55942b8b 100644 --- a/pyfakefs/fake_pathlib.py +++ b/pyfakefs/fake_pathlib.py @@ -49,7 +49,7 @@ from pyfakefs.fake_filesystem import FakeFilesystem from pyfakefs.fake_open import fake_open from pyfakefs.fake_os import FakeOsModule, use_original_os -from pyfakefs.helpers import IS_PYPY +from pyfakefs.helpers import IS_PYPY, is_called_from_skipped_module def init_module(filesystem): @@ -93,26 +93,47 @@ def init_module(filesystem): setattr(FakePathlibModule.PureWindowsPath, parser_name, fake_pure_nt_os.path) -def _wrap_strfunc(strfunc): - @functools.wraps(strfunc) +def _wrap_strfunc(fake_fct, original_fct): + @functools.wraps(fake_fct) def _wrapped(pathobj, *args, **kwargs): - return strfunc(pathobj.filesystem, str(pathobj), *args, **kwargs) + fs: FakeFilesystem = pathobj.filesystem + if fs.patcher: + if is_called_from_skipped_module( + skip_names=fs.patcher.skip_names, + case_sensitive=fs.is_case_sensitive, + ): + return original_fct(str(pathobj), *args, **kwargs) + return fake_fct(fs, str(pathobj), *args, **kwargs) return staticmethod(_wrapped) -def _wrap_binary_strfunc(strfunc): - @functools.wraps(strfunc) +def _wrap_binary_strfunc(fake_fct, original_fct): + @functools.wraps(fake_fct) def _wrapped(pathobj1, pathobj2, *args): - return strfunc(pathobj1.filesystem, str(pathobj1), str(pathobj2), *args) + fs: FakeFilesystem = pathobj1.filesystem + if fs.patcher: + if is_called_from_skipped_module( + skip_names=fs.patcher.skip_names, + case_sensitive=fs.is_case_sensitive, + ): + return original_fct(str(pathobj1), str(pathobj2), *args) + return fake_fct(fs, str(pathobj1), str(pathobj2), *args) return staticmethod(_wrapped) -def _wrap_binary_strfunc_reverse(strfunc): - @functools.wraps(strfunc) +def _wrap_binary_strfunc_reverse(fake_fct, original_fct): + @functools.wraps(fake_fct) def _wrapped(pathobj1, pathobj2, *args): - return strfunc(pathobj2.filesystem, str(pathobj2), str(pathobj1), *args) + fs: FakeFilesystem = pathobj2.filesystem + if fs.patcher: + if is_called_from_skipped_module( + skip_names=fs.patcher.skip_names, + case_sensitive=fs.is_case_sensitive, + ): + return original_fct(str(pathobj2), str(pathobj1), *args) + return fake_fct(fs, str(pathobj2), str(pathobj1), *args) return staticmethod(_wrapped) @@ -128,20 +149,21 @@ class _FakeAccessor(accessor): # type: ignore[valid-type, misc] methods. """ - stat = _wrap_strfunc(FakeFilesystem.stat) + stat = _wrap_strfunc(FakeFilesystem.stat, os.stat) lstat = _wrap_strfunc( - lambda fs, path: FakeFilesystem.stat(fs, path, follow_symlinks=False) + lambda fs, path: FakeFilesystem.stat(fs, path, follow_symlinks=False), os.lstat ) - listdir = _wrap_strfunc(FakeFilesystem.listdir) - scandir = _wrap_strfunc(fake_scandir.scandir) + listdir = _wrap_strfunc(FakeFilesystem.listdir, os.listdir) + scandir = _wrap_strfunc(fake_scandir.scandir, os.scandir) if hasattr(os, "lchmod"): lchmod = _wrap_strfunc( lambda fs, path, mode: FakeFilesystem.chmod( fs, path, mode, follow_symlinks=False - ) + ), + os.lchmod, ) else: @@ -164,47 +186,51 @@ def chmod(self, pathobj, *args, **kwargs): ) return pathobj.filesystem.chmod(str(pathobj), *args, **kwargs) - mkdir = _wrap_strfunc(FakeFilesystem.makedir) + mkdir = _wrap_strfunc(FakeFilesystem.makedir, os.mkdir) - unlink = _wrap_strfunc(FakeFilesystem.remove) + unlink = _wrap_strfunc(FakeFilesystem.remove, os.unlink) - rmdir = _wrap_strfunc(FakeFilesystem.rmdir) + rmdir = _wrap_strfunc(FakeFilesystem.rmdir, os.rmdir) - rename = _wrap_binary_strfunc(FakeFilesystem.rename) + rename = _wrap_binary_strfunc(FakeFilesystem.rename, os.rename) replace = _wrap_binary_strfunc( lambda fs, old_path, new_path: FakeFilesystem.rename( fs, old_path, new_path, force_replace=True - ) + ), + os.replace, ) symlink = _wrap_binary_strfunc_reverse( lambda fs, fpath, target, target_is_dir: FakeFilesystem.create_symlink( fs, fpath, target, create_missing_dirs=False - ) + ), + os.symlink, ) if (3, 8) <= sys.version_info: link_to = _wrap_binary_strfunc( lambda fs, file_path, link_target: FakeFilesystem.link( fs, file_path, link_target - ) + ), + os.link, ) if sys.version_info >= (3, 10): link = _wrap_binary_strfunc( lambda fs, file_path, link_target: FakeFilesystem.link( fs, file_path, link_target - ) + ), + os.link, ) # this will use the fake filesystem because os is patched def getcwd(self): return os.getcwd() - readlink = _wrap_strfunc(FakeFilesystem.readlink) + readlink = _wrap_strfunc(FakeFilesystem.readlink, os.readlink) - utime = _wrap_strfunc(FakeFilesystem.utime) + utime = _wrap_strfunc(FakeFilesystem.utime, os.utime) _fake_accessor = _FakeAccessor() diff --git a/pyfakefs/helpers.py b/pyfakefs/helpers.py index cf84936d..674ce8fa 100644 --- a/pyfakefs/helpers.py +++ b/pyfakefs/helpers.py @@ -19,7 +19,9 @@ import platform import stat import sys +import sysconfig import time +import traceback from collections import namedtuple from copy import copy from stat import S_IFLNK @@ -39,6 +41,13 @@ PERM_DEF_FILE = 0o666 # Default permission bits (regular file) PERM_ALL = 0o7777 # All permission bits. +STDLIB_PATH = os.path.realpath(sysconfig.get_path("stdlib")) +PYFAKEFS_PATH = os.path.dirname(__file__) +PYFAKEFS_TEST_PATHS = [ + os.path.join(PYFAKEFS_PATH, "tests"), + os.path.join(PYFAKEFS_PATH, "pytest_tests"), +] + _OpenModes = namedtuple( "_OpenModes", "must_exist can_read can_write truncate append must_not_exist", @@ -429,3 +438,56 @@ def getvalue(self) -> bytes: def putvalue(self, value: bytes) -> None: self._bytestream.write(value) + + +def is_called_from_skipped_module(skip_names: list, case_sensitive: bool) -> bool: + def starts_with(path, string): + if case_sensitive: + return path.startswith(string) + return path.lower().startswith(string.lower()) + + stack = traceback.extract_stack() + + # handle the case that we try to call the original `open_code` + # (since Python 3.12) + # The stack in this case is: + # -1: helpers.is_called_from_skipped_module: 'stack = traceback.extract_stack()' + # -2: fake_open.fake_open: 'if is_called_from_skipped_module(' + # -3: fake_io.open: 'return fake_open(' + # -4: fake_io.open_code : 'return self._io_module.open_code(path)' + if ( + sys.version_info >= (3, 12) + and stack[-4].name == "open_code" + and stack[-4].line == "return self._io_module.open_code(path)" + ): + return True + + caller_filename = next( + ( + frame.filename + for frame in stack[::-1] + if not frame.filename.startswith("