From 95e9fb5ee24c759f333be6eb3b95e811149c15ea Mon Sep 17 00:00:00 2001 From: mrbean-bremen Date: Sun, 29 May 2022 15:56:33 +0200 Subject: [PATCH] Add some support for Python 3.11 - add Python 3.11 to CI tests - adapt the pathlib tests to work with Python 3.11 - adapt handling of pathlib in unfaked modules: need to ensure that the original os module is used, as pathlib has removed the accessor layer and now directly accesses os - add target_is_directory argument to symlink (ignored) - 'U' open mode is no longer allowed in Python 3.11 - closes #677 --- .github/workflows/pythonpackage.yml | 4 +- CHANGES.md | 3 +- README.md | 2 +- pyfakefs/fake_filesystem.py | 47 ++++++++++++-- pyfakefs/fake_pathlib.py | 95 +++++++++++++++++------------ pyfakefs/helpers.py | 8 +-- pyfakefs/tests/fake_open_test.py | 9 +++ pyfakefs/tests/fake_pathlib_test.py | 50 +++++++++------ pyfakefs/tests/test_utils.py | 30 ++++----- setup.py | 1 + 10 files changed, 165 insertions(+), 84 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 31762018..8f85d10d 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -38,7 +38,7 @@ jobs: - name: Install dependencies run: | python -m pip install -r requirements.txt -r extra_requirements.txt - python -m pip install mypy==0.812 + python -m pip install mypy==0.950 - name: Run typing checks run: python -m mypy . @@ -48,7 +48,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macOS-latest, windows-latest] - python-version: [3.7, 3.8, 3.9, "3.10"] + python-version: [3.7, 3.8, 3.9, "3.10", "3.11-dev"] include: - python-version: pypy3 os: ubuntu-latest diff --git a/CHANGES.md b/CHANGES.md index 5f475fd0..2aded265 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,12 +3,13 @@ The released versions correspond to PyPi releases. ## Unreleased - ### Changes * Python 3.6 has reached its end of life on 2021/12/23 and is no longer officially supported by pyfakefs ** `os.stat_float_times` has been removed in Python 3.7 and is therefore no longer supported +* added some support for the upcoming Python version 3.11 + (see [#677](../../issues/677)) * under Windows, the root path is now effectively `C:\` instead of `\`; a path starting with `\` points to the current drive as in the real file system (see [#673](../../issues/673)) diff --git a/README.md b/README.md index 5edfb07a..704f0f61 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ For example, pyfakefs will not work with [`lxml`](http://lxml.de/). In this cas ### Continuous integration pyfakefs is currently automatically tested on Linux, MacOS and Windows, with -Python 3.7 to 3.10, and with PyPy3 on Linux, using +Python 3.7 to 3.11, and with PyPy3 on Linux, using [GitHub Actions](https://github.com/jmcgeheeiv/pyfakefs/actions). ### Running pyfakefs unit tests diff --git a/pyfakefs/fake_filesystem.py b/pyfakefs/fake_filesystem.py index d51cb003..0bfa4639 100644 --- a/pyfakefs/fake_filesystem.py +++ b/pyfakefs/fake_filesystem.py @@ -94,7 +94,9 @@ True """ import errno +import functools import heapq +import inspect import io import locale import os @@ -103,6 +105,7 @@ import traceback import uuid from collections import namedtuple, OrderedDict +from contextlib import contextmanager from doctest import TestResults from enum import Enum from stat import ( @@ -3851,6 +3854,8 @@ class FakeOsModule: my_os_module = fake_filesystem.FakeOsModule(filesystem) """ + use_original = False + @staticmethod def dir() -> List[str]: """Return the list of patched function names. Used for patching @@ -3879,7 +3884,7 @@ def __init__(self, filesystem: FakeFilesystem): filesystem: FakeFilesystem used to provide file system information """ self.filesystem = filesystem - self._os_module: Any = os + self.os_module: Any = os self.path = FakePathModule(self.filesystem, self) @property @@ -4790,13 +4795,15 @@ def mknod(self, path: AnyStr, mode: Optional[int] = None, tail, mode & ~self.filesystem.umask, filesystem=self.filesystem)) - def symlink(self, src: AnyStr, dst: AnyStr, *, + def symlink(self, src: AnyStr, dst: AnyStr, + target_is_directory: bool = False, *, dir_fd: Optional[int] = None) -> None: """Creates the specified symlink, pointed at the specified link target. Args: src: The target of the symlink. dst: Path to the symlink to create. + target_is_directory: Currently ignored. dir_fd: If not `None`, the file descriptor of a directory, with `src` being relative to this directory. @@ -4915,7 +4922,38 @@ def sendfile(self, fd_out: int, fd_in: int, def __getattr__(self, name: str) -> Any: """Forwards any unfaked calls to the standard os module.""" - return getattr(self._os_module, name) + return getattr(self.os_module, name) + + +if sys.version_info > (3, 10): + def handle_original_call(f: Callable) -> Callable: + """Decorator used for real pathlib Path methods to ensure that + real os functions instead of faked ones are used. + Applied to all non-private methods of `FakeOsModule`.""" + @functools.wraps(f) + def wrapped(*args, **kwargs): + if not f.__name__.startswith('_') and FakeOsModule.use_original: + # remove the `self` argument for FakeOsModule methods + if args and isinstance(args[0], FakeOsModule): + args = args[1:] + return getattr(os, f.__name__)(*args, **kwargs) + return f(*args, **kwargs) + return wrapped + + for name, fn in inspect.getmembers(FakeOsModule, inspect.isfunction): + setattr(FakeOsModule, name, handle_original_call(fn)) + + +@contextmanager +def use_original_os(): + """Temporarily use original os functions instead of faked ones. + Used to ensure that skipped modules do not use faked calls. + """ + try: + FakeOsModule.use_original = True + yield + finally: + FakeOsModule.use_original = False class FakeIoModule: @@ -5847,7 +5885,8 @@ def _handle_file_mode( _OpenModes]: orig_modes = mode # Save original modes for error messages. # Normalize modes. Handle 't' and 'U'. - if 'b' in mode and 't' in mode: + if (('b' in mode and 't' in mode) or + (sys.version_info > (3, 10) and 'U' in mode)): raise ValueError('Invalid mode: ' + mode) mode = mode.replace('t', '').replace('b', '') mode = mode.replace('rU', 'r').replace('U', 'r') diff --git a/pyfakefs/fake_pathlib.py b/pyfakefs/fake_pathlib.py index 375843f9..5935c616 100644 --- a/pyfakefs/fake_pathlib.py +++ b/pyfakefs/fake_pathlib.py @@ -31,16 +31,20 @@ import errno import fnmatch import functools +import inspect import os import pathlib -from pathlib import PurePath import re import sys +from pathlib import PurePath +from typing import Callable from urllib.parse import quote_from_bytes as urlquote_from_bytes from pyfakefs import fake_scandir from pyfakefs.extra_packages import use_scandir -from pyfakefs.fake_filesystem import FakeFileOpen, FakeFilesystem +from pyfakefs.fake_filesystem import ( + FakeFileOpen, FakeFilesystem, use_original_os +) def init_module(filesystem): @@ -98,27 +102,26 @@ class _FakeAccessor(accessor): # type: ignore [valid-type, misc] if use_scandir: scandir = _wrap_strfunc(fake_scandir.scandir) - chmod = _wrap_strfunc(FakeFilesystem.chmod) - if hasattr(os, "lchmod"): lchmod = _wrap_strfunc(lambda fs, path, mode: FakeFilesystem.chmod( fs, path, mode, follow_symlinks=False)) else: - def lchmod(self, pathobj, *args, **kwargs): + def lchmod(self, pathobj, *args, **kwargs): """Raises not implemented for Windows systems.""" raise NotImplementedError("lchmod() not available on this system") - def chmod(self, pathobj, *args, **kwargs): - if "follow_symlinks" in kwargs: - if sys.version_info < (3, 10): - raise TypeError("chmod() got an unexpected keyword " - "argument 'follow_synlinks'") - if (not kwargs["follow_symlinks"] and - os.chmod not in os.supports_follow_symlinks): - raise NotImplementedError( - "`follow_symlinks` for chmod() is not available " - "on this system") - return pathobj.filesystem.chmod(str(pathobj), *args, **kwargs) + def chmod(self, pathobj, *args, **kwargs): + if "follow_symlinks" in kwargs: + if sys.version_info < (3, 10): + raise TypeError("chmod() got an unexpected keyword " + "argument 'follow_symlinks'") + + if (not kwargs["follow_symlinks"] and + os.os_module.chmod not in os.supports_follow_symlinks): + raise NotImplementedError( + "`follow_symlinks` for chmod() is not available " + "on this system") + return pathobj.filesystem.chmod(str(pathobj), *args, **kwargs) mkdir = _wrap_strfunc(FakeFilesystem.makedir) @@ -793,6 +796,8 @@ class RealPath(pathlib.Path): Needed because `Path` in `pathlib` is always faked, even if `pathlib` itself is not. """ + _flavour = (pathlib._WindowsFlavour() if os.name == 'nt' # type:ignore + else pathlib._PosixFlavour()) # type:ignore def __new__(cls, *args, **kwargs): """Creates the correct subclass based on OS.""" @@ -803,6 +808,41 @@ def __new__(cls, *args, **kwargs): return self +if sys.version_info > (3, 10): + def with_original_os(f: Callable) -> Callable: + """Decorator used for real pathlib Path methods to ensure that + real os functions instead of faked ones are used.""" + @functools.wraps(f) + def wrapped(*args, **kwargs): + with use_original_os(): + return f(*args, **kwargs) + return wrapped + + for name, fn in inspect.getmembers(RealPath, inspect.isfunction): + setattr(RealPath, name, with_original_os(fn)) + + +class RealPathlibPathModule: + """Patches `pathlib.Path` by passing all calls to RealPathlibModule.""" + real_pathlib = None + + @classmethod + def __instancecheck__(cls, instance): + # as we cannot derive from pathlib.Path, we fake + # the inheritance to pass isinstance checks - see #666 + return isinstance(instance, PurePath) + + def __init__(self): + if self.real_pathlib is None: + self.__class__.real_pathlib = RealPathlibModule() + + def __call__(self, *args, **kwargs): + return RealPath(*args, **kwargs) + + def __getattr__(self, name): + return getattr(self.real_pathlib.Path, name) + + class RealPathlibModule: """Used to replace `pathlib` for skipped modules. As the original `pathlib` is always patched to use the fake path, @@ -810,8 +850,6 @@ class RealPathlibModule: """ def __init__(self): - RealPathlibModule.PureWindowsPath._flavour = pathlib._WindowsFlavour() - RealPathlibModule.PurePosixPath._flavour = pathlib._PosixFlavour() self._pathlib_module = pathlib class PurePosixPath(PurePath): @@ -840,24 +878,3 @@ class PosixPath(RealPath, PurePosixPath): def __getattr__(self, name): """Forwards any unfaked calls to the standard pathlib module.""" return getattr(self._pathlib_module, name) - - -class RealPathlibPathModule: - """Patches `pathlib.Path` by passing all calls to RealPathlibModule.""" - real_pathlib = None - - @classmethod - def __instancecheck__(cls, instance): - # as we cannot derive from pathlib.Path, we fake - # the inheritance to pass isinstance checks - see #666 - return isinstance(instance, PurePath) - - def __init__(self): - if self.real_pathlib is None: - self.__class__.real_pathlib = RealPathlibModule() - - def __call__(self, *args, **kwargs): - return self.real_pathlib.Path(*args, **kwargs) - - def __getattr__(self, name): - return getattr(self.real_pathlib.Path, name) diff --git a/pyfakefs/helpers.py b/pyfakefs/helpers.py index ec8c2300..3e1ae69b 100644 --- a/pyfakefs/helpers.py +++ b/pyfakefs/helpers.py @@ -221,13 +221,13 @@ def st_file_attributes(self) -> int: mode = 0 st_mode = self.st_mode if st_mode & stat.S_IFDIR: - mode |= stat.FILE_ATTRIBUTE_DIRECTORY + mode |= stat.FILE_ATTRIBUTE_DIRECTORY # type:ignore[attr-defined] if st_mode & stat.S_IFREG: - mode |= stat.FILE_ATTRIBUTE_NORMAL + mode |= stat.FILE_ATTRIBUTE_NORMAL # type:ignore[attr-defined] if st_mode & (stat.S_IFCHR | stat.S_IFBLK): - mode |= stat.FILE_ATTRIBUTE_DEVICE + mode |= stat.FILE_ATTRIBUTE_DEVICE # type:ignore[attr-defined] if st_mode & stat.S_IFLNK: - mode |= stat.FILE_ATTRIBUTE_REPARSE_POINT + mode |= stat.FILE_ATTRIBUTE_REPARSE_POINT # type:ignore return mode @property diff --git a/pyfakefs/tests/fake_open_test.py b/pyfakefs/tests/fake_open_test.py index 9e942f98..399660cc 100644 --- a/pyfakefs/tests/fake_open_test.py +++ b/pyfakefs/tests/fake_open_test.py @@ -1486,6 +1486,7 @@ def test_readlines_with_newline_arg(self): with self.open(file_path, mode='r', newline='\r\n') as f: self.assertEqual(['1\r\n', '2\n3\r4'], f.readlines()) + @unittest.skipIf(sys.version_info >= (3, 10), "U flag no longer supported") def test_read_with_ignored_universal_newlines_flag(self): file_path = self.make_path('some_file') file_contents = b'1\r\n2\n3\r4' @@ -1497,6 +1498,14 @@ def test_read_with_ignored_universal_newlines_flag(self): with self.open(file_path, mode='U', newline='\r') as f: self.assertEqual('1\r\n2\n3\r4', f.read()) + @unittest.skipIf(sys.version_info < (3, 11), "U flag still supported") + def test_universal_newlines_flag_not_supported(self): + file_path = self.make_path('some_file') + file_contents = b'1\r\n2\n3\r4' + self.create_file(file_path, contents=file_contents) + with self.assertRaises(ValueError): + self.open(file_path, mode='U', newline='\r') + def test_write_with_newline_arg(self): file_path = self.make_path('some_file') with self.open(file_path, 'w', newline='') as f: diff --git a/pyfakefs/tests/fake_pathlib_test.py b/pyfakefs/tests/fake_pathlib_test.py index 1a3cdfca..0ffe082c 100644 --- a/pyfakefs/tests/fake_pathlib_test.py +++ b/pyfakefs/tests/fake_pathlib_test.py @@ -27,26 +27,36 @@ import sys import unittest +from pyfakefs import fake_pathlib, fake_filesystem, fake_filesystem_unittest from pyfakefs.fake_filesystem import is_root - -from pyfakefs import fake_pathlib, fake_filesystem from pyfakefs.helpers import IS_PYPY -from pyfakefs.tests.test_utils import RealFsTestCase +from pyfakefs.tests.test_utils import RealFsTestMixin is_windows = sys.platform == 'win32' -class RealPathlibTestCase(RealFsTestCase): +class RealPathlibTestCase(fake_filesystem_unittest.TestCase, RealFsTestMixin): + is_windows = sys.platform == 'win32' + def __init__(self, methodName='runTest'): - super(RealPathlibTestCase, self).__init__(methodName) - self.pathlib = pathlib - self.path = None + fake_filesystem_unittest.TestCase.__init__(self, methodName) + RealFsTestMixin.__init__(self) def setUp(self): - super().setUp() + RealFsTestMixin.setUp(self) + self.filesystem = None + self.real_os = os if not self.use_real_fs(): - self.pathlib = fake_pathlib.FakePathlibModule(self.filesystem) - self.path = self.pathlib.Path + self.setUpPyfakefs() + self.filesystem = self.fs + self.create_basepath() + self.pathlib = pathlib + self.path = pathlib.Path + self.os = os + self.open = open + + def use_real_fs(self): + return False class FakePathlibInitializationTest(RealPathlibTestCase): @@ -178,7 +188,8 @@ def test_is_absolute(self): class RealPathlibInitializationWithDriveTest( - FakePathlibInitializationWithDriveTest): + FakePathlibInitializationWithDriveTest +): def use_real_fs(self): return True @@ -383,7 +394,7 @@ def test_lchmod(self): self.skip_if_symlink_not_supported() file_stat = self.os.stat(self.file_path) link_stat = self.os.lstat(self.file_link_path) - if not hasattr(os, "lchmod"): + if not hasattr(self.real_os, "lchmod"): with self.assertRaises(NotImplementedError): self.path(self.file_link_path).lchmod(0o444) else: @@ -393,13 +404,16 @@ def test_lchmod(self): self.assertEqual(link_stat.st_mode & 0o777700, stat.S_IFLNK | 0o700) - @unittest.skipIf(sys.version_info < (3, 10), - "follow_symlinks argument new in Python 3.10") + @unittest.skipIf( + sys.version_info < (3, 10), + "follow_symlinks argument new in Python 3.10" + ) def test_chmod_no_followsymlinks(self): self.skip_if_symlink_not_supported() file_stat = self.os.stat(self.file_path) link_stat = self.os.lstat(self.file_link_path) - if os.chmod not in os.supports_follow_symlinks or IS_PYPY: + if (self.real_os.chmod not in os.supports_follow_symlinks + or IS_PYPY): with self.assertRaises(NotImplementedError): self.path(self.file_link_path).chmod(0o444, follow_symlinks=False) @@ -443,11 +457,11 @@ def test_iterdir_in_unreadable_dir(self): file_path = self.os.path.join(dir_path, 'some_file') self.create_file(file_path) self.os.chmod(dir_path, 0o000) - iter = self.path(dir_path).iterdir() + it = self.path(dir_path).iterdir() if not is_root(): - self.assert_raises_os_error(errno.EACCES, list, iter) + self.assert_raises_os_error(errno.EACCES, list, it) else: - path = str(list(iter)[0]) + path = str(list(it)[0]) self.assertTrue(path.endswith('some_file')) def test_resolve_nonexisting_file(self): diff --git a/pyfakefs/tests/test_utils.py b/pyfakefs/tests/test_utils.py index eea81c49..b8cf79a8 100644 --- a/pyfakefs/tests/test_utils.py +++ b/pyfakefs/tests/test_utils.py @@ -81,17 +81,6 @@ def raises_os_error(self, subtype): else: self.assertEqual(subtype, exc.errno) - def assert_raises_os_error(self, subtype, expression, *args, **kwargs): - """Asserts that a specific subtype of OSError is raised.""" - try: - expression(*args, **kwargs) - self.fail('No exception was raised, OSError expected') - except OSError as exc: - if isinstance(subtype, list): - self.assertIn(exc.errno, subtype) - else: - self.assertEqual(subtype, exc.errno) - class RealFsTestMixin: """Test mixin to allow tests to run both in the fake filesystem and in the @@ -149,6 +138,9 @@ def use_real_fs(self): """Return True if the real file system shall be tested.""" return False + def setUpFileSystem(self): + pass + def path_separator(self): """Can be overwritten to use a specific separator in the fake filesystem.""" @@ -362,7 +354,7 @@ def create_basepath(self): if self.filesystem is not None: old_base_path = self.base_path self.base_path = self.filesystem.path_separator + 'basepath' - if self.is_windows_fs: + if self.filesystem.is_windows_fs: self.base_path = 'C:' + self.base_path if old_base_path != self.base_path: if old_base_path is not None: @@ -402,6 +394,17 @@ def mock_time(self, start=200, step=20): DummyTime(start, step)) return DummyMock() + def assert_raises_os_error(self, subtype, expression, *args, **kwargs): + """Asserts that a specific subtype of OSError is raised.""" + try: + expression(*args, **kwargs) + self.fail('No exception was raised, OSError expected') + except OSError as exc: + if isinstance(subtype, list): + self.assertIn(exc.errno, subtype) + else: + self.assertEqual(subtype, exc.errno) + class RealFsTestCase(TestCase, RealFsTestMixin): """Can be used as base class for tests also running in the real @@ -426,9 +429,6 @@ def setUp(self): def tearDown(self): RealFsTestMixin.tearDown(self) - def setUpFileSystem(self): - pass - @property def is_windows_fs(self): if self.use_real_fs(): diff --git a/setup.py b/setup.py index c3a449e9..60b5b111 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Operating System :: POSIX',