Skip to content

Commit

Permalink
Add some support for Python 3.11
Browse files Browse the repository at this point in the history
- add Python 3.11 to CI tests
- adapt the pathlib tests to work with Python 3.11
- adapt handling of pathlib in unfaked modules:
  need to ensure that the original os module is used,
  as pathlib has removed the accessor layer and now
  directly accesses os
- add target_is_directory argument to symlink (ignored)
- 'U' open mode is no longer allowed in Python 3.11
- closes #677
  • Loading branch information
mrbean-bremen committed May 31, 2022
1 parent 8c4f409 commit 95e9fb5
Show file tree
Hide file tree
Showing 10 changed files with 165 additions and 84 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/pythonpackage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ jobs:
- name: Install dependencies
run: |
python -m pip install -r requirements.txt -r extra_requirements.txt
python -m pip install mypy==0.812
python -m pip install mypy==0.950
- name: Run typing checks
run: python -m mypy .

Expand All @@ -48,7 +48,7 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, macOS-latest, windows-latest]
python-version: [3.7, 3.8, 3.9, "3.10"]
python-version: [3.7, 3.8, 3.9, "3.10", "3.11-dev"]
include:
- python-version: pypy3
os: ubuntu-latest
Expand Down
3 changes: 2 additions & 1 deletion CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@ The released versions correspond to PyPi releases.

## Unreleased


### Changes
* Python 3.6 has reached its end of life on 2021/12/23 and is no
longer officially supported by pyfakefs
** `os.stat_float_times` has been removed in Python 3.7 and is therefore no
longer supported
* added some support for the upcoming Python version 3.11
(see [#677](../../issues/677))
* under Windows, the root path is now effectively `C:\` instead of `\`; a
path starting with `\` points to the current drive as in the real file
system (see [#673](../../issues/673))
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ For example, pyfakefs will not work with [`lxml`](http://lxml.de/). In this cas
### Continuous integration

pyfakefs is currently automatically tested on Linux, MacOS and Windows, with
Python 3.7 to 3.10, and with PyPy3 on Linux, using
Python 3.7 to 3.11, and with PyPy3 on Linux, using
[GitHub Actions](https://github.com/jmcgeheeiv/pyfakefs/actions).

### Running pyfakefs unit tests
Expand Down
47 changes: 43 additions & 4 deletions pyfakefs/fake_filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,9 @@
True
"""
import errno
import functools
import heapq
import inspect
import io
import locale
import os
Expand All @@ -103,6 +105,7 @@
import traceback
import uuid
from collections import namedtuple, OrderedDict
from contextlib import contextmanager
from doctest import TestResults
from enum import Enum
from stat import (
Expand Down Expand Up @@ -3851,6 +3854,8 @@ class FakeOsModule:
my_os_module = fake_filesystem.FakeOsModule(filesystem)
"""

use_original = False

@staticmethod
def dir() -> List[str]:
"""Return the list of patched function names. Used for patching
Expand Down Expand Up @@ -3879,7 +3884,7 @@ def __init__(self, filesystem: FakeFilesystem):
filesystem: FakeFilesystem used to provide file system information
"""
self.filesystem = filesystem
self._os_module: Any = os
self.os_module: Any = os
self.path = FakePathModule(self.filesystem, self)

@property
Expand Down Expand Up @@ -4790,13 +4795,15 @@ def mknod(self, path: AnyStr, mode: Optional[int] = None,
tail, mode & ~self.filesystem.umask,
filesystem=self.filesystem))

def symlink(self, src: AnyStr, dst: AnyStr, *,
def symlink(self, src: AnyStr, dst: AnyStr,
target_is_directory: bool = False, *,
dir_fd: Optional[int] = None) -> None:
"""Creates the specified symlink, pointed at the specified link target.
Args:
src: The target of the symlink.
dst: Path to the symlink to create.
target_is_directory: Currently ignored.
dir_fd: If not `None`, the file descriptor of a directory,
with `src` being relative to this directory.
Expand Down Expand Up @@ -4915,7 +4922,38 @@ def sendfile(self, fd_out: int, fd_in: int,

def __getattr__(self, name: str) -> Any:
"""Forwards any unfaked calls to the standard os module."""
return getattr(self._os_module, name)
return getattr(self.os_module, name)


if sys.version_info > (3, 10):
def handle_original_call(f: Callable) -> Callable:
"""Decorator used for real pathlib Path methods to ensure that
real os functions instead of faked ones are used.
Applied to all non-private methods of `FakeOsModule`."""
@functools.wraps(f)
def wrapped(*args, **kwargs):
if not f.__name__.startswith('_') and FakeOsModule.use_original:
# remove the `self` argument for FakeOsModule methods
if args and isinstance(args[0], FakeOsModule):
args = args[1:]
return getattr(os, f.__name__)(*args, **kwargs)
return f(*args, **kwargs)
return wrapped

for name, fn in inspect.getmembers(FakeOsModule, inspect.isfunction):
setattr(FakeOsModule, name, handle_original_call(fn))


@contextmanager
def use_original_os():
"""Temporarily use original os functions instead of faked ones.
Used to ensure that skipped modules do not use faked calls.
"""
try:
FakeOsModule.use_original = True
yield
finally:
FakeOsModule.use_original = False


class FakeIoModule:
Expand Down Expand Up @@ -5847,7 +5885,8 @@ def _handle_file_mode(
_OpenModes]:
orig_modes = mode # Save original modes for error messages.
# Normalize modes. Handle 't' and 'U'.
if 'b' in mode and 't' in mode:
if (('b' in mode and 't' in mode) or
(sys.version_info > (3, 10) and 'U' in mode)):
raise ValueError('Invalid mode: ' + mode)
mode = mode.replace('t', '').replace('b', '')
mode = mode.replace('rU', 'r').replace('U', 'r')
Expand Down
95 changes: 56 additions & 39 deletions pyfakefs/fake_pathlib.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,16 +31,20 @@
import errno
import fnmatch
import functools
import inspect
import os
import pathlib
from pathlib import PurePath
import re
import sys
from pathlib import PurePath
from typing import Callable
from urllib.parse import quote_from_bytes as urlquote_from_bytes

from pyfakefs import fake_scandir
from pyfakefs.extra_packages import use_scandir
from pyfakefs.fake_filesystem import FakeFileOpen, FakeFilesystem
from pyfakefs.fake_filesystem import (
FakeFileOpen, FakeFilesystem, use_original_os
)


def init_module(filesystem):
Expand Down Expand Up @@ -98,27 +102,26 @@ class _FakeAccessor(accessor): # type: ignore [valid-type, misc]
if use_scandir:
scandir = _wrap_strfunc(fake_scandir.scandir)

chmod = _wrap_strfunc(FakeFilesystem.chmod)

if hasattr(os, "lchmod"):
lchmod = _wrap_strfunc(lambda fs, path, mode: FakeFilesystem.chmod(
fs, path, mode, follow_symlinks=False))
else:
def lchmod(self, pathobj, *args, **kwargs):
def lchmod(self, pathobj, *args, **kwargs):
"""Raises not implemented for Windows systems."""
raise NotImplementedError("lchmod() not available on this system")

def chmod(self, pathobj, *args, **kwargs):
if "follow_symlinks" in kwargs:
if sys.version_info < (3, 10):
raise TypeError("chmod() got an unexpected keyword "
"argument 'follow_synlinks'")
if (not kwargs["follow_symlinks"] and
os.chmod not in os.supports_follow_symlinks):
raise NotImplementedError(
"`follow_symlinks` for chmod() is not available "
"on this system")
return pathobj.filesystem.chmod(str(pathobj), *args, **kwargs)
def chmod(self, pathobj, *args, **kwargs):
if "follow_symlinks" in kwargs:
if sys.version_info < (3, 10):
raise TypeError("chmod() got an unexpected keyword "
"argument 'follow_symlinks'")

if (not kwargs["follow_symlinks"] and
os.os_module.chmod not in os.supports_follow_symlinks):
raise NotImplementedError(
"`follow_symlinks` for chmod() is not available "
"on this system")
return pathobj.filesystem.chmod(str(pathobj), *args, **kwargs)

mkdir = _wrap_strfunc(FakeFilesystem.makedir)

Expand Down Expand Up @@ -793,6 +796,8 @@ class RealPath(pathlib.Path):
Needed because `Path` in `pathlib` is always faked, even if `pathlib`
itself is not.
"""
_flavour = (pathlib._WindowsFlavour() if os.name == 'nt' # type:ignore
else pathlib._PosixFlavour()) # type:ignore

def __new__(cls, *args, **kwargs):
"""Creates the correct subclass based on OS."""
Expand All @@ -803,15 +808,48 @@ def __new__(cls, *args, **kwargs):
return self


if sys.version_info > (3, 10):
def with_original_os(f: Callable) -> Callable:
"""Decorator used for real pathlib Path methods to ensure that
real os functions instead of faked ones are used."""
@functools.wraps(f)
def wrapped(*args, **kwargs):
with use_original_os():
return f(*args, **kwargs)
return wrapped

for name, fn in inspect.getmembers(RealPath, inspect.isfunction):
setattr(RealPath, name, with_original_os(fn))


class RealPathlibPathModule:
"""Patches `pathlib.Path` by passing all calls to RealPathlibModule."""
real_pathlib = None

@classmethod
def __instancecheck__(cls, instance):
# as we cannot derive from pathlib.Path, we fake
# the inheritance to pass isinstance checks - see #666
return isinstance(instance, PurePath)

def __init__(self):
if self.real_pathlib is None:
self.__class__.real_pathlib = RealPathlibModule()

def __call__(self, *args, **kwargs):
return RealPath(*args, **kwargs)

def __getattr__(self, name):
return getattr(self.real_pathlib.Path, name)


class RealPathlibModule:
"""Used to replace `pathlib` for skipped modules.
As the original `pathlib` is always patched to use the fake path,
we need to provide a version which does not do this.
"""

def __init__(self):
RealPathlibModule.PureWindowsPath._flavour = pathlib._WindowsFlavour()
RealPathlibModule.PurePosixPath._flavour = pathlib._PosixFlavour()
self._pathlib_module = pathlib

class PurePosixPath(PurePath):
Expand Down Expand Up @@ -840,24 +878,3 @@ class PosixPath(RealPath, PurePosixPath):
def __getattr__(self, name):
"""Forwards any unfaked calls to the standard pathlib module."""
return getattr(self._pathlib_module, name)


class RealPathlibPathModule:
"""Patches `pathlib.Path` by passing all calls to RealPathlibModule."""
real_pathlib = None

@classmethod
def __instancecheck__(cls, instance):
# as we cannot derive from pathlib.Path, we fake
# the inheritance to pass isinstance checks - see #666
return isinstance(instance, PurePath)

def __init__(self):
if self.real_pathlib is None:
self.__class__.real_pathlib = RealPathlibModule()

def __call__(self, *args, **kwargs):
return self.real_pathlib.Path(*args, **kwargs)

def __getattr__(self, name):
return getattr(self.real_pathlib.Path, name)
8 changes: 4 additions & 4 deletions pyfakefs/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,13 +221,13 @@ def st_file_attributes(self) -> int:
mode = 0
st_mode = self.st_mode
if st_mode & stat.S_IFDIR:
mode |= stat.FILE_ATTRIBUTE_DIRECTORY
mode |= stat.FILE_ATTRIBUTE_DIRECTORY # type:ignore[attr-defined]
if st_mode & stat.S_IFREG:
mode |= stat.FILE_ATTRIBUTE_NORMAL
mode |= stat.FILE_ATTRIBUTE_NORMAL # type:ignore[attr-defined]
if st_mode & (stat.S_IFCHR | stat.S_IFBLK):
mode |= stat.FILE_ATTRIBUTE_DEVICE
mode |= stat.FILE_ATTRIBUTE_DEVICE # type:ignore[attr-defined]
if st_mode & stat.S_IFLNK:
mode |= stat.FILE_ATTRIBUTE_REPARSE_POINT
mode |= stat.FILE_ATTRIBUTE_REPARSE_POINT # type:ignore
return mode

@property
Expand Down
9 changes: 9 additions & 0 deletions pyfakefs/tests/fake_open_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -1486,6 +1486,7 @@ def test_readlines_with_newline_arg(self):
with self.open(file_path, mode='r', newline='\r\n') as f:
self.assertEqual(['1\r\n', '2\n3\r4'], f.readlines())

@unittest.skipIf(sys.version_info >= (3, 10), "U flag no longer supported")
def test_read_with_ignored_universal_newlines_flag(self):
file_path = self.make_path('some_file')
file_contents = b'1\r\n2\n3\r4'
Expand All @@ -1497,6 +1498,14 @@ def test_read_with_ignored_universal_newlines_flag(self):
with self.open(file_path, mode='U', newline='\r') as f:
self.assertEqual('1\r\n2\n3\r4', f.read())

@unittest.skipIf(sys.version_info < (3, 11), "U flag still supported")
def test_universal_newlines_flag_not_supported(self):
file_path = self.make_path('some_file')
file_contents = b'1\r\n2\n3\r4'
self.create_file(file_path, contents=file_contents)
with self.assertRaises(ValueError):
self.open(file_path, mode='U', newline='\r')

def test_write_with_newline_arg(self):
file_path = self.make_path('some_file')
with self.open(file_path, 'w', newline='') as f:
Expand Down
Loading

0 comments on commit 95e9fb5

Please sign in to comment.