diff --git a/docs/conf.py b/docs/conf.py index a5bfe62..f08abdf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # maxminddb documentation build configuration file, created by # sphinx-quickstart on Tue Apr 9 13:34:57 2013. @@ -12,8 +11,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os +import sys sys.path.insert(0, os.path.abspath("..")) import maxminddb diff --git a/examples/benchmark.py b/examples/benchmark.py index 2c947c9..c07a5d1 100644 --- a/examples/benchmark.py +++ b/examples/benchmark.py @@ -1,13 +1,13 @@ #!/usr/bin/python -# -*- coding: utf-8 -*- import argparse -import maxminddb import random import socket import struct import timeit +import maxminddb + parser = argparse.ArgumentParser(description="Benchmark maxminddb.") parser.add_argument("--count", default=250000, type=int, help="number of lookups") parser.add_argument("--mode", default=0, type=int, help="reader mode to use") @@ -19,9 +19,9 @@ reader = maxminddb.open_database(args.file, args.mode) -def lookup_ip_address(): +def lookup_ip_address() -> None: ip = socket.inet_ntoa(struct.pack("!L", random.getrandbits(32))) - record = reader.get(str(ip)) + reader.get(str(ip)) elapsed = timeit.timeit( @@ -30,4 +30,4 @@ def lookup_ip_address(): number=args.count, ) -print("{:,}".format(int(args.count / elapsed)), "lookups per second") +print(f"{int(args.count / elapsed):,}", "lookups per second") diff --git a/maxminddb/__init__.py b/maxminddb/__init__.py index 2c61745..8e83734 100644 --- a/maxminddb/__init__.py +++ b/maxminddb/__init__.py @@ -21,13 +21,13 @@ __all__ = [ - "InvalidDatabaseError", "MODE_AUTO", "MODE_FD", "MODE_FILE", "MODE_MEMORY", "MODE_MMAP", "MODE_MMAP_EXT", + "InvalidDatabaseError", "Reader", "open_database", ] @@ -37,7 +37,7 @@ def open_database( database: Union[AnyStr, int, os.PathLike, IO], mode: int = MODE_AUTO, ) -> Reader: - """Open a MaxMind DB database + """Open a MaxMind DB database. Arguments: database -- A path to a valid MaxMind DB file such as a GeoIP2 database @@ -51,6 +51,7 @@ def open_database( a path. This mode implies MODE_MEMORY. * MODE_AUTO - tries MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that order. Default mode. + """ if mode not in ( MODE_AUTO, @@ -70,7 +71,7 @@ def open_database( if not has_extension: raise ValueError( - "MODE_MMAP_EXT requires the maxminddb.extension module to be available" + "MODE_MMAP_EXT requires the maxminddb.extension module to be available", ) # The C type exposes the same API as the Python Reader, so for type diff --git a/maxminddb/const.py b/maxminddb/const.py index 45222c2..959078d 100644 --- a/maxminddb/const.py +++ b/maxminddb/const.py @@ -1,4 +1,4 @@ -"""Constants used in the API""" +"""Constants used in the API.""" MODE_AUTO = 0 MODE_MMAP_EXT = 1 diff --git a/maxminddb/decoder.py b/maxminddb/decoder.py index 7add3fd..4237e2b 100644 --- a/maxminddb/decoder.py +++ b/maxminddb/decoder.py @@ -7,7 +7,7 @@ """ import struct -from typing import cast, Dict, List, Tuple, Union +from typing import Dict, List, Tuple, Union, cast try: # pylint: disable=unused-import @@ -23,7 +23,7 @@ class Decoder: # pylint: disable=too-few-public-methods - """Decoder for the data section of the MaxMind DB""" + """Decoder for the data section of the MaxMind DB.""" def __init__( self, @@ -31,12 +31,13 @@ def __init__( pointer_base: int = 0, pointer_test: bool = False, ) -> None: - """Created a Decoder for a MaxMind DB + """Created a Decoder for a MaxMind DB. Arguments: database_buffer -- an mmap'd MaxMind DB file. pointer_base -- the base number to use when decoding a pointer pointer_test -- used for internal unit testing of pointer code + """ self._pointer_test = pointer_test self._buffer = database_buffer @@ -138,10 +139,11 @@ def _decode_utf8_string(self, size: int, offset: int) -> Tuple[str, int]: } def decode(self, offset: int) -> Tuple[Record, int]: - """Decode a section of the data section starting at offset + """Decode a section of the data section starting at offset. Arguments: offset -- the location of the data structure to decode + """ new_offset = offset + 1 ctrl_byte = self._buffer[offset] @@ -154,7 +156,7 @@ def decode(self, offset: int) -> Tuple[Record, int]: decoder = self._type_decoder[type_num] except KeyError as ex: raise InvalidDatabaseError( - f"Unexpected type number ({type_num}) encountered" + f"Unexpected type number ({type_num}) encountered", ) from ex (size, new_offset) = self._size_from_ctrl_byte(ctrl_byte, new_offset, type_num) @@ -166,7 +168,7 @@ def _read_extended(self, offset: int) -> Tuple[int, int]: if type_num < 7: raise InvalidDatabaseError( "Something went horribly wrong in the decoder. An " - f"extended type resolved to a type number < 8 ({type_num})" + f"extended type resolved to a type number < 8 ({type_num})", ) return type_num, offset + 1 @@ -175,11 +177,14 @@ def _verify_size(expected: int, actual: int) -> None: if expected != actual: raise InvalidDatabaseError( "The MaxMind DB file's data section contains bad data " - "(unknown data type or corrupt data)" + "(unknown data type or corrupt data)", ) def _size_from_ctrl_byte( - self, ctrl_byte: int, offset: int, type_num: int + self, + ctrl_byte: int, + offset: int, + type_num: int, ) -> Tuple[int, int]: size = ctrl_byte & 0x1F if type_num == 1 or size < 29: diff --git a/maxminddb/extension.pyi b/maxminddb/extension.pyi index 56883d2..c4dee56 100644 --- a/maxminddb/extension.pyi +++ b/maxminddb/extension.pyi @@ -6,22 +6,25 @@ This module contains the C extension database reader and related classes. """ +# pylint: disable=E0601,E0602 from ipaddress import IPv4Address, IPv6Address from os import PathLike -from typing import Any, AnyStr, Dict, IO, List, Optional, Tuple, Union +from typing import IO, Any, AnyStr, Optional, Tuple, Union + from maxminddb import MODE_AUTO from maxminddb.types import Record class Reader: - """ - A C extension implementation of a reader for the MaxMind DB format. IP + """A C extension implementation of a reader for the MaxMind DB format. IP addresses can be looked up using the ``get`` method. """ closed: bool = ... def __init__( - self, database: Union[AnyStr, int, PathLike, IO], mode: int = MODE_AUTO + self, + database: Union[AnyStr, int, PathLike, IO], + mode: int = MODE_AUTO, ) -> None: """Reader for the MaxMind DB file format @@ -30,6 +33,7 @@ class Reader: file, or a file descriptor in the case of MODE_FD. mode -- mode to open the database with. The only supported modes are MODE_AUTO and MODE_MMAP_EXT. + """ def close(self) -> None: @@ -38,25 +42,26 @@ class Reader: def get(self, ip_address: Union[str, IPv6Address, IPv4Address]) -> Optional[Record]: """Return the record for the ip_address in the MaxMind DB - Arguments: ip_address -- an IP address in the standard string notation + """ def get_with_prefix_len( - self, ip_address: Union[str, IPv6Address, IPv4Address] + self, + ip_address: Union[str, IPv6Address, IPv4Address], ) -> Tuple[Optional[Record], int]: """Return a tuple with the record and the associated prefix length - Arguments: ip_address -- an IP address in the standard string notation + """ - def metadata(self) -> "Metadata": + def metadata(self) -> Metadata: """Return the metadata associated with the MaxMind DB file""" - def __enter__(self) -> "Reader": ... + def __enter__(self) -> Reader: ... def __exit__(self, *args) -> None: ... # pylint: disable=too-few-public-methods @@ -85,7 +90,7 @@ class Metadata: A string identifying the database type, e.g., "GeoIP2-City". """ - description: Dict[str, str] + description: dict[str, str] """ A map from locales to text descriptions of the database. """ @@ -97,7 +102,7 @@ class Metadata: both IPv4 and IPv6 lookups. """ - languages: List[str] + languages: list[str] """ A list of locale codes supported by the databse. """ diff --git a/maxminddb/file.py b/maxminddb/file.py index 4dbecba..541e730 100644 --- a/maxminddb/file.py +++ b/maxminddb/file.py @@ -11,7 +11,7 @@ class FileBuffer: - """A slice-able file reader""" + """A slice-able file reader.""" def __init__(self, database: str) -> None: # pylint: disable=consider-using-with @@ -28,31 +28,31 @@ def __getitem__(self, key: Union[slice, int]): raise TypeError("Invalid argument type.") def rfind(self, needle: bytes, start: int) -> int: - """Reverse find needle from start""" + """Reverse find needle from start.""" pos = self._read(self._size - start - 1, start).rfind(needle) if pos == -1: return pos return start + pos def size(self) -> int: - """Size of file""" + """Size of file.""" return self._size def close(self) -> None: - """Close file""" + """Close file.""" self._handle.close() if hasattr(os, "pread"): def _read(self, buffersize: int, offset: int) -> bytes: - """read that uses pread""" + """Read that uses pread.""" # pylint: disable=no-member return os.pread(self._handle.fileno(), buffersize, offset) # type: ignore else: def _read(self, buffersize: int, offset: int) -> bytes: - """read with a lock + """Read with a lock. This lock is necessary as after a fork, the different processes will share the same file table entry, even if we dup the fd, and diff --git a/maxminddb/reader.py b/maxminddb/reader.py index 606aa28..403139c 100644 --- a/maxminddb/reader.py +++ b/maxminddb/reader.py @@ -16,9 +16,9 @@ import struct from ipaddress import IPv4Address, IPv6Address from os import PathLike -from typing import Any, AnyStr, Dict, IO, List, Optional, Tuple, Union +from typing import IO, Any, AnyStr, Dict, Iterator, List, Optional, Tuple, Union -from maxminddb.const import MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY, MODE_FD +from maxminddb.const import MODE_AUTO, MODE_FD, MODE_FILE, MODE_MEMORY, MODE_MMAP from maxminddb.decoder import Decoder from maxminddb.errors import InvalidDatabaseError from maxminddb.file import FileBuffer @@ -34,7 +34,7 @@ class Reader: """ _DATA_SECTION_SEPARATOR_SIZE = 16 - _METADATA_START_MARKER = b"\xAB\xCD\xEFMaxMind.com" + _METADATA_START_MARKER = b"\xab\xcd\xefMaxMind.com" _buffer: Union[bytes, FileBuffer, "mmap.mmap"] _buffer_size: int @@ -44,9 +44,11 @@ class Reader: _ipv4_start: int def __init__( - self, database: Union[AnyStr, int, PathLike, IO], mode: int = MODE_AUTO + self, + database: Union[AnyStr, int, PathLike, IO], + mode: int = MODE_AUTO, ) -> None: - """Reader for the MaxMind DB file format + """Reader for the MaxMind DB file format. Arguments: database -- A path to a valid MaxMind DB file such as a GeoIP2 database @@ -58,6 +60,7 @@ def __init__( * MODE_AUTO - tries MODE_MMAP and then MODE_FILE. Default. * MODE_FD - the param passed via database is a file descriptor, not a path. This mode implies MODE_MEMORY. + """ filename: Any if (mode == MODE_AUTO and mmap) or mode == MODE_MMAP: @@ -83,18 +86,19 @@ def __init__( raise ValueError( f"Unsupported open mode ({mode}). Only MODE_AUTO, MODE_FILE, " "MODE_MEMORY and MODE_FD are supported by the pure Python " - "Reader" + "Reader", ) metadata_start = self._buffer.rfind( - self._METADATA_START_MARKER, max(0, self._buffer_size - 128 * 1024) + self._METADATA_START_MARKER, + max(0, self._buffer_size - 128 * 1024), ) if metadata_start == -1: self.close() raise InvalidDatabaseError( f"Error opening database file ({filename}). " - "Is this a valid MaxMind DB file?" + "Is this a valid MaxMind DB file?", ) metadata_start += len(self._METADATA_START_MARKER) @@ -103,7 +107,7 @@ def __init__( if not isinstance(metadata, dict): raise InvalidDatabaseError( - f"Error reading metadata in database file ({filename})." + f"Error reading metadata in database file ({filename}).", ) self._metadata = Metadata(**metadata) # pylint: disable=bad-option-value @@ -128,27 +132,28 @@ def __init__( self._ipv4_start = ipv4_start def metadata(self) -> "Metadata": - """Return the metadata associated with the MaxMind DB file""" + """Return the metadata associated with the MaxMind DB file.""" return self._metadata def get(self, ip_address: Union[str, IPv6Address, IPv4Address]) -> Optional[Record]: - """Return the record for the ip_address in the MaxMind DB - + """Return the record for the ip_address in the MaxMind DB. Arguments: ip_address -- an IP address in the standard string notation + """ (record, _) = self.get_with_prefix_len(ip_address) return record def get_with_prefix_len( - self, ip_address: Union[str, IPv6Address, IPv4Address] + self, + ip_address: Union[str, IPv6Address, IPv4Address], ) -> Tuple[Optional[Record], int]: - """Return a tuple with the record and the associated prefix length - + """Return a tuple with the record and the associated prefix length. Arguments: ip_address -- an IP address in the standard string notation + """ if isinstance(ip_address, str): address = ipaddress.ip_address(ip_address) @@ -163,7 +168,7 @@ def get_with_prefix_len( if address.version == 6 and self._metadata.ip_version == 4: raise ValueError( f"Error looking up {ip_address}. You attempted to look up " - "an IPv6 address in an IPv4-only database." + "an IPv6 address in an IPv4-only database.", ) (pointer, prefix_len) = self._find_address_in_tree(packed_address) @@ -172,10 +177,10 @@ def get_with_prefix_len( return self._resolve_data_pointer(pointer), prefix_len return None, prefix_len - def __iter__(self): + def __iter__(self) -> Iterator: return self._generate_children(0, 0, 0) - def _generate_children(self, node, depth, ip_acc): + def _generate_children(self, node, depth, ip_acc) -> Iterator: if ip_acc != 0 and node == self._ipv4_start: # Skip nodes aliased to IPv4 return @@ -187,7 +192,7 @@ def _generate_children(self, node, depth, ip_acc): if ip_acc <= _IPV4_MAX_NUM and bits == 128: depth -= 96 yield ipaddress.ip_network((ip_acc, depth)), self._resolve_data_pointer( - node + node, ) elif node < node_count: left = self._read_node(node, 0) @@ -240,20 +245,22 @@ def _read_node(self, node_number: int, index: int) -> int: offset = base_offset + index * 4 node_bytes = self._buffer[offset : offset + 4] else: - raise InvalidDatabaseError(f"Unknown record size: {record_size}") + msg = f"Unknown record size: {record_size}" + raise InvalidDatabaseError(msg) return struct.unpack(b"!I", node_bytes)[0] def _resolve_data_pointer(self, pointer: int) -> Record: resolved = pointer - self._metadata.node_count + self._metadata.search_tree_size if resolved >= self._buffer_size: - raise InvalidDatabaseError("The MaxMind DB file's search tree is corrupt") + msg = "The MaxMind DB file's search tree is corrupt" + raise InvalidDatabaseError(msg) (data, _) = self._decoder.decode(resolved) return data def close(self) -> None: - """Closes the MaxMind DB file and returns the resources to the system""" + """Closes the MaxMind DB file and returns the resources to the system.""" try: self._buffer.close() # type: ignore except AttributeError: @@ -265,13 +272,14 @@ def __exit__(self, *args) -> None: def __enter__(self) -> "Reader": if self.closed: - raise ValueError("Attempt to reopen a closed MaxMind DB") + msg = "Attempt to reopen a closed MaxMind DB" + raise ValueError(msg) return self # pylint: disable=too-many-instance-attributes,R0801 class Metadata: - """Metadata for the MaxMind DB reader""" + """Metadata for the MaxMind DB reader.""" binary_format_major_version: int """ @@ -323,7 +331,7 @@ class Metadata: """ def __init__(self, **kwargs) -> None: - """Creates new Metadata object. kwargs are key/value pairs from spec""" + """Creates new Metadata object. kwargs are key/value pairs from spec.""" # Although I could just update __dict__, that is less obvious and it # doesn't work well with static analysis tools and some IDEs self.node_count = kwargs["node_count"] @@ -338,7 +346,7 @@ def __init__(self, **kwargs) -> None: @property def node_byte_size(self) -> int: - """The size of a node in bytes + """The size of a node in bytes. :type: int """ @@ -346,12 +354,12 @@ def node_byte_size(self) -> int: @property def search_tree_size(self) -> int: - """The size of the search tree + """The size of the search tree. :type: int """ return self.node_count * self.node_byte_size - def __repr__(self): + def __repr__(self) -> str: args = ", ".join(f"{k}={v!r}" for k, v in self.__dict__.items()) return f"{self.__module__}.{self.__class__.__name__}({args})" diff --git a/maxminddb/types.py b/maxminddb/types.py index 03a02e6..9ab76ca 100644 --- a/maxminddb/types.py +++ b/maxminddb/types.py @@ -12,12 +12,8 @@ class RecordList(List[Record]): # pylint: disable=too-few-public-methods - """ - RecordList is a type for lists in a database record. - """ + """RecordList is a type for lists in a database record.""" class RecordDict(Dict[str, Record]): # pylint: disable=too-few-public-methods - """ - RecordDict is a type for dicts in a database record. - """ + """RecordDict is a type for dicts in a database record.""" diff --git a/pyproject.toml b/pyproject.toml index 7377bdc..a90aee9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,3 +43,29 @@ maxminddb = ["py.typed"] # src is showing up in our GitHub linting builds. It seems to # contain deps. extend-exclude = '^/src/' + +[tool.ruff.lint] +select = ["ALL"] +ignore = [ + # Skip type annotation on **_ + "ANN003", + + # documenting magic methods + "D105", + + # Line length. We let black handle this for now. + "E501", + + # Don't bother with future imports for type annotations + "FA100", + + # Magic numbers for HTTP status codes seem ok most of the time. + "PLR2004", + + # pytest rules + "PT009", + "PT027", +] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["ANN201", "D"] diff --git a/setup.py b/setup.py index 8b50d18..5b5a5d9 100644 --- a/setup.py +++ b/setup.py @@ -2,20 +2,23 @@ import re import sys -from setuptools import setup, Extension +from setuptools import Extension, setup from setuptools.command.build_ext import build_ext from wheel.bdist_wheel import bdist_wheel - # These were only added to setuptools in 59.0.1. try: - from setuptools.errors import CCompilerError - from setuptools.errors import DistutilsExecError - from setuptools.errors import DistutilsPlatformError + from setuptools.errors import ( + CCompilerError, + DistutilsExecError, + DistutilsPlatformError, + ) except ImportError: - from distutils.errors import CCompilerError - from distutils.errors import DistutilsExecError - from distutils.errors import DistutilsPlatformError + from distutils.errors import ( + CCompilerError, + DistutilsExecError, + DistutilsPlatformError, + ) cmdclass = {} PYPY = hasattr(sys, "pypy_version_info") @@ -34,10 +37,10 @@ ext_module = [ Extension( "maxminddb.extension", - libraries=["maxminddb"] + libraries, + libraries=["maxminddb", *libraries], sources=["extension/maxminddb.c"], extra_compile_args=compile_args, - ) + ), ] else: ext_module = [ @@ -66,7 +69,7 @@ "extension/libmaxminddb/src", ], extra_compile_args=compile_args, - ) + ), ] # Cargo cult code for installing extension with pure Python fallback. @@ -75,34 +78,33 @@ class BuildFailed(Exception): - def __init__(self): + def __init__(self) -> None: self.cause = sys.exc_info()[1] class ve_build_ext(build_ext): # This class allows C extension building to fail. - def run(self): + def run(self) -> None: try: build_ext.run(self) except DistutilsPlatformError: - raise BuildFailed() + raise BuildFailed - def build_extension(self, ext): + def build_extension(self, ext) -> None: try: build_ext.build_extension(self, ext) except ext_errors: - raise BuildFailed() + raise BuildFailed except ValueError: # this can happen on Windows 64 bit, see Python issue 7511 if "'path'" in str(sys.exc_info()[1]): - raise BuildFailed() + raise BuildFailed raise cmdclass["build_ext"] = ve_build_ext -# ROOT = os.path.dirname(__file__) @@ -112,7 +114,9 @@ def build_extension(self, ext): with open(os.path.join(ROOT, "maxminddb", "__init__.py"), "rb") as fd: maxminddb_text = fd.read().decode("utf8") VERSION = ( - re.compile(r".*__version__ = \"(.*?)\"", re.S).match(maxminddb_text).group(1) + re.compile(r".*__version__ = \"(.*?)\"", re.DOTALL) + .match(maxminddb_text) + .group(1) ) @@ -126,14 +130,14 @@ def status_msgs(*msgs): def find_packages(location): packages = [] for pkg in ["maxminddb"]: - for _dir, subdirectories, files in os.walk(os.path.join(location, pkg)): + for _dir, _subdirectories, files in os.walk(os.path.join(location, pkg)): if "__init__.py" in files: tokens = _dir.split(os.sep)[len(location.split(os.sep)) :] packages.append(".".join(tokens)) return packages -def run_setup(with_cext): +def run_setup(with_cext) -> None: kwargs = {} loc_cmdclass = cmdclass.copy() if with_cext: @@ -154,7 +158,7 @@ def run_setup(with_cext): run_setup(True) except BuildFailed as exc: if os.getenv("MAXMINDDB_REQUIRE_EXTENSION"): - raise exc + raise status_msgs( exc.cause, "WARNING: The C extension could not be compiled, " diff --git a/tests/decoder_test.py b/tests/decoder_test.py index 6089aec..e965643 100644 --- a/tests/decoder_test.py +++ b/tests/decoder_test.py @@ -1,15 +1,13 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- import mmap +import unittest from maxminddb.decoder import Decoder -import unittest - class TestDecoder(unittest.TestCase): - def test_arrays(self): + def test_arrays(self) -> None: arrays = { b"\x00\x04": [], b"\x01\x04\x43\x46\x6f\x6f": ["Foo"], @@ -17,41 +15,41 @@ def test_arrays(self): } self.validate_type_decoding("arrays", arrays) - def test_boolean(self): + def test_boolean(self) -> None: booleans = { b"\x00\x07": False, b"\x01\x07": True, } self.validate_type_decoding("booleans", booleans) - def test_double(self): + def test_double(self) -> None: doubles = { b"\x68\x00\x00\x00\x00\x00\x00\x00\x00": 0.0, - b"\x68\x3F\xE0\x00\x00\x00\x00\x00\x00": 0.5, - b"\x68\x40\x09\x21\xFB\x54\x44\x2E\xEA": 3.14159265359, - b"\x68\x40\x5E\xC0\x00\x00\x00\x00\x00": 123.0, - b"\x68\x41\xD0\x00\x00\x00\x07\xF8\xF4": 1073741824.12457, - b"\x68\xBF\xE0\x00\x00\x00\x00\x00\x00": -0.5, - b"\x68\xC0\x09\x21\xFB\x54\x44\x2E\xEA": -3.14159265359, - b"\x68\xC1\xD0\x00\x00\x00\x07\xF8\xF4": -1073741824.12457, + b"\x68\x3f\xe0\x00\x00\x00\x00\x00\x00": 0.5, + b"\x68\x40\x09\x21\xfb\x54\x44\x2e\xea": 3.14159265359, + b"\x68\x40\x5e\xc0\x00\x00\x00\x00\x00": 123.0, + b"\x68\x41\xd0\x00\x00\x00\x07\xf8\xf4": 1073741824.12457, + b"\x68\xbf\xe0\x00\x00\x00\x00\x00\x00": -0.5, + b"\x68\xc0\x09\x21\xfb\x54\x44\x2e\xea": -3.14159265359, + b"\x68\xc1\xd0\x00\x00\x00\x07\xf8\xf4": -1073741824.12457, } self.validate_type_decoding("double", doubles) - def test_float(self): + def test_float(self) -> None: floats = { b"\x04\x08\x00\x00\x00\x00": 0.0, - b"\x04\x08\x3F\x80\x00\x00": 1.0, - b"\x04\x08\x3F\x8C\xCC\xCD": 1.1, - b"\x04\x08\x40\x48\xF5\xC3": 3.14, - b"\x04\x08\x46\x1C\x3F\xF6": 9999.99, - b"\x04\x08\xBF\x80\x00\x00": -1.0, - b"\x04\x08\xBF\x8C\xCC\xCD": -1.1, - b"\x04\x08\xC0\x48\xF5\xC3": -3.14, - b"\x04\x08\xC6\x1C\x3F\xF6": -9999.99, + b"\x04\x08\x3f\x80\x00\x00": 1.0, + b"\x04\x08\x3f\x8c\xcc\xcd": 1.1, + b"\x04\x08\x40\x48\xf5\xc3": 3.14, + b"\x04\x08\x46\x1c\x3f\xf6": 9999.99, + b"\x04\x08\xbf\x80\x00\x00": -1.0, + b"\x04\x08\xbf\x8c\xcc\xcd": -1.1, + b"\x04\x08\xc0\x48\xf5\xc3": -3.14, + b"\x04\x08\xc6\x1c\x3f\xf6": -9999.99, } self.validate_type_decoding("float", floats) - def test_int32(self): + def test_int32(self) -> None: int32 = { b"\x00\x01": 0, b"\x04\x01\xff\xff\xff\xff": -1, @@ -68,7 +66,7 @@ def test_int32(self): } self.validate_type_decoding("int32", int32) - def test_map(self): + def test_map(self) -> None: maps = { b"\xe0": {}, b"\xe1\x42\x65\x6e\x43\x46\x6f\x6f": {"en": "Foo"}, @@ -87,7 +85,7 @@ def test_map(self): } self.validate_type_decoding("maps", maps) - def test_pointer(self): + def test_pointer(self) -> None: pointers = { b"\x20\x00": 0, b"\x20\x05": 5, @@ -106,7 +104,7 @@ def test_pointer(self): strings = { b"\x40": "", b"\x41\x31": "1", - b"\x43\xE4\xBA\xBA": "人", + b"\x43\xe4\xba\xba": "人", ( b"\x5b\x31\x32\x33\x34" b"\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35" @@ -135,17 +133,17 @@ def test_pointer(self): b"\x5f\x00\x10\x53" + 70000 * b"\x78": "x" * 70000, } - def test_string(self): + def test_string(self) -> None: self.validate_type_decoding("string", self.strings) - def test_byte(self): + def test_byte(self) -> None: b = { bytes([0xC0 ^ k[0]]) + k[1:]: v.encode("utf-8") for k, v in self.strings.items() } self.validate_type_decoding("byte", b) - def test_uint16(self): + def test_uint16(self) -> None: uint16 = { b"\xa0": 0, b"\xa1\xff": 255, @@ -155,7 +153,7 @@ def test_uint16(self): } self.validate_type_decoding("uint16", uint16) - def test_uint32(self): + def test_uint32(self) -> None: uint32 = { b"\xc0": 0, b"\xc1\xff": 255, @@ -167,7 +165,7 @@ def test_uint32(self): } self.validate_type_decoding("uint32", uint32) - def generate_large_uint(self, bits): + def generate_large_uint(self, bits) -> dict: ctrl_byte = b"\x02" if bits == 64 else b"\x03" uints = { b"\x00" + ctrl_byte: 0, @@ -180,17 +178,17 @@ def generate_large_uint(self, bits): uints[input] = expected return uints - def test_uint64(self): + def test_uint64(self) -> None: self.validate_type_decoding("uint64", self.generate_large_uint(64)) - def test_uint128(self): + def test_uint128(self) -> None: self.validate_type_decoding("uint128", self.generate_large_uint(128)) - def validate_type_decoding(self, type, tests): + def validate_type_decoding(self, type, tests) -> None: for input, expected in tests.items(): self.check_decoding(type, input, expected) - def check_decoding(self, type, input, expected, name=None): + def check_decoding(self, type, input, expected, name=None) -> None: name = name or expected db = mmap.mmap(-1, len(input)) db.write(input) @@ -206,7 +204,7 @@ def check_decoding(self, type, input, expected, name=None): else: self.assertEqual(expected, actual, type) - def test_real_pointers(self): + def test_real_pointers(self) -> None: with open("tests/data/test-data/maps-with-pointers.raw", "r+b") as db_file: mm = mmap.mmap(db_file.fileno(), 0) decoder = Decoder(mm, 0) diff --git a/tests/reader_test.py b/tests/reader_test.py index 315200d..569baf6 100644 --- a/tests/reader_test.py +++ b/tests/reader_test.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- import ipaddress import multiprocessing @@ -7,9 +6,8 @@ import pathlib import threading import unittest -import unittest.mock as mock -from multiprocessing import Process, Pipe -from typing import Union, Type +from typing import Type, Union, cast +from unittest import mock import maxminddb @@ -18,18 +16,19 @@ except ImportError: maxminddb.extension = None # type: ignore -from maxminddb import open_database, InvalidDatabaseError +from maxminddb import InvalidDatabaseError, open_database from maxminddb.const import ( MODE_AUTO, - MODE_MMAP_EXT, - MODE_MMAP, + MODE_FD, MODE_FILE, MODE_MEMORY, - MODE_FD, + MODE_MMAP, + MODE_MMAP_EXT, ) +from maxminddb.reader import Reader -def get_reader_from_file_descriptor(filepath, mode): +def get_reader_from_file_descriptor(filepath, mode) -> Reader: """Patches open_database() for class TestFDReader().""" if mode == MODE_FD: with open(filepath, "rb") as mmdb_fh: @@ -41,9 +40,11 @@ def get_reader_from_file_descriptor(filepath, mode): return maxminddb.open_database(filepath, mode) -class BaseTestReader(object): - readerClass: Union[ - Type["maxminddb.extension.Reader"], Type["maxminddb.reader.Reader"] +class BaseTestReader(unittest.TestCase): + mode: int + reader_class: Union[ + Type["maxminddb.extension.Reader"], + Type["maxminddb.reader.Reader"], ] use_ip_objects = False @@ -52,12 +53,12 @@ class BaseTestReader(object): if os.name != "nt": mp = multiprocessing.get_context("fork") - def ipf(self, ip): + def ipf(self, ip) -> Union[ipaddress.IPv4Address, ipaddress.IPv6Address]: if self.use_ip_objects: return ipaddress.ip_address(ip) return ip - def test_reader(self): + def test_reader(self) -> None: for record_size in [24, 28, 32]: for ip_version in [4, 6]: file_name = ( @@ -77,7 +78,7 @@ def test_reader(self): self._check_ip_v6(reader, file_name) reader.close() - def test_get_with_prefix_len(self): + def test_get_with_prefix_len(self) -> None: decoder_record = { "array": [1, 2, 3], "boolean": True, @@ -175,9 +176,10 @@ def test_get_with_prefix_len(self): for test in tests: with open_database( - "tests/data/test-data/" + test["file_name"], self.mode + "tests/data/test-data/" + cast(str, test["file_name"]), + self.mode, ) as reader: - (record, prefix_len) = reader.get_with_prefix_len(test["ip"]) + (record, prefix_len) = reader.get_with_prefix_len(cast(str, test["ip"])) self.assertEqual( prefix_len, @@ -188,10 +190,13 @@ def test_get_with_prefix_len(self): self.assertEqual( record, test["expected_record"], - "expected_record for " + test["ip"] + " in " + test["file_name"], + "expected_record for " + + cast(str, test["ip"]) + + " in " + + cast(str, test["file_name"]), ) - def test_iterator(self): + def test_iterator(self) -> None: tests = ( { "database": "ipv4", @@ -239,11 +244,12 @@ def test_iterator(self): networks = [str(n) for (n, _) in reader] self.assertEqual(networks, test["expected"], f) - def test_decoder(self): + def test_decoder(self) -> None: reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) - record = reader.get(self.ipf("::1.1.1.0")) + record = cast(dict, reader.get(self.ipf("::1.1.1.0"))) self.assertEqual(record["array"], [1, 2, 3]) self.assertEqual(record["boolean"], True) @@ -266,27 +272,30 @@ def test_decoder(self): self.assertEqual(1329227995784915872903807060280344576, record["uint128"]) reader.close() - def test_metadata_pointers(self): + def test_metadata_pointers(self) -> None: with open_database( - "tests/data/test-data/MaxMind-DB-test-metadata-pointers.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-metadata-pointers.mmdb", + self.mode, ) as reader: self.assertEqual( "Lots of pointers in metadata", reader.metadata().database_type, ) - def test_no_ipv4_search_tree(self): + def test_no_ipv4_search_tree(self) -> None: reader = open_database( - "tests/data/test-data/MaxMind-DB-no-ipv4-search-tree.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-no-ipv4-search-tree.mmdb", + self.mode, ) self.assertEqual(reader.get(self.ipf("1.1.1.1")), "::0/64") self.assertEqual(reader.get(self.ipf("192.1.1.1")), "::0/64") reader.close() - def test_ipv6_address_in_ipv4_database(self): + def test_ipv6_address_in_ipv4_database(self) -> None: reader = open_database( - "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", + self.mode, ) with self.assertRaisesRegex( ValueError, @@ -297,27 +306,29 @@ def test_ipv6_address_in_ipv4_database(self): reader.get(self.ipf("2001::")) reader.close() - def test_opening_path(self): + def test_opening_path(self) -> None: with open_database( - pathlib.Path("tests/data/test-data/MaxMind-DB-test-decoder.mmdb"), self.mode + pathlib.Path("tests/data/test-data/MaxMind-DB-test-decoder.mmdb"), + self.mode, ) as reader: self.assertEqual(reader.metadata().database_type, "MaxMind DB Decoder Test") - def test_no_extension_exception(self): + def test_no_extension_exception(self) -> None: real_extension = maxminddb._extension - maxminddb._extension = None + maxminddb._extension = None # type: ignore with self.assertRaisesRegex( ValueError, "MODE_MMAP_EXT requires the maxminddb.extension module to be available", ): open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", MODE_MMAP_EXT + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + MODE_MMAP_EXT, ) maxminddb._extension = real_extension - def test_broken_database(self): + def test_broken_database(self) -> None: reader = open_database( - "tests/data/test-data/" "GeoIP2-City-Test-Broken-Double-Format.mmdb", + "tests/data/test-data/GeoIP2-City-Test-Broken-Double-Format.mmdb", self.mode, ) with self.assertRaisesRegex( @@ -329,21 +340,23 @@ def test_broken_database(self): reader.get(self.ipf("2001:220::")) reader.close() - def test_ip_validation(self): + def test_ip_validation(self) -> None: reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) with self.assertRaisesRegex( - ValueError, "'not_ip' does not appear to be an IPv4 or " "IPv6 address" + ValueError, + "'not_ip' does not appear to be an IPv4 or IPv6 address", ): reader.get("not_ip") reader.close() - def test_missing_database(self): + def test_missing_database(self) -> None: with self.assertRaisesRegex(FileNotFoundError, "No such file or directory"): open_database("file-does-not-exist.mmdb", self.mode) - def test_nondatabase(self): + def test_nondatabase(self) -> None: with self.assertRaisesRegex( InvalidDatabaseError, r"Error opening database file \(README.rst\). " @@ -352,7 +365,7 @@ def test_nondatabase(self): open_database("README.rst", self.mode) # This is from https://github.com/maxmind/MaxMind-DB-Reader-python/issues/58 - def test_database_with_invalid_utf8_key(self): + def test_database_with_invalid_utf8_key(self) -> None: reader = open_database( "tests/data/bad-data/maxminddb-python/bad-unicode-in-map-key.mmdb", self.mode, @@ -360,15 +373,15 @@ def test_database_with_invalid_utf8_key(self): with self.assertRaises(UnicodeDecodeError): reader.get_with_prefix_len("163.254.149.39") - def test_too_many_constructor_args(self): + def test_too_many_constructor_args(self) -> None: with self.assertRaises(TypeError): - self.readerClass("README.md", self.mode, 1) + self.reader_class("README.md", self.mode, 1) # type: ignore - def test_bad_constructor_mode(self): + def test_bad_constructor_mode(self) -> None: with self.assertRaisesRegex(ValueError, r"Unsupported open mode \(100\)"): - self.readerClass("README.md", mode=100) + self.reader_class("README.md", mode=100) # type: ignore - def test_no_constructor_args(self): + def test_no_constructor_args(self) -> None: with self.assertRaisesRegex( TypeError, r" 1 required positional argument|" @@ -376,96 +389,109 @@ def test_no_constructor_args(self): r"takes at least 2 arguments|" r"function missing required argument \'database\' \(pos 1\)", ): - self.readerClass() + self.reader_class() # type: ignore - def test_too_many_get_args(self): + def test_too_many_get_args(self) -> None: reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) with self.assertRaises(TypeError): - reader.get(self.ipf("1.1.1.1"), "blah") + reader.get(self.ipf("1.1.1.1"), "blah") # type: ignore reader.close() - def test_no_get_args(self): + def test_no_get_args(self) -> None: reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) with self.assertRaises(TypeError): - reader.get() + reader.get() # type: ignore reader.close() - def test_incorrect_get_arg_type(self): + def test_incorrect_get_arg_type(self) -> None: reader = open_database("tests/data/test-data/GeoIP2-City-Test.mmdb", self.mode) with self.assertRaisesRegex( - TypeError, "argument 1 must be a string or ipaddress object" + TypeError, + "argument 1 must be a string or ipaddress object", ): - reader.get(1) + reader.get(1) # type: ignore reader.close() - def test_metadata_args(self): + def test_metadata_args(self) -> None: reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) with self.assertRaises(TypeError): - reader.metadata("blah") + reader.metadata("blah") # type: ignore reader.close() - def test_metadata_unknown_attribute(self): + def test_metadata_unknown_attribute(self) -> None: reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) metadata = reader.metadata() with self.assertRaisesRegex( - AttributeError, "'Metadata' object has no " "attribute 'blah'" + AttributeError, + "'Metadata' object has no attribute 'blah'", ): - metadata.blah + metadata.blah # type: ignore reader.close() - def test_close(self): + def test_close(self) -> None: reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) reader.close() - def test_double_close(self): + def test_double_close(self) -> None: reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) reader.close() - self.assertIsNone(reader.close(), "Double close does not throw an exception") + self.assertIsNone( + reader.close(), "Double close does not throw an exception" # type: ignore + ) - def test_closed_get(self): + def test_closed_get(self) -> None: if self.mode in [MODE_MEMORY, MODE_FD]: return reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) reader.close() with self.assertRaisesRegex( - ValueError, "Attempt to read from a closed MaxMind DB.|closed" + ValueError, + "Attempt to read from a closed MaxMind DB.|closed", ): reader.get(self.ipf("1.1.1.1")) - def test_with_statement(self): + def test_with_statement(self) -> None: filename = "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb" with open_database(filename, self.mode) as reader: self._check_ip_v4(reader, filename) self.assertEqual(reader.closed, True) - def test_with_statement_close(self): + def test_with_statement_close(self) -> None: filename = "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb" reader = open_database(filename, self.mode) reader.close() with self.assertRaisesRegex( - ValueError, "Attempt to reopen a closed MaxMind DB" - ): - with reader: - pass + ValueError, + "Attempt to reopen a closed MaxMind DB", + ), reader: + pass - def test_closed(self): + def test_closed(self) -> None: reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) self.assertEqual(reader.closed, False) reader.close() @@ -475,9 +501,10 @@ def test_closed(self): # extension and the pure Python reader. If we do, the pure Python # reader will need to throw an exception or the extension will need # to keep the metadata in memory. - def test_closed_metadata(self): + def test_closed_metadata(self) -> None: reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) reader.close() @@ -485,7 +512,7 @@ def test_closed_metadata(self): # segfault try: metadata = reader.metadata() - except IOError as ex: + except OSError as ex: self.assertEqual( "Attempt to read from a closed MaxMind DB.", str(ex), @@ -502,12 +529,13 @@ def test_multiprocessing(self): def test_threading(self): self._check_concurrency(threading.Thread) - def _check_concurrency(self, worker_class): + def _check_concurrency(self, worker_class) -> None: reader = open_database( - "tests/data/test-data/GeoIP2-Domain-Test.mmdb", self.mode + "tests/data/test-data/GeoIP2-Domain-Test.mmdb", + self.mode, ) - def lookup(pipe): + def lookup(pipe) -> None: try: for i in range(32): reader.get(self.ipf(f"65.115.240.{i}")) @@ -515,7 +543,7 @@ def lookup(pipe): except: pipe.send(0) finally: - if worker_class is self.mp.Process: + if worker_class is self.mp.Process: # type: ignore reader.close() pipe.close() @@ -532,7 +560,7 @@ def lookup(pipe): self.assertEqual(count, 32, "expected number of successful lookups") - def _check_metadata(self, reader, ip_version, record_size): + def _check_metadata(self, reader, ip_version, record_size) -> None: metadata = reader.metadata() self.assertEqual(2, metadata.binary_format_major_version, "major version") @@ -541,7 +569,8 @@ def _check_metadata(self, reader, ip_version, record_size): self.assertEqual(metadata.database_type, "Test") self.assertEqual( - {"en": "Test Database", "zh": "Test Database Chinese"}, metadata.description + {"en": "Test Database", "zh": "Test Database Chinese"}, + metadata.description, ) self.assertEqual(metadata.ip_version, ip_version) self.assertEqual(metadata.languages, ["en", "zh"]) @@ -549,7 +578,7 @@ def _check_metadata(self, reader, ip_version, record_size): self.assertEqual(metadata.record_size, record_size) - def _check_ip_v4(self, reader, file_name): + def _check_ip_v4(self, reader, file_name) -> None: for i in range(6): address = "1.1.1." + str(pow(2, i)) self.assertEqual( @@ -579,7 +608,7 @@ def _check_ip_v4(self, reader, file_name): for ip in ["1.1.1.33", "255.254.253.123"]: self.assertIsNone(reader.get(self.ipf(ip))) - def _check_ip_v6(self, reader, file_name): + def _check_ip_v6(self, reader, file_name) -> None: subnets = ["::1:ffff:ffff", "::2:0:0", "::2:0:40", "::2:0:50", "::2:0:58"] for address in subnets: @@ -611,83 +640,89 @@ def _check_ip_v6(self, reader, file_name): self.assertIsNone(reader.get(self.ipf(ip))) -def has_maxminddb_extension(): - return maxminddb.extension and hasattr(maxminddb.extension, "Reader") +def has_maxminddb_extension() -> bool: + return maxminddb.extension and hasattr( + maxminddb.extension, "Reader" + ) # type: ignore @unittest.skipIf( not has_maxminddb_extension() and not os.environ.get("MM_FORCE_EXT_TESTS"), "No C extension module found. Skipping tests", ) -class TestExtensionReader(BaseTestReader, unittest.TestCase): +class TestExtensionReader(BaseTestReader): mode = MODE_MMAP_EXT if has_maxminddb_extension(): - readerClass = maxminddb.extension.Reader + reader_class = maxminddb.extension.Reader @unittest.skipIf( not has_maxminddb_extension() and not os.environ.get("MM_FORCE_EXT_TESTS"), "No C extension module found. Skipping tests", ) -class TestExtensionReaderWithIPObjects(BaseTestReader, unittest.TestCase): +class TestExtensionReaderWithIPObjects(BaseTestReader): mode = MODE_MMAP_EXT use_ip_objects = True if has_maxminddb_extension(): - readerClass = maxminddb.extension.Reader + reader_class = maxminddb.extension.Reader -class TestAutoReader(BaseTestReader, unittest.TestCase): +class TestAutoReader(BaseTestReader): mode = MODE_AUTO - readerClass: Union[ - Type["maxminddb.extension.Reader"], Type["maxminddb.reader.Reader"] + reader_class: Union[ + Type["maxminddb.extension.Reader"], + Type["maxminddb.reader.Reader"], ] if has_maxminddb_extension(): - readerClass = maxminddb.extension.Reader + reader_class = maxminddb.extension.Reader else: - readerClass = maxminddb.reader.Reader + reader_class = maxminddb.reader.Reader -class TestMMAPReader(BaseTestReader, unittest.TestCase): +class TestMMAPReader(BaseTestReader): mode = MODE_MMAP - readerClass = maxminddb.reader.Reader + reader_class = maxminddb.reader.Reader # We want one pure Python test to use IP objects, it doesn't # really matter which one. -class TestMMAPReaderWithIPObjects(BaseTestReader, unittest.TestCase): +class TestMMAPReaderWithIPObjects(BaseTestReader): mode = MODE_MMAP use_ip_objects = True - readerClass = maxminddb.reader.Reader + reader_class = maxminddb.reader.Reader -class TestFileReader(BaseTestReader, unittest.TestCase): +class TestFileReader(BaseTestReader): mode = MODE_FILE - readerClass = maxminddb.reader.Reader + reader_class = maxminddb.reader.Reader -class TestMemoryReader(BaseTestReader, unittest.TestCase): +class TestMemoryReader(BaseTestReader): mode = MODE_MEMORY - readerClass = maxminddb.reader.Reader + reader_class = maxminddb.reader.Reader -class TestFDReader(BaseTestReader, unittest.TestCase): - def setUp(self): +class TestFDReader(BaseTestReader): + def setUp(self) -> None: self.open_database_patcher = mock.patch(__name__ + ".open_database") self.addCleanup(self.open_database_patcher.stop) self.open_database = self.open_database_patcher.start() self.open_database.side_effect = get_reader_from_file_descriptor mode = MODE_FD - readerClass = maxminddb.reader.Reader + reader_class = maxminddb.reader.Reader class TestOldReader(unittest.TestCase): - def test_old_reader(self): + def test_old_reader(self) -> None: reader = maxminddb.Reader("tests/data/test-data/MaxMind-DB-test-decoder.mmdb") - record = reader.get("::1.1.1.0") + record = cast(dict, reader.get("::1.1.1.0")) self.assertEqual(record["array"], [1, 2, 3]) reader.close() + + +del BaseTestReader