From f6d14ddaa8db876ab3fd656d135db62dbd0113e2 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 29 Jan 2025 11:36:37 -0800 Subject: [PATCH 1/8] Run latest black version on code --- maxminddb/reader.py | 2 +- tests/decoder_test.py | 32 ++++++++++++++++---------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/maxminddb/reader.py b/maxminddb/reader.py index 606aa28..64e3962 100644 --- a/maxminddb/reader.py +++ b/maxminddb/reader.py @@ -34,7 +34,7 @@ class Reader: """ _DATA_SECTION_SEPARATOR_SIZE = 16 - _METADATA_START_MARKER = b"\xAB\xCD\xEFMaxMind.com" + _METADATA_START_MARKER = b"\xab\xcd\xefMaxMind.com" _buffer: Union[bytes, FileBuffer, "mmap.mmap"] _buffer_size: int diff --git a/tests/decoder_test.py b/tests/decoder_test.py index 6089aec..d55f03f 100644 --- a/tests/decoder_test.py +++ b/tests/decoder_test.py @@ -27,27 +27,27 @@ def test_boolean(self): def test_double(self): doubles = { b"\x68\x00\x00\x00\x00\x00\x00\x00\x00": 0.0, - b"\x68\x3F\xE0\x00\x00\x00\x00\x00\x00": 0.5, - b"\x68\x40\x09\x21\xFB\x54\x44\x2E\xEA": 3.14159265359, - b"\x68\x40\x5E\xC0\x00\x00\x00\x00\x00": 123.0, - b"\x68\x41\xD0\x00\x00\x00\x07\xF8\xF4": 1073741824.12457, - b"\x68\xBF\xE0\x00\x00\x00\x00\x00\x00": -0.5, - b"\x68\xC0\x09\x21\xFB\x54\x44\x2E\xEA": -3.14159265359, - b"\x68\xC1\xD0\x00\x00\x00\x07\xF8\xF4": -1073741824.12457, + b"\x68\x3f\xe0\x00\x00\x00\x00\x00\x00": 0.5, + b"\x68\x40\x09\x21\xfb\x54\x44\x2e\xea": 3.14159265359, + b"\x68\x40\x5e\xc0\x00\x00\x00\x00\x00": 123.0, + b"\x68\x41\xd0\x00\x00\x00\x07\xf8\xf4": 1073741824.12457, + b"\x68\xbf\xe0\x00\x00\x00\x00\x00\x00": -0.5, + b"\x68\xc0\x09\x21\xfb\x54\x44\x2e\xea": -3.14159265359, + b"\x68\xc1\xd0\x00\x00\x00\x07\xf8\xf4": -1073741824.12457, } self.validate_type_decoding("double", doubles) def test_float(self): floats = { b"\x04\x08\x00\x00\x00\x00": 0.0, - b"\x04\x08\x3F\x80\x00\x00": 1.0, - b"\x04\x08\x3F\x8C\xCC\xCD": 1.1, - b"\x04\x08\x40\x48\xF5\xC3": 3.14, - b"\x04\x08\x46\x1C\x3F\xF6": 9999.99, - b"\x04\x08\xBF\x80\x00\x00": -1.0, - b"\x04\x08\xBF\x8C\xCC\xCD": -1.1, - b"\x04\x08\xC0\x48\xF5\xC3": -3.14, - b"\x04\x08\xC6\x1C\x3F\xF6": -9999.99, + b"\x04\x08\x3f\x80\x00\x00": 1.0, + b"\x04\x08\x3f\x8c\xcc\xcd": 1.1, + b"\x04\x08\x40\x48\xf5\xc3": 3.14, + b"\x04\x08\x46\x1c\x3f\xf6": 9999.99, + b"\x04\x08\xbf\x80\x00\x00": -1.0, + b"\x04\x08\xbf\x8c\xcc\xcd": -1.1, + b"\x04\x08\xc0\x48\xf5\xc3": -3.14, + b"\x04\x08\xc6\x1c\x3f\xf6": -9999.99, } self.validate_type_decoding("float", floats) @@ -106,7 +106,7 @@ def test_pointer(self): strings = { b"\x40": "", b"\x41\x31": "1", - b"\x43\xE4\xBA\xBA": "人", + b"\x43\xe4\xba\xba": "人", ( b"\x5b\x31\x32\x33\x34" b"\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35" From ea11c450a28d19d151682eefe7daff52914b381c Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 29 Jan 2025 11:37:38 -0800 Subject: [PATCH 2/8] Add basic ruff config --- pyproject.toml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 7377bdc..a90aee9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,3 +43,29 @@ maxminddb = ["py.typed"] # src is showing up in our GitHub linting builds. It seems to # contain deps. extend-exclude = '^/src/' + +[tool.ruff.lint] +select = ["ALL"] +ignore = [ + # Skip type annotation on **_ + "ANN003", + + # documenting magic methods + "D105", + + # Line length. We let black handle this for now. + "E501", + + # Don't bother with future imports for type annotations + "FA100", + + # Magic numbers for HTTP status codes seem ok most of the time. + "PLR2004", + + # pytest rules + "PT009", + "PT027", +] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["ANN201", "D"] From 62c70843932cac65bf19f6fdbf995c462d2a25c7 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 29 Jan 2025 11:39:54 -0800 Subject: [PATCH 3/8] Run ruff's auto-fixer --- docs/conf.py | 3 +- examples/benchmark.py | 6 +-- maxminddb/__init__.py | 5 ++- maxminddb/decoder.py | 15 ++++--- maxminddb/extension.pyi | 24 +++++----- maxminddb/file.py | 4 +- maxminddb/reader.py | 29 +++++++----- maxminddb/types.py | 8 +--- setup.py | 34 +++++++------- tests/decoder_test.py | 4 +- tests/reader_test.py | 98 +++++++++++++++++++++++++---------------- 11 files changed, 133 insertions(+), 97 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index a5bfe62..f08abdf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # maxminddb documentation build configuration file, created by # sphinx-quickstart on Tue Apr 9 13:34:57 2013. @@ -12,8 +11,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os +import sys sys.path.insert(0, os.path.abspath("..")) import maxminddb diff --git a/examples/benchmark.py b/examples/benchmark.py index 2c947c9..90e32fa 100644 --- a/examples/benchmark.py +++ b/examples/benchmark.py @@ -1,13 +1,13 @@ #!/usr/bin/python -# -*- coding: utf-8 -*- import argparse -import maxminddb import random import socket import struct import timeit +import maxminddb + parser = argparse.ArgumentParser(description="Benchmark maxminddb.") parser.add_argument("--count", default=250000, type=int, help="number of lookups") parser.add_argument("--mode", default=0, type=int, help="reader mode to use") @@ -30,4 +30,4 @@ def lookup_ip_address(): number=args.count, ) -print("{:,}".format(int(args.count / elapsed)), "lookups per second") +print(f"{int(args.count / elapsed):,}", "lookups per second") diff --git a/maxminddb/__init__.py b/maxminddb/__init__.py index 2c61745..73f0d79 100644 --- a/maxminddb/__init__.py +++ b/maxminddb/__init__.py @@ -21,13 +21,13 @@ __all__ = [ - "InvalidDatabaseError", "MODE_AUTO", "MODE_FD", "MODE_FILE", "MODE_MEMORY", "MODE_MMAP", "MODE_MMAP_EXT", + "InvalidDatabaseError", "Reader", "open_database", ] @@ -51,6 +51,7 @@ def open_database( a path. This mode implies MODE_MEMORY. * MODE_AUTO - tries MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that order. Default mode. + """ if mode not in ( MODE_AUTO, @@ -70,7 +71,7 @@ def open_database( if not has_extension: raise ValueError( - "MODE_MMAP_EXT requires the maxminddb.extension module to be available" + "MODE_MMAP_EXT requires the maxminddb.extension module to be available", ) # The C type exposes the same API as the Python Reader, so for type diff --git a/maxminddb/decoder.py b/maxminddb/decoder.py index 7add3fd..fae91d3 100644 --- a/maxminddb/decoder.py +++ b/maxminddb/decoder.py @@ -7,7 +7,7 @@ """ import struct -from typing import cast, Dict, List, Tuple, Union +from typing import Dict, List, Tuple, Union, cast try: # pylint: disable=unused-import @@ -37,6 +37,7 @@ def __init__( database_buffer -- an mmap'd MaxMind DB file. pointer_base -- the base number to use when decoding a pointer pointer_test -- used for internal unit testing of pointer code + """ self._pointer_test = pointer_test self._buffer = database_buffer @@ -142,6 +143,7 @@ def decode(self, offset: int) -> Tuple[Record, int]: Arguments: offset -- the location of the data structure to decode + """ new_offset = offset + 1 ctrl_byte = self._buffer[offset] @@ -154,7 +156,7 @@ def decode(self, offset: int) -> Tuple[Record, int]: decoder = self._type_decoder[type_num] except KeyError as ex: raise InvalidDatabaseError( - f"Unexpected type number ({type_num}) encountered" + f"Unexpected type number ({type_num}) encountered", ) from ex (size, new_offset) = self._size_from_ctrl_byte(ctrl_byte, new_offset, type_num) @@ -166,7 +168,7 @@ def _read_extended(self, offset: int) -> Tuple[int, int]: if type_num < 7: raise InvalidDatabaseError( "Something went horribly wrong in the decoder. An " - f"extended type resolved to a type number < 8 ({type_num})" + f"extended type resolved to a type number < 8 ({type_num})", ) return type_num, offset + 1 @@ -175,11 +177,14 @@ def _verify_size(expected: int, actual: int) -> None: if expected != actual: raise InvalidDatabaseError( "The MaxMind DB file's data section contains bad data " - "(unknown data type or corrupt data)" + "(unknown data type or corrupt data)", ) def _size_from_ctrl_byte( - self, ctrl_byte: int, offset: int, type_num: int + self, + ctrl_byte: int, + offset: int, + type_num: int, ) -> Tuple[int, int]: size = ctrl_byte & 0x1F if type_num == 1 or size < 29: diff --git a/maxminddb/extension.pyi b/maxminddb/extension.pyi index 56883d2..4cab21a 100644 --- a/maxminddb/extension.pyi +++ b/maxminddb/extension.pyi @@ -6,22 +6,24 @@ This module contains the C extension database reader and related classes. """ +# pylint: disable=E0601,E0602 from ipaddress import IPv4Address, IPv6Address from os import PathLike -from typing import Any, AnyStr, Dict, IO, List, Optional, Tuple, Union -from maxminddb import MODE_AUTO +from typing import IO, Any, AnyStr, Dict, List, Optional, Tuple, Union + from maxminddb.types import Record class Reader: - """ - A C extension implementation of a reader for the MaxMind DB format. IP + """A C extension implementation of a reader for the MaxMind DB format. IP addresses can be looked up using the ``get`` method. """ closed: bool = ... def __init__( - self, database: Union[AnyStr, int, PathLike, IO], mode: int = MODE_AUTO + self, + database: Union[AnyStr, int, PathLike, IO], + mode: int = ..., ) -> None: """Reader for the MaxMind DB file format @@ -30,6 +32,7 @@ class Reader: file, or a file descriptor in the case of MODE_FD. mode -- mode to open the database with. The only supported modes are MODE_AUTO and MODE_MMAP_EXT. + """ def close(self) -> None: @@ -38,25 +41,26 @@ class Reader: def get(self, ip_address: Union[str, IPv6Address, IPv4Address]) -> Optional[Record]: """Return the record for the ip_address in the MaxMind DB - Arguments: ip_address -- an IP address in the standard string notation + """ def get_with_prefix_len( - self, ip_address: Union[str, IPv6Address, IPv4Address] + self, + ip_address: Union[str, IPv6Address, IPv4Address], ) -> Tuple[Optional[Record], int]: """Return a tuple with the record and the associated prefix length - Arguments: ip_address -- an IP address in the standard string notation + """ - def metadata(self) -> "Metadata": + def metadata(self) -> Metadata: """Return the metadata associated with the MaxMind DB file""" - def __enter__(self) -> "Reader": ... + def __enter__(self) -> Reader: ... def __exit__(self, *args) -> None: ... # pylint: disable=too-few-public-methods diff --git a/maxminddb/file.py b/maxminddb/file.py index 4dbecba..516d958 100644 --- a/maxminddb/file.py +++ b/maxminddb/file.py @@ -45,14 +45,14 @@ def close(self) -> None: if hasattr(os, "pread"): def _read(self, buffersize: int, offset: int) -> bytes: - """read that uses pread""" + """Read that uses pread""" # pylint: disable=no-member return os.pread(self._handle.fileno(), buffersize, offset) # type: ignore else: def _read(self, buffersize: int, offset: int) -> bytes: - """read with a lock + """Read with a lock This lock is necessary as after a fork, the different processes will share the same file table entry, even if we dup the fd, and diff --git a/maxminddb/reader.py b/maxminddb/reader.py index 64e3962..290caee 100644 --- a/maxminddb/reader.py +++ b/maxminddb/reader.py @@ -16,9 +16,9 @@ import struct from ipaddress import IPv4Address, IPv6Address from os import PathLike -from typing import Any, AnyStr, Dict, IO, List, Optional, Tuple, Union +from typing import IO, Any, AnyStr, Dict, List, Optional, Tuple, Union -from maxminddb.const import MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY, MODE_FD +from maxminddb.const import MODE_AUTO, MODE_FD, MODE_FILE, MODE_MEMORY, MODE_MMAP from maxminddb.decoder import Decoder from maxminddb.errors import InvalidDatabaseError from maxminddb.file import FileBuffer @@ -44,7 +44,9 @@ class Reader: _ipv4_start: int def __init__( - self, database: Union[AnyStr, int, PathLike, IO], mode: int = MODE_AUTO + self, + database: Union[AnyStr, int, PathLike, IO], + mode: int = MODE_AUTO, ) -> None: """Reader for the MaxMind DB file format @@ -58,6 +60,7 @@ def __init__( * MODE_AUTO - tries MODE_MMAP and then MODE_FILE. Default. * MODE_FD - the param passed via database is a file descriptor, not a path. This mode implies MODE_MEMORY. + """ filename: Any if (mode == MODE_AUTO and mmap) or mode == MODE_MMAP: @@ -83,18 +86,19 @@ def __init__( raise ValueError( f"Unsupported open mode ({mode}). Only MODE_AUTO, MODE_FILE, " "MODE_MEMORY and MODE_FD are supported by the pure Python " - "Reader" + "Reader", ) metadata_start = self._buffer.rfind( - self._METADATA_START_MARKER, max(0, self._buffer_size - 128 * 1024) + self._METADATA_START_MARKER, + max(0, self._buffer_size - 128 * 1024), ) if metadata_start == -1: self.close() raise InvalidDatabaseError( f"Error opening database file ({filename}). " - "Is this a valid MaxMind DB file?" + "Is this a valid MaxMind DB file?", ) metadata_start += len(self._METADATA_START_MARKER) @@ -103,7 +107,7 @@ def __init__( if not isinstance(metadata, dict): raise InvalidDatabaseError( - f"Error reading metadata in database file ({filename})." + f"Error reading metadata in database file ({filename}).", ) self._metadata = Metadata(**metadata) # pylint: disable=bad-option-value @@ -134,21 +138,22 @@ def metadata(self) -> "Metadata": def get(self, ip_address: Union[str, IPv6Address, IPv4Address]) -> Optional[Record]: """Return the record for the ip_address in the MaxMind DB - Arguments: ip_address -- an IP address in the standard string notation + """ (record, _) = self.get_with_prefix_len(ip_address) return record def get_with_prefix_len( - self, ip_address: Union[str, IPv6Address, IPv4Address] + self, + ip_address: Union[str, IPv6Address, IPv4Address], ) -> Tuple[Optional[Record], int]: """Return a tuple with the record and the associated prefix length - Arguments: ip_address -- an IP address in the standard string notation + """ if isinstance(ip_address, str): address = ipaddress.ip_address(ip_address) @@ -163,7 +168,7 @@ def get_with_prefix_len( if address.version == 6 and self._metadata.ip_version == 4: raise ValueError( f"Error looking up {ip_address}. You attempted to look up " - "an IPv6 address in an IPv4-only database." + "an IPv6 address in an IPv4-only database.", ) (pointer, prefix_len) = self._find_address_in_tree(packed_address) @@ -187,7 +192,7 @@ def _generate_children(self, node, depth, ip_acc): if ip_acc <= _IPV4_MAX_NUM and bits == 128: depth -= 96 yield ipaddress.ip_network((ip_acc, depth)), self._resolve_data_pointer( - node + node, ) elif node < node_count: left = self._read_node(node, 0) diff --git a/maxminddb/types.py b/maxminddb/types.py index 03a02e6..9ab76ca 100644 --- a/maxminddb/types.py +++ b/maxminddb/types.py @@ -12,12 +12,8 @@ class RecordList(List[Record]): # pylint: disable=too-few-public-methods - """ - RecordList is a type for lists in a database record. - """ + """RecordList is a type for lists in a database record.""" class RecordDict(Dict[str, Record]): # pylint: disable=too-few-public-methods - """ - RecordDict is a type for dicts in a database record. - """ + """RecordDict is a type for dicts in a database record.""" diff --git a/setup.py b/setup.py index 8b50d18..4e85c73 100644 --- a/setup.py +++ b/setup.py @@ -2,20 +2,23 @@ import re import sys -from setuptools import setup, Extension +from setuptools import Extension, setup from setuptools.command.build_ext import build_ext from wheel.bdist_wheel import bdist_wheel - # These were only added to setuptools in 59.0.1. try: - from setuptools.errors import CCompilerError - from setuptools.errors import DistutilsExecError - from setuptools.errors import DistutilsPlatformError + from setuptools.errors import ( + CCompilerError, + DistutilsExecError, + DistutilsPlatformError, + ) except ImportError: - from distutils.errors import CCompilerError - from distutils.errors import DistutilsExecError - from distutils.errors import DistutilsPlatformError + from distutils.errors import ( + CCompilerError, + DistutilsExecError, + DistutilsPlatformError, + ) cmdclass = {} PYPY = hasattr(sys, "pypy_version_info") @@ -37,7 +40,7 @@ libraries=["maxminddb"] + libraries, sources=["extension/maxminddb.c"], extra_compile_args=compile_args, - ) + ), ] else: ext_module = [ @@ -66,7 +69,7 @@ "extension/libmaxminddb/src", ], extra_compile_args=compile_args, - ) + ), ] # Cargo cult code for installing extension with pure Python fallback. @@ -86,23 +89,22 @@ def run(self): try: build_ext.run(self) except DistutilsPlatformError: - raise BuildFailed() + raise BuildFailed def build_extension(self, ext): try: build_ext.build_extension(self, ext) except ext_errors: - raise BuildFailed() + raise BuildFailed except ValueError: # this can happen on Windows 64 bit, see Python issue 7511 if "'path'" in str(sys.exc_info()[1]): - raise BuildFailed() + raise BuildFailed raise cmdclass["build_ext"] = ve_build_ext -# ROOT = os.path.dirname(__file__) @@ -112,7 +114,9 @@ def build_extension(self, ext): with open(os.path.join(ROOT, "maxminddb", "__init__.py"), "rb") as fd: maxminddb_text = fd.read().decode("utf8") VERSION = ( - re.compile(r".*__version__ = \"(.*?)\"", re.S).match(maxminddb_text).group(1) + re.compile(r".*__version__ = \"(.*?)\"", re.DOTALL) + .match(maxminddb_text) + .group(1) ) diff --git a/tests/decoder_test.py b/tests/decoder_test.py index d55f03f..de3c999 100644 --- a/tests/decoder_test.py +++ b/tests/decoder_test.py @@ -1,12 +1,10 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- import mmap +import unittest from maxminddb.decoder import Decoder -import unittest - class TestDecoder(unittest.TestCase): def test_arrays(self): diff --git a/tests/reader_test.py b/tests/reader_test.py index 315200d..bce6201 100644 --- a/tests/reader_test.py +++ b/tests/reader_test.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- import ipaddress import multiprocessing @@ -7,9 +6,8 @@ import pathlib import threading import unittest -import unittest.mock as mock -from multiprocessing import Process, Pipe -from typing import Union, Type +from typing import Type, Union +from unittest import mock import maxminddb @@ -18,14 +16,14 @@ except ImportError: maxminddb.extension = None # type: ignore -from maxminddb import open_database, InvalidDatabaseError +from maxminddb import InvalidDatabaseError, open_database from maxminddb.const import ( MODE_AUTO, - MODE_MMAP_EXT, - MODE_MMAP, + MODE_FD, MODE_FILE, MODE_MEMORY, - MODE_FD, + MODE_MMAP, + MODE_MMAP_EXT, ) @@ -41,9 +39,10 @@ def get_reader_from_file_descriptor(filepath, mode): return maxminddb.open_database(filepath, mode) -class BaseTestReader(object): +class BaseTestReader: readerClass: Union[ - Type["maxminddb.extension.Reader"], Type["maxminddb.reader.Reader"] + Type["maxminddb.extension.Reader"], + Type["maxminddb.reader.Reader"], ] use_ip_objects = False @@ -175,7 +174,8 @@ def test_get_with_prefix_len(self): for test in tests: with open_database( - "tests/data/test-data/" + test["file_name"], self.mode + "tests/data/test-data/" + test["file_name"], + self.mode, ) as reader: (record, prefix_len) = reader.get_with_prefix_len(test["ip"]) @@ -241,7 +241,8 @@ def test_iterator(self): def test_decoder(self): reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) record = reader.get(self.ipf("::1.1.1.0")) @@ -268,7 +269,8 @@ def test_decoder(self): def test_metadata_pointers(self): with open_database( - "tests/data/test-data/MaxMind-DB-test-metadata-pointers.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-metadata-pointers.mmdb", + self.mode, ) as reader: self.assertEqual( "Lots of pointers in metadata", @@ -277,7 +279,8 @@ def test_metadata_pointers(self): def test_no_ipv4_search_tree(self): reader = open_database( - "tests/data/test-data/MaxMind-DB-no-ipv4-search-tree.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-no-ipv4-search-tree.mmdb", + self.mode, ) self.assertEqual(reader.get(self.ipf("1.1.1.1")), "::0/64") @@ -286,7 +289,8 @@ def test_no_ipv4_search_tree(self): def test_ipv6_address_in_ipv4_database(self): reader = open_database( - "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", + self.mode, ) with self.assertRaisesRegex( ValueError, @@ -299,7 +303,8 @@ def test_ipv6_address_in_ipv4_database(self): def test_opening_path(self): with open_database( - pathlib.Path("tests/data/test-data/MaxMind-DB-test-decoder.mmdb"), self.mode + pathlib.Path("tests/data/test-data/MaxMind-DB-test-decoder.mmdb"), + self.mode, ) as reader: self.assertEqual(reader.metadata().database_type, "MaxMind DB Decoder Test") @@ -311,13 +316,14 @@ def test_no_extension_exception(self): "MODE_MMAP_EXT requires the maxminddb.extension module to be available", ): open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", MODE_MMAP_EXT + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + MODE_MMAP_EXT, ) maxminddb._extension = real_extension def test_broken_database(self): reader = open_database( - "tests/data/test-data/" "GeoIP2-City-Test-Broken-Double-Format.mmdb", + "tests/data/test-data/GeoIP2-City-Test-Broken-Double-Format.mmdb", self.mode, ) with self.assertRaisesRegex( @@ -331,10 +337,12 @@ def test_broken_database(self): def test_ip_validation(self): reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) with self.assertRaisesRegex( - ValueError, "'not_ip' does not appear to be an IPv4 or " "IPv6 address" + ValueError, + "'not_ip' does not appear to be an IPv4 or IPv6 address", ): reader.get("not_ip") reader.close() @@ -380,7 +388,8 @@ def test_no_constructor_args(self): def test_too_many_get_args(self): reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) with self.assertRaises(TypeError): reader.get(self.ipf("1.1.1.1"), "blah") @@ -388,7 +397,8 @@ def test_too_many_get_args(self): def test_no_get_args(self): reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) with self.assertRaises(TypeError): reader.get() @@ -397,14 +407,16 @@ def test_no_get_args(self): def test_incorrect_get_arg_type(self): reader = open_database("tests/data/test-data/GeoIP2-City-Test.mmdb", self.mode) with self.assertRaisesRegex( - TypeError, "argument 1 must be a string or ipaddress object" + TypeError, + "argument 1 must be a string or ipaddress object", ): reader.get(1) reader.close() def test_metadata_args(self): reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) with self.assertRaises(TypeError): reader.metadata("blah") @@ -412,24 +424,28 @@ def test_metadata_args(self): def test_metadata_unknown_attribute(self): reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) metadata = reader.metadata() with self.assertRaisesRegex( - AttributeError, "'Metadata' object has no " "attribute 'blah'" + AttributeError, + "'Metadata' object has no attribute 'blah'", ): metadata.blah reader.close() def test_close(self): reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) reader.close() def test_double_close(self): reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) reader.close() self.assertIsNone(reader.close(), "Double close does not throw an exception") @@ -438,11 +454,13 @@ def test_closed_get(self): if self.mode in [MODE_MEMORY, MODE_FD]: return reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) reader.close() with self.assertRaisesRegex( - ValueError, "Attempt to read from a closed MaxMind DB.|closed" + ValueError, + "Attempt to read from a closed MaxMind DB.|closed", ): reader.get(self.ipf("1.1.1.1")) @@ -458,14 +476,16 @@ def test_with_statement_close(self): reader.close() with self.assertRaisesRegex( - ValueError, "Attempt to reopen a closed MaxMind DB" + ValueError, + "Attempt to reopen a closed MaxMind DB", ): with reader: pass def test_closed(self): reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) self.assertEqual(reader.closed, False) reader.close() @@ -477,7 +497,8 @@ def test_closed(self): # to keep the metadata in memory. def test_closed_metadata(self): reader = open_database( - "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode + "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", + self.mode, ) reader.close() @@ -485,7 +506,7 @@ def test_closed_metadata(self): # segfault try: metadata = reader.metadata() - except IOError as ex: + except OSError as ex: self.assertEqual( "Attempt to read from a closed MaxMind DB.", str(ex), @@ -504,7 +525,8 @@ def test_threading(self): def _check_concurrency(self, worker_class): reader = open_database( - "tests/data/test-data/GeoIP2-Domain-Test.mmdb", self.mode + "tests/data/test-data/GeoIP2-Domain-Test.mmdb", + self.mode, ) def lookup(pipe): @@ -541,7 +563,8 @@ def _check_metadata(self, reader, ip_version, record_size): self.assertEqual(metadata.database_type, "Test") self.assertEqual( - {"en": "Test Database", "zh": "Test Database Chinese"}, metadata.description + {"en": "Test Database", "zh": "Test Database Chinese"}, + metadata.description, ) self.assertEqual(metadata.ip_version, ip_version) self.assertEqual(metadata.languages, ["en", "zh"]) @@ -642,7 +665,8 @@ class TestAutoReader(BaseTestReader, unittest.TestCase): mode = MODE_AUTO readerClass: Union[ - Type["maxminddb.extension.Reader"], Type["maxminddb.reader.Reader"] + Type["maxminddb.extension.Reader"], + Type["maxminddb.reader.Reader"], ] if has_maxminddb_extension(): readerClass = maxminddb.extension.Reader From 147f44e49dfbfc2afe84a0079463e6f975548676 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 29 Jan 2025 11:44:21 -0800 Subject: [PATCH 4/8] Apply unsafe fixes from ruff --- examples/benchmark.py | 4 ++-- maxminddb/__init__.py | 2 +- maxminddb/const.py | 2 +- maxminddb/decoder.py | 6 +++--- maxminddb/extension.pyi | 6 +++--- maxminddb/file.py | 12 ++++++------ maxminddb/reader.py | 29 ++++++++++++++++------------- setup.py | 22 ++++++++++------------ tests/reader_test.py | 15 +++++++-------- 9 files changed, 49 insertions(+), 49 deletions(-) diff --git a/examples/benchmark.py b/examples/benchmark.py index 90e32fa..c07a5d1 100644 --- a/examples/benchmark.py +++ b/examples/benchmark.py @@ -19,9 +19,9 @@ reader = maxminddb.open_database(args.file, args.mode) -def lookup_ip_address(): +def lookup_ip_address() -> None: ip = socket.inet_ntoa(struct.pack("!L", random.getrandbits(32))) - record = reader.get(str(ip)) + reader.get(str(ip)) elapsed = timeit.timeit( diff --git a/maxminddb/__init__.py b/maxminddb/__init__.py index 73f0d79..8e83734 100644 --- a/maxminddb/__init__.py +++ b/maxminddb/__init__.py @@ -37,7 +37,7 @@ def open_database( database: Union[AnyStr, int, os.PathLike, IO], mode: int = MODE_AUTO, ) -> Reader: - """Open a MaxMind DB database + """Open a MaxMind DB database. Arguments: database -- A path to a valid MaxMind DB file such as a GeoIP2 database diff --git a/maxminddb/const.py b/maxminddb/const.py index 45222c2..959078d 100644 --- a/maxminddb/const.py +++ b/maxminddb/const.py @@ -1,4 +1,4 @@ -"""Constants used in the API""" +"""Constants used in the API.""" MODE_AUTO = 0 MODE_MMAP_EXT = 1 diff --git a/maxminddb/decoder.py b/maxminddb/decoder.py index fae91d3..4237e2b 100644 --- a/maxminddb/decoder.py +++ b/maxminddb/decoder.py @@ -23,7 +23,7 @@ class Decoder: # pylint: disable=too-few-public-methods - """Decoder for the data section of the MaxMind DB""" + """Decoder for the data section of the MaxMind DB.""" def __init__( self, @@ -31,7 +31,7 @@ def __init__( pointer_base: int = 0, pointer_test: bool = False, ) -> None: - """Created a Decoder for a MaxMind DB + """Created a Decoder for a MaxMind DB. Arguments: database_buffer -- an mmap'd MaxMind DB file. @@ -139,7 +139,7 @@ def _decode_utf8_string(self, size: int, offset: int) -> Tuple[str, int]: } def decode(self, offset: int) -> Tuple[Record, int]: - """Decode a section of the data section starting at offset + """Decode a section of the data section starting at offset. Arguments: offset -- the location of the data structure to decode diff --git a/maxminddb/extension.pyi b/maxminddb/extension.pyi index 4cab21a..8507311 100644 --- a/maxminddb/extension.pyi +++ b/maxminddb/extension.pyi @@ -9,7 +9,7 @@ This module contains the C extension database reader and related classes. # pylint: disable=E0601,E0602 from ipaddress import IPv4Address, IPv6Address from os import PathLike -from typing import IO, Any, AnyStr, Dict, List, Optional, Tuple, Union +from typing import IO, Any, AnyStr, Optional, Tuple, Union from maxminddb.types import Record @@ -89,7 +89,7 @@ class Metadata: A string identifying the database type, e.g., "GeoIP2-City". """ - description: Dict[str, str] + description: dict[str, str] """ A map from locales to text descriptions of the database. """ @@ -101,7 +101,7 @@ class Metadata: both IPv4 and IPv6 lookups. """ - languages: List[str] + languages: list[str] """ A list of locale codes supported by the databse. """ diff --git a/maxminddb/file.py b/maxminddb/file.py index 516d958..541e730 100644 --- a/maxminddb/file.py +++ b/maxminddb/file.py @@ -11,7 +11,7 @@ class FileBuffer: - """A slice-able file reader""" + """A slice-able file reader.""" def __init__(self, database: str) -> None: # pylint: disable=consider-using-with @@ -28,31 +28,31 @@ def __getitem__(self, key: Union[slice, int]): raise TypeError("Invalid argument type.") def rfind(self, needle: bytes, start: int) -> int: - """Reverse find needle from start""" + """Reverse find needle from start.""" pos = self._read(self._size - start - 1, start).rfind(needle) if pos == -1: return pos return start + pos def size(self) -> int: - """Size of file""" + """Size of file.""" return self._size def close(self) -> None: - """Close file""" + """Close file.""" self._handle.close() if hasattr(os, "pread"): def _read(self, buffersize: int, offset: int) -> bytes: - """Read that uses pread""" + """Read that uses pread.""" # pylint: disable=no-member return os.pread(self._handle.fileno(), buffersize, offset) # type: ignore else: def _read(self, buffersize: int, offset: int) -> bytes: - """Read with a lock + """Read with a lock. This lock is necessary as after a fork, the different processes will share the same file table entry, even if we dup the fd, and diff --git a/maxminddb/reader.py b/maxminddb/reader.py index 290caee..7feef2c 100644 --- a/maxminddb/reader.py +++ b/maxminddb/reader.py @@ -48,7 +48,7 @@ def __init__( database: Union[AnyStr, int, PathLike, IO], mode: int = MODE_AUTO, ) -> None: - """Reader for the MaxMind DB file format + """Reader for the MaxMind DB file format. Arguments: database -- A path to a valid MaxMind DB file such as a GeoIP2 database @@ -132,11 +132,11 @@ def __init__( self._ipv4_start = ipv4_start def metadata(self) -> "Metadata": - """Return the metadata associated with the MaxMind DB file""" + """Return the metadata associated with the MaxMind DB file.""" return self._metadata def get(self, ip_address: Union[str, IPv6Address, IPv4Address]) -> Optional[Record]: - """Return the record for the ip_address in the MaxMind DB + """Return the record for the ip_address in the MaxMind DB. Arguments: ip_address -- an IP address in the standard string notation @@ -149,7 +149,7 @@ def get_with_prefix_len( self, ip_address: Union[str, IPv6Address, IPv4Address], ) -> Tuple[Optional[Record], int]: - """Return a tuple with the record and the associated prefix length + """Return a tuple with the record and the associated prefix length. Arguments: ip_address -- an IP address in the standard string notation @@ -245,20 +245,22 @@ def _read_node(self, node_number: int, index: int) -> int: offset = base_offset + index * 4 node_bytes = self._buffer[offset : offset + 4] else: - raise InvalidDatabaseError(f"Unknown record size: {record_size}") + msg = f"Unknown record size: {record_size}" + raise InvalidDatabaseError(msg) return struct.unpack(b"!I", node_bytes)[0] def _resolve_data_pointer(self, pointer: int) -> Record: resolved = pointer - self._metadata.node_count + self._metadata.search_tree_size if resolved >= self._buffer_size: - raise InvalidDatabaseError("The MaxMind DB file's search tree is corrupt") + msg = "The MaxMind DB file's search tree is corrupt" + raise InvalidDatabaseError(msg) (data, _) = self._decoder.decode(resolved) return data def close(self) -> None: - """Closes the MaxMind DB file and returns the resources to the system""" + """Closes the MaxMind DB file and returns the resources to the system.""" try: self._buffer.close() # type: ignore except AttributeError: @@ -270,13 +272,14 @@ def __exit__(self, *args) -> None: def __enter__(self) -> "Reader": if self.closed: - raise ValueError("Attempt to reopen a closed MaxMind DB") + msg = "Attempt to reopen a closed MaxMind DB" + raise ValueError(msg) return self # pylint: disable=too-many-instance-attributes,R0801 class Metadata: - """Metadata for the MaxMind DB reader""" + """Metadata for the MaxMind DB reader.""" binary_format_major_version: int """ @@ -328,7 +331,7 @@ class Metadata: """ def __init__(self, **kwargs) -> None: - """Creates new Metadata object. kwargs are key/value pairs from spec""" + """Creates new Metadata object. kwargs are key/value pairs from spec.""" # Although I could just update __dict__, that is less obvious and it # doesn't work well with static analysis tools and some IDEs self.node_count = kwargs["node_count"] @@ -343,7 +346,7 @@ def __init__(self, **kwargs) -> None: @property def node_byte_size(self) -> int: - """The size of a node in bytes + """The size of a node in bytes. :type: int """ @@ -351,12 +354,12 @@ def node_byte_size(self) -> int: @property def search_tree_size(self) -> int: - """The size of the search tree + """The size of the search tree. :type: int """ return self.node_count * self.node_byte_size - def __repr__(self): + def __repr__(self) -> str: args = ", ".join(f"{k}={v!r}" for k, v in self.__dict__.items()) return f"{self.__module__}.{self.__class__.__name__}({args})" diff --git a/setup.py b/setup.py index 4e85c73..7ee1885 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ ext_module = [ Extension( "maxminddb.extension", - libraries=["maxminddb"] + libraries, + libraries=["maxminddb", *libraries], sources=["extension/maxminddb.c"], extra_compile_args=compile_args, ), @@ -78,20 +78,20 @@ class BuildFailed(Exception): - def __init__(self): + def __init__(self) -> None: self.cause = sys.exc_info()[1] class ve_build_ext(build_ext): # This class allows C extension building to fail. - def run(self): + def run(self) -> None: try: build_ext.run(self) except DistutilsPlatformError: raise BuildFailed - def build_extension(self, ext): + def build_extension(self, ext) -> None: try: build_ext.build_extension(self, ext) except ext_errors: @@ -120,24 +120,22 @@ def build_extension(self, ext): ) -def status_msgs(*msgs): - print("*" * 75) - for msg in msgs: - print(msg) - print("*" * 75) +def status_msgs(*msgs) -> None: + for _msg in msgs: + pass def find_packages(location): packages = [] for pkg in ["maxminddb"]: - for _dir, subdirectories, files in os.walk(os.path.join(location, pkg)): + for _dir, _subdirectories, files in os.walk(os.path.join(location, pkg)): if "__init__.py" in files: tokens = _dir.split(os.sep)[len(location.split(os.sep)) :] packages.append(".".join(tokens)) return packages -def run_setup(with_cext): +def run_setup(with_cext) -> None: kwargs = {} loc_cmdclass = cmdclass.copy() if with_cext: @@ -158,7 +156,7 @@ def run_setup(with_cext): run_setup(True) except BuildFailed as exc: if os.getenv("MAXMINDDB_REQUIRE_EXTENSION"): - raise exc + raise status_msgs( exc.cause, "WARNING: The C extension could not be compiled, " diff --git a/tests/reader_test.py b/tests/reader_test.py index bce6201..31bf585 100644 --- a/tests/reader_test.py +++ b/tests/reader_test.py @@ -478,9 +478,8 @@ def test_with_statement_close(self): with self.assertRaisesRegex( ValueError, "Attempt to reopen a closed MaxMind DB", - ): - with reader: - pass + ), reader: + pass def test_closed(self): reader = open_database( @@ -523,13 +522,13 @@ def test_multiprocessing(self): def test_threading(self): self._check_concurrency(threading.Thread) - def _check_concurrency(self, worker_class): + def _check_concurrency(self, worker_class) -> None: reader = open_database( "tests/data/test-data/GeoIP2-Domain-Test.mmdb", self.mode, ) - def lookup(pipe): + def lookup(pipe) -> None: try: for i in range(32): reader.get(self.ipf(f"65.115.240.{i}")) @@ -554,7 +553,7 @@ def lookup(pipe): self.assertEqual(count, 32, "expected number of successful lookups") - def _check_metadata(self, reader, ip_version, record_size): + def _check_metadata(self, reader, ip_version, record_size) -> None: metadata = reader.metadata() self.assertEqual(2, metadata.binary_format_major_version, "major version") @@ -572,7 +571,7 @@ def _check_metadata(self, reader, ip_version, record_size): self.assertEqual(metadata.record_size, record_size) - def _check_ip_v4(self, reader, file_name): + def _check_ip_v4(self, reader, file_name) -> None: for i in range(6): address = "1.1.1." + str(pow(2, i)) self.assertEqual( @@ -602,7 +601,7 @@ def _check_ip_v4(self, reader, file_name): for ip in ["1.1.1.33", "255.254.253.123"]: self.assertIsNone(reader.get(self.ipf(ip))) - def _check_ip_v6(self, reader, file_name): + def _check_ip_v6(self, reader, file_name) -> None: subnets = ["::1:ffff:ffff", "::2:0:0", "::2:0:40", "::2:0:50", "::2:0:58"] for address in subnets: From 0311fcd59771ef7593db1108cb91bc8970947561 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 29 Jan 2025 11:49:16 -0800 Subject: [PATCH 5/8] Add missing types --- maxminddb/reader.py | 6 +- tests/decoder_test.py | 34 +++++------ tests/reader_test.py | 132 +++++++++++++++++++++++------------------- 3 files changed, 92 insertions(+), 80 deletions(-) diff --git a/maxminddb/reader.py b/maxminddb/reader.py index 7feef2c..403139c 100644 --- a/maxminddb/reader.py +++ b/maxminddb/reader.py @@ -16,7 +16,7 @@ import struct from ipaddress import IPv4Address, IPv6Address from os import PathLike -from typing import IO, Any, AnyStr, Dict, List, Optional, Tuple, Union +from typing import IO, Any, AnyStr, Dict, Iterator, List, Optional, Tuple, Union from maxminddb.const import MODE_AUTO, MODE_FD, MODE_FILE, MODE_MEMORY, MODE_MMAP from maxminddb.decoder import Decoder @@ -177,10 +177,10 @@ def get_with_prefix_len( return self._resolve_data_pointer(pointer), prefix_len return None, prefix_len - def __iter__(self): + def __iter__(self) -> Iterator: return self._generate_children(0, 0, 0) - def _generate_children(self, node, depth, ip_acc): + def _generate_children(self, node, depth, ip_acc) -> Iterator: if ip_acc != 0 and node == self._ipv4_start: # Skip nodes aliased to IPv4 return diff --git a/tests/decoder_test.py b/tests/decoder_test.py index de3c999..e965643 100644 --- a/tests/decoder_test.py +++ b/tests/decoder_test.py @@ -7,7 +7,7 @@ class TestDecoder(unittest.TestCase): - def test_arrays(self): + def test_arrays(self) -> None: arrays = { b"\x00\x04": [], b"\x01\x04\x43\x46\x6f\x6f": ["Foo"], @@ -15,14 +15,14 @@ def test_arrays(self): } self.validate_type_decoding("arrays", arrays) - def test_boolean(self): + def test_boolean(self) -> None: booleans = { b"\x00\x07": False, b"\x01\x07": True, } self.validate_type_decoding("booleans", booleans) - def test_double(self): + def test_double(self) -> None: doubles = { b"\x68\x00\x00\x00\x00\x00\x00\x00\x00": 0.0, b"\x68\x3f\xe0\x00\x00\x00\x00\x00\x00": 0.5, @@ -35,7 +35,7 @@ def test_double(self): } self.validate_type_decoding("double", doubles) - def test_float(self): + def test_float(self) -> None: floats = { b"\x04\x08\x00\x00\x00\x00": 0.0, b"\x04\x08\x3f\x80\x00\x00": 1.0, @@ -49,7 +49,7 @@ def test_float(self): } self.validate_type_decoding("float", floats) - def test_int32(self): + def test_int32(self) -> None: int32 = { b"\x00\x01": 0, b"\x04\x01\xff\xff\xff\xff": -1, @@ -66,7 +66,7 @@ def test_int32(self): } self.validate_type_decoding("int32", int32) - def test_map(self): + def test_map(self) -> None: maps = { b"\xe0": {}, b"\xe1\x42\x65\x6e\x43\x46\x6f\x6f": {"en": "Foo"}, @@ -85,7 +85,7 @@ def test_map(self): } self.validate_type_decoding("maps", maps) - def test_pointer(self): + def test_pointer(self) -> None: pointers = { b"\x20\x00": 0, b"\x20\x05": 5, @@ -133,17 +133,17 @@ def test_pointer(self): b"\x5f\x00\x10\x53" + 70000 * b"\x78": "x" * 70000, } - def test_string(self): + def test_string(self) -> None: self.validate_type_decoding("string", self.strings) - def test_byte(self): + def test_byte(self) -> None: b = { bytes([0xC0 ^ k[0]]) + k[1:]: v.encode("utf-8") for k, v in self.strings.items() } self.validate_type_decoding("byte", b) - def test_uint16(self): + def test_uint16(self) -> None: uint16 = { b"\xa0": 0, b"\xa1\xff": 255, @@ -153,7 +153,7 @@ def test_uint16(self): } self.validate_type_decoding("uint16", uint16) - def test_uint32(self): + def test_uint32(self) -> None: uint32 = { b"\xc0": 0, b"\xc1\xff": 255, @@ -165,7 +165,7 @@ def test_uint32(self): } self.validate_type_decoding("uint32", uint32) - def generate_large_uint(self, bits): + def generate_large_uint(self, bits) -> dict: ctrl_byte = b"\x02" if bits == 64 else b"\x03" uints = { b"\x00" + ctrl_byte: 0, @@ -178,17 +178,17 @@ def generate_large_uint(self, bits): uints[input] = expected return uints - def test_uint64(self): + def test_uint64(self) -> None: self.validate_type_decoding("uint64", self.generate_large_uint(64)) - def test_uint128(self): + def test_uint128(self) -> None: self.validate_type_decoding("uint128", self.generate_large_uint(128)) - def validate_type_decoding(self, type, tests): + def validate_type_decoding(self, type, tests) -> None: for input, expected in tests.items(): self.check_decoding(type, input, expected) - def check_decoding(self, type, input, expected, name=None): + def check_decoding(self, type, input, expected, name=None) -> None: name = name or expected db = mmap.mmap(-1, len(input)) db.write(input) @@ -204,7 +204,7 @@ def check_decoding(self, type, input, expected, name=None): else: self.assertEqual(expected, actual, type) - def test_real_pointers(self): + def test_real_pointers(self) -> None: with open("tests/data/test-data/maps-with-pointers.raw", "r+b") as db_file: mm = mmap.mmap(db_file.fileno(), 0) decoder = Decoder(mm, 0) diff --git a/tests/reader_test.py b/tests/reader_test.py index 31bf585..6bd1be9 100644 --- a/tests/reader_test.py +++ b/tests/reader_test.py @@ -6,7 +6,7 @@ import pathlib import threading import unittest -from typing import Type, Union +from typing import Type, Union, cast from unittest import mock import maxminddb @@ -25,9 +25,10 @@ MODE_MMAP, MODE_MMAP_EXT, ) +from maxminddb.reader import Reader -def get_reader_from_file_descriptor(filepath, mode): +def get_reader_from_file_descriptor(filepath, mode) -> Reader: """Patches open_database() for class TestFDReader().""" if mode == MODE_FD: with open(filepath, "rb") as mmdb_fh: @@ -39,7 +40,8 @@ def get_reader_from_file_descriptor(filepath, mode): return maxminddb.open_database(filepath, mode) -class BaseTestReader: +class BaseTestReader(unittest.TestCase): + mode: int readerClass: Union[ Type["maxminddb.extension.Reader"], Type["maxminddb.reader.Reader"], @@ -51,12 +53,12 @@ class BaseTestReader: if os.name != "nt": mp = multiprocessing.get_context("fork") - def ipf(self, ip): + def ipf(self, ip) -> Union[ipaddress.IPv4Address, ipaddress.IPv6Address]: if self.use_ip_objects: return ipaddress.ip_address(ip) return ip - def test_reader(self): + def test_reader(self) -> None: for record_size in [24, 28, 32]: for ip_version in [4, 6]: file_name = ( @@ -76,7 +78,7 @@ def test_reader(self): self._check_ip_v6(reader, file_name) reader.close() - def test_get_with_prefix_len(self): + def test_get_with_prefix_len(self) -> None: decoder_record = { "array": [1, 2, 3], "boolean": True, @@ -174,10 +176,10 @@ def test_get_with_prefix_len(self): for test in tests: with open_database( - "tests/data/test-data/" + test["file_name"], + "tests/data/test-data/" + cast(str, test["file_name"]), self.mode, ) as reader: - (record, prefix_len) = reader.get_with_prefix_len(test["ip"]) + (record, prefix_len) = reader.get_with_prefix_len(cast(str, test["ip"])) self.assertEqual( prefix_len, @@ -188,10 +190,13 @@ def test_get_with_prefix_len(self): self.assertEqual( record, test["expected_record"], - "expected_record for " + test["ip"] + " in " + test["file_name"], + "expected_record for " + + cast(str, test["ip"]) + + " in " + + cast(str, test["file_name"]), ) - def test_iterator(self): + def test_iterator(self) -> None: tests = ( { "database": "ipv4", @@ -239,12 +244,12 @@ def test_iterator(self): networks = [str(n) for (n, _) in reader] self.assertEqual(networks, test["expected"], f) - def test_decoder(self): + def test_decoder(self) -> None: reader = open_database( "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode, ) - record = reader.get(self.ipf("::1.1.1.0")) + record = cast(dict, reader.get(self.ipf("::1.1.1.0"))) self.assertEqual(record["array"], [1, 2, 3]) self.assertEqual(record["boolean"], True) @@ -267,7 +272,7 @@ def test_decoder(self): self.assertEqual(1329227995784915872903807060280344576, record["uint128"]) reader.close() - def test_metadata_pointers(self): + def test_metadata_pointers(self) -> None: with open_database( "tests/data/test-data/MaxMind-DB-test-metadata-pointers.mmdb", self.mode, @@ -277,7 +282,7 @@ def test_metadata_pointers(self): reader.metadata().database_type, ) - def test_no_ipv4_search_tree(self): + def test_no_ipv4_search_tree(self) -> None: reader = open_database( "tests/data/test-data/MaxMind-DB-no-ipv4-search-tree.mmdb", self.mode, @@ -287,7 +292,7 @@ def test_no_ipv4_search_tree(self): self.assertEqual(reader.get(self.ipf("192.1.1.1")), "::0/64") reader.close() - def test_ipv6_address_in_ipv4_database(self): + def test_ipv6_address_in_ipv4_database(self) -> None: reader = open_database( "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", self.mode, @@ -301,16 +306,16 @@ def test_ipv6_address_in_ipv4_database(self): reader.get(self.ipf("2001::")) reader.close() - def test_opening_path(self): + def test_opening_path(self) -> None: with open_database( pathlib.Path("tests/data/test-data/MaxMind-DB-test-decoder.mmdb"), self.mode, ) as reader: self.assertEqual(reader.metadata().database_type, "MaxMind DB Decoder Test") - def test_no_extension_exception(self): + def test_no_extension_exception(self) -> None: real_extension = maxminddb._extension - maxminddb._extension = None + maxminddb._extension = None # type: ignore with self.assertRaisesRegex( ValueError, "MODE_MMAP_EXT requires the maxminddb.extension module to be available", @@ -321,7 +326,7 @@ def test_no_extension_exception(self): ) maxminddb._extension = real_extension - def test_broken_database(self): + def test_broken_database(self) -> None: reader = open_database( "tests/data/test-data/GeoIP2-City-Test-Broken-Double-Format.mmdb", self.mode, @@ -335,7 +340,7 @@ def test_broken_database(self): reader.get(self.ipf("2001:220::")) reader.close() - def test_ip_validation(self): + def test_ip_validation(self) -> None: reader = open_database( "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode, @@ -347,11 +352,11 @@ def test_ip_validation(self): reader.get("not_ip") reader.close() - def test_missing_database(self): + def test_missing_database(self) -> None: with self.assertRaisesRegex(FileNotFoundError, "No such file or directory"): open_database("file-does-not-exist.mmdb", self.mode) - def test_nondatabase(self): + def test_nondatabase(self) -> None: with self.assertRaisesRegex( InvalidDatabaseError, r"Error opening database file \(README.rst\). " @@ -360,7 +365,7 @@ def test_nondatabase(self): open_database("README.rst", self.mode) # This is from https://github.com/maxmind/MaxMind-DB-Reader-python/issues/58 - def test_database_with_invalid_utf8_key(self): + def test_database_with_invalid_utf8_key(self) -> None: reader = open_database( "tests/data/bad-data/maxminddb-python/bad-unicode-in-map-key.mmdb", self.mode, @@ -368,15 +373,15 @@ def test_database_with_invalid_utf8_key(self): with self.assertRaises(UnicodeDecodeError): reader.get_with_prefix_len("163.254.149.39") - def test_too_many_constructor_args(self): + def test_too_many_constructor_args(self) -> None: with self.assertRaises(TypeError): - self.readerClass("README.md", self.mode, 1) + self.readerClass("README.md", self.mode, 1) # type: ignore - def test_bad_constructor_mode(self): + def test_bad_constructor_mode(self) -> None: with self.assertRaisesRegex(ValueError, r"Unsupported open mode \(100\)"): - self.readerClass("README.md", mode=100) + self.readerClass("README.md", mode=100) # type: ignore - def test_no_constructor_args(self): + def test_no_constructor_args(self) -> None: with self.assertRaisesRegex( TypeError, r" 1 required positional argument|" @@ -384,45 +389,45 @@ def test_no_constructor_args(self): r"takes at least 2 arguments|" r"function missing required argument \'database\' \(pos 1\)", ): - self.readerClass() + self.readerClass() # type: ignore - def test_too_many_get_args(self): + def test_too_many_get_args(self) -> None: reader = open_database( "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode, ) with self.assertRaises(TypeError): - reader.get(self.ipf("1.1.1.1"), "blah") + reader.get(self.ipf("1.1.1.1"), "blah") # type: ignore reader.close() - def test_no_get_args(self): + def test_no_get_args(self) -> None: reader = open_database( "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode, ) with self.assertRaises(TypeError): - reader.get() + reader.get() # type: ignore reader.close() - def test_incorrect_get_arg_type(self): + def test_incorrect_get_arg_type(self) -> None: reader = open_database("tests/data/test-data/GeoIP2-City-Test.mmdb", self.mode) with self.assertRaisesRegex( TypeError, "argument 1 must be a string or ipaddress object", ): - reader.get(1) + reader.get(1) # type: ignore reader.close() - def test_metadata_args(self): + def test_metadata_args(self) -> None: reader = open_database( "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode, ) with self.assertRaises(TypeError): - reader.metadata("blah") + reader.metadata("blah") # type: ignore reader.close() - def test_metadata_unknown_attribute(self): + def test_metadata_unknown_attribute(self) -> None: reader = open_database( "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode, @@ -432,25 +437,27 @@ def test_metadata_unknown_attribute(self): AttributeError, "'Metadata' object has no attribute 'blah'", ): - metadata.blah + metadata.blah # type: ignore reader.close() - def test_close(self): + def test_close(self) -> None: reader = open_database( "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode, ) reader.close() - def test_double_close(self): + def test_double_close(self) -> None: reader = open_database( "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode, ) reader.close() - self.assertIsNone(reader.close(), "Double close does not throw an exception") + self.assertIsNone( + reader.close(), "Double close does not throw an exception" # type: ignore + ) - def test_closed_get(self): + def test_closed_get(self) -> None: if self.mode in [MODE_MEMORY, MODE_FD]: return reader = open_database( @@ -464,13 +471,13 @@ def test_closed_get(self): ): reader.get(self.ipf("1.1.1.1")) - def test_with_statement(self): + def test_with_statement(self) -> None: filename = "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb" with open_database(filename, self.mode) as reader: self._check_ip_v4(reader, filename) self.assertEqual(reader.closed, True) - def test_with_statement_close(self): + def test_with_statement_close(self) -> None: filename = "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb" reader = open_database(filename, self.mode) reader.close() @@ -481,7 +488,7 @@ def test_with_statement_close(self): ), reader: pass - def test_closed(self): + def test_closed(self) -> None: reader = open_database( "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode, @@ -494,7 +501,7 @@ def test_closed(self): # extension and the pure Python reader. If we do, the pure Python # reader will need to throw an exception or the extension will need # to keep the metadata in memory. - def test_closed_metadata(self): + def test_closed_metadata(self) -> None: reader = open_database( "tests/data/test-data/MaxMind-DB-test-decoder.mmdb", self.mode, @@ -633,15 +640,17 @@ def _check_ip_v6(self, reader, file_name) -> None: self.assertIsNone(reader.get(self.ipf(ip))) -def has_maxminddb_extension(): - return maxminddb.extension and hasattr(maxminddb.extension, "Reader") +def has_maxminddb_extension() -> bool: + return maxminddb.extension and hasattr( + maxminddb.extension, "Reader" + ) # type: ignore @unittest.skipIf( not has_maxminddb_extension() and not os.environ.get("MM_FORCE_EXT_TESTS"), "No C extension module found. Skipping tests", ) -class TestExtensionReader(BaseTestReader, unittest.TestCase): +class TestExtensionReader(BaseTestReader): mode = MODE_MMAP_EXT if has_maxminddb_extension(): @@ -652,7 +661,7 @@ class TestExtensionReader(BaseTestReader, unittest.TestCase): not has_maxminddb_extension() and not os.environ.get("MM_FORCE_EXT_TESTS"), "No C extension module found. Skipping tests", ) -class TestExtensionReaderWithIPObjects(BaseTestReader, unittest.TestCase): +class TestExtensionReaderWithIPObjects(BaseTestReader): mode = MODE_MMAP_EXT use_ip_objects = True @@ -660,7 +669,7 @@ class TestExtensionReaderWithIPObjects(BaseTestReader, unittest.TestCase): readerClass = maxminddb.extension.Reader -class TestAutoReader(BaseTestReader, unittest.TestCase): +class TestAutoReader(BaseTestReader): mode = MODE_AUTO readerClass: Union[ @@ -673,31 +682,31 @@ class TestAutoReader(BaseTestReader, unittest.TestCase): readerClass = maxminddb.reader.Reader -class TestMMAPReader(BaseTestReader, unittest.TestCase): +class TestMMAPReader(BaseTestReader): mode = MODE_MMAP readerClass = maxminddb.reader.Reader # We want one pure Python test to use IP objects, it doesn't # really matter which one. -class TestMMAPReaderWithIPObjects(BaseTestReader, unittest.TestCase): +class TestMMAPReaderWithIPObjects(BaseTestReader): mode = MODE_MMAP use_ip_objects = True readerClass = maxminddb.reader.Reader -class TestFileReader(BaseTestReader, unittest.TestCase): +class TestFileReader(BaseTestReader): mode = MODE_FILE readerClass = maxminddb.reader.Reader -class TestMemoryReader(BaseTestReader, unittest.TestCase): +class TestMemoryReader(BaseTestReader): mode = MODE_MEMORY readerClass = maxminddb.reader.Reader -class TestFDReader(BaseTestReader, unittest.TestCase): - def setUp(self): +class TestFDReader(BaseTestReader): + def setUp(self) -> None: self.open_database_patcher = mock.patch(__name__ + ".open_database") self.addCleanup(self.open_database_patcher.stop) self.open_database = self.open_database_patcher.start() @@ -708,9 +717,12 @@ def setUp(self): class TestOldReader(unittest.TestCase): - def test_old_reader(self): + def test_old_reader(self) -> None: reader = maxminddb.Reader("tests/data/test-data/MaxMind-DB-test-decoder.mmdb") - record = reader.get("::1.1.1.0") + record = cast(dict, reader.get("::1.1.1.0")) self.assertEqual(record["array"], [1, 2, 3]) reader.close() + + +del BaseTestReader From f119692b0f67757835d09e3439d56a988bc08c17 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 29 Jan 2025 12:15:27 -0800 Subject: [PATCH 6/8] Do not use camelcase --- tests/reader_test.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/reader_test.py b/tests/reader_test.py index 6bd1be9..9d7de60 100644 --- a/tests/reader_test.py +++ b/tests/reader_test.py @@ -42,7 +42,7 @@ def get_reader_from_file_descriptor(filepath, mode) -> Reader: class BaseTestReader(unittest.TestCase): mode: int - readerClass: Union[ + reader_class: Union[ Type["maxminddb.extension.Reader"], Type["maxminddb.reader.Reader"], ] @@ -375,11 +375,11 @@ def test_database_with_invalid_utf8_key(self) -> None: def test_too_many_constructor_args(self) -> None: with self.assertRaises(TypeError): - self.readerClass("README.md", self.mode, 1) # type: ignore + self.reader_class("README.md", self.mode, 1) # type: ignore def test_bad_constructor_mode(self) -> None: with self.assertRaisesRegex(ValueError, r"Unsupported open mode \(100\)"): - self.readerClass("README.md", mode=100) # type: ignore + self.reader_class("README.md", mode=100) # type: ignore def test_no_constructor_args(self) -> None: with self.assertRaisesRegex( @@ -389,7 +389,7 @@ def test_no_constructor_args(self) -> None: r"takes at least 2 arguments|" r"function missing required argument \'database\' \(pos 1\)", ): - self.readerClass() # type: ignore + self.reader_class() # type: ignore def test_too_many_get_args(self) -> None: reader = open_database( @@ -654,7 +654,7 @@ class TestExtensionReader(BaseTestReader): mode = MODE_MMAP_EXT if has_maxminddb_extension(): - readerClass = maxminddb.extension.Reader + reader_class = maxminddb.extension.Reader @unittest.skipIf( @@ -666,25 +666,25 @@ class TestExtensionReaderWithIPObjects(BaseTestReader): use_ip_objects = True if has_maxminddb_extension(): - readerClass = maxminddb.extension.Reader + reader_class = maxminddb.extension.Reader class TestAutoReader(BaseTestReader): mode = MODE_AUTO - readerClass: Union[ + reader_class: Union[ Type["maxminddb.extension.Reader"], Type["maxminddb.reader.Reader"], ] if has_maxminddb_extension(): - readerClass = maxminddb.extension.Reader + reader_class = maxminddb.extension.Reader else: - readerClass = maxminddb.reader.Reader + reader_class = maxminddb.reader.Reader class TestMMAPReader(BaseTestReader): mode = MODE_MMAP - readerClass = maxminddb.reader.Reader + reader_class = maxminddb.reader.Reader # We want one pure Python test to use IP objects, it doesn't @@ -692,17 +692,17 @@ class TestMMAPReader(BaseTestReader): class TestMMAPReaderWithIPObjects(BaseTestReader): mode = MODE_MMAP use_ip_objects = True - readerClass = maxminddb.reader.Reader + reader_class = maxminddb.reader.Reader class TestFileReader(BaseTestReader): mode = MODE_FILE - readerClass = maxminddb.reader.Reader + reader_class = maxminddb.reader.Reader class TestMemoryReader(BaseTestReader): mode = MODE_MEMORY - readerClass = maxminddb.reader.Reader + reader_class = maxminddb.reader.Reader class TestFDReader(BaseTestReader): @@ -713,7 +713,7 @@ def setUp(self) -> None: self.open_database.side_effect = get_reader_from_file_descriptor mode = MODE_FD - readerClass = maxminddb.reader.Reader + reader_class = maxminddb.reader.Reader class TestOldReader(unittest.TestCase): From dc8378e19def1055f799c2ea2f6f8e4d1dd4e280 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 29 Jan 2025 13:31:36 -0800 Subject: [PATCH 7/8] Revert some unsafe fixes from ruff --- maxminddb/extension.pyi | 3 ++- setup.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/maxminddb/extension.pyi b/maxminddb/extension.pyi index 8507311..c4dee56 100644 --- a/maxminddb/extension.pyi +++ b/maxminddb/extension.pyi @@ -11,6 +11,7 @@ from ipaddress import IPv4Address, IPv6Address from os import PathLike from typing import IO, Any, AnyStr, Optional, Tuple, Union +from maxminddb import MODE_AUTO from maxminddb.types import Record class Reader: @@ -23,7 +24,7 @@ class Reader: def __init__( self, database: Union[AnyStr, int, PathLike, IO], - mode: int = ..., + mode: int = MODE_AUTO, ) -> None: """Reader for the MaxMind DB file format diff --git a/setup.py b/setup.py index 7ee1885..5b5a5d9 100644 --- a/setup.py +++ b/setup.py @@ -120,9 +120,11 @@ def build_extension(self, ext) -> None: ) -def status_msgs(*msgs) -> None: - for _msg in msgs: - pass +def status_msgs(*msgs): + print("*" * 75) + for msg in msgs: + print(msg) + print("*" * 75) def find_packages(location): From 4c9eb57d577b991920c9179973d729deea47a9a2 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 29 Jan 2025 13:40:31 -0800 Subject: [PATCH 8/8] Fix mypy heisenfailure on Windows --- tests/reader_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/reader_test.py b/tests/reader_test.py index 9d7de60..569baf6 100644 --- a/tests/reader_test.py +++ b/tests/reader_test.py @@ -543,7 +543,7 @@ def lookup(pipe) -> None: except: pipe.send(0) finally: - if worker_class is self.mp.Process: + if worker_class is self.mp.Process: # type: ignore reader.close() pipe.close()