From f8215d7acfca943daa4396c42f58b79cc51640b6 Mon Sep 17 00:00:00 2001 From: Maxwell G Date: Sat, 2 Dec 2023 00:55:11 +0000 Subject: [PATCH] add compatibility for pydantic v2 This uses the `pydantic.v1` compat module (well really, it's just an in-tree copy of the latest pydantic v1 release) so we can also support pydantic v2. Major changes to root models make it infeasible to support pydantic v2 natively without dropping support for v1. The `_pydantic_compat` module simplifies imports of the `pydantic.v1` compat module with pydantic v2. The pydantic.v1 compat module is always used when type checking. We have to force one of them when type checking, as type checkers cannot understand `try-except` imports. --- .pylintrc.automated | 2 +- pyproject.toml | 8 ++- src/antsibull_docs/__init__.py | 23 ++++++++ src/antsibull_docs/_pydantic_compat.py | 57 +++++++++++++++++++ .../cli/doc_commands/collection.py | 2 +- .../cli/doc_commands/collection_plugins.py | 2 +- src/antsibull_docs/cli/doc_commands/devel.py | 2 +- src/antsibull_docs/cli/doc_commands/stable.py | 2 +- src/antsibull_docs/collection_config.py | 17 ++++-- src/antsibull_docs/collection_links.py | 17 ++++-- src/antsibull_docs/process_docs.py | 7 ++- src/antsibull_docs/schemas/app_context.py | 7 ++- .../schemas/collection_config.py | 2 +- .../schemas/collection_links.py | 2 +- src/antsibull_docs/schemas/docs/base.py | 2 +- src/antsibull_docs/schemas/docs/callback.py | 2 +- src/antsibull_docs/schemas/docs/module.py | 2 +- src/antsibull_docs/schemas/docs/plugin.py | 2 +- src/antsibull_docs/schemas/docs/positional.py | 2 +- src/antsibull_docs/schemas/docs/role.py | 2 +- src/sphinx_antsibull_ext/directive_helper.py | 8 +-- .../schemas/ansible_links.py | 2 +- 22 files changed, 133 insertions(+), 39 deletions(-) create mode 100644 src/antsibull_docs/_pydantic_compat.py diff --git a/.pylintrc.automated b/.pylintrc.automated index d878f0c4..1b1403c8 100644 --- a/.pylintrc.automated +++ b/.pylintrc.automated @@ -449,7 +449,7 @@ max-branches=12 max-locals=15 # Maximum number of parents for a class (see R0901). -max-parents=7 +max-parents=8 # Maximum number of public methods for a class (see R0904). max-public-methods=20 diff --git a/pyproject.toml b/pyproject.toml index 4708d255..9447fc3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,11 +30,13 @@ dependencies = [ "asyncio-pool", "docutils", "jinja2 >= 3.0", - "packaging", + # Support for Version.major + "packaging >= 20.0", "rstcheck >= 3.0.0, < 7.0.0", "sphinx", - # pydantic v2 is a major rewrite - "pydantic >= 1.0.0, < 2.0.0", + # pydantic v2 contains deprecated features that we need for pydantic v1 + # compat + "pydantic >= 1.0.0, < 3.0.0", "semantic_version", "aiohttp >= 3.0.0", "twiggy", diff --git a/src/antsibull_docs/__init__.py b/src/antsibull_docs/__init__.py index 3329024c..fd17010f 100644 --- a/src/antsibull_docs/__init__.py +++ b/src/antsibull_docs/__init__.py @@ -4,4 +4,27 @@ # SPDX-FileCopyrightText: 2020, Ansible Project """The main antsibull-docs module. Contains versioning information.""" +import os +import warnings + +import pydantic + __version__ = "2.6.1.post0" + + +def _filter_pydantic_v2_warnings() -> None: + """ + Filter DeprecationWarnings from Pydantic v2. We cannot fix these without + dropping support for v1 entirely, and we don't want to break setups with + PYTHONWARNINGS=error. + """ + + typ: type[DeprecationWarning] | None + if typ := getattr(pydantic, "PydanticDeprecatedSince20", None): + warnings.simplefilter(action="ignore", category=typ) + + +if "_ANTSIBULL_SHOW_PYDANTIC_WARNINGS" not in os.environ: + _filter_pydantic_v2_warnings() + +__all__ = ("__version__",) diff --git a/src/antsibull_docs/_pydantic_compat.py b/src/antsibull_docs/_pydantic_compat.py new file mode 100644 index 00000000..5d179049 --- /dev/null +++ b/src/antsibull_docs/_pydantic_compat.py @@ -0,0 +1,57 @@ +# Copyright (C) 2023 Maxwell G +# SPDX-License-Identifier: GPL-3.0-or-later +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) + +""" +Utilities to help maintain compatibility between Pydantic v1 and Pydantic v2 +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING, Any + +import pydantic as p +import pydantic.version +from packaging.version import Version + +pydantic_version = Version(p.version.VERSION) +HAS_PYDANTIC_V2 = pydantic_version.major == 2 + +if TYPE_CHECKING or HAS_PYDANTIC_V2: + # These pragmas are only applicable when running linters with pydantic v1 + # installed. pylint and mypy will work correctly when run against pydantic + # v2. + + # pylint: disable-next=no-name-in-module,useless-suppression + from pydantic import v1 # type: ignore + +else: + v1 = p + +_PYDANTIC_FIELD_RENAMES: dict[str, tuple[str, Callable | None]] = { + "min_items": ("min_length", None), + "max_items": ("max_length", None), + "regex": ("pattern", None), + "allow_mutation": ("frozen", lambda v: not v), +} + + +def Field(*args: Any, **kwargs: Any) -> Any: + """ + Compatibility shim between pydantic v1 and pydantic v2's `Field`. + """ + + if HAS_PYDANTIC_V2: + for key, value in tuple(kwargs.items()): + if key in _PYDANTIC_FIELD_RENAMES: + new_key, transform = _PYDANTIC_FIELD_RENAMES[key] + if transform: + value = transform(value) + kwargs[new_key] = value + del kwargs[key] + return pydantic.Field(*args, **kwargs) + + +__all__ = ("pydantic_version", "HAS_PYDANTIC_V2", "Field", "v1") diff --git a/src/antsibull_docs/cli/doc_commands/collection.py b/src/antsibull_docs/cli/doc_commands/collection.py index 2d4bf486..96a31c31 100644 --- a/src/antsibull_docs/cli/doc_commands/collection.py +++ b/src/antsibull_docs/cli/doc_commands/collection.py @@ -136,7 +136,7 @@ def generate_docs() -> int: app_ctx.extra["collections"], collection_version, tmp_dir, - galaxy_server=app_ctx.galaxy_url, + galaxy_server=str(app_ctx.galaxy_url), collection_cache=app_ctx.collection_cache, ) ) diff --git a/src/antsibull_docs/cli/doc_commands/collection_plugins.py b/src/antsibull_docs/cli/doc_commands/collection_plugins.py index d29ec3ce..52cadf79 100644 --- a/src/antsibull_docs/cli/doc_commands/collection_plugins.py +++ b/src/antsibull_docs/cli/doc_commands/collection_plugins.py @@ -89,7 +89,7 @@ def generate_docs() -> int: list(app_ctx.extra["collection"]), collection_version, tmp_dir, - galaxy_server=app_ctx.galaxy_url, + galaxy_server=str(app_ctx.galaxy_url), collection_cache=app_ctx.collection_cache, ) ) diff --git a/src/antsibull_docs/cli/doc_commands/devel.py b/src/antsibull_docs/cli/doc_commands/devel.py index acfb0118..268b124b 100644 --- a/src/antsibull_docs/cli/doc_commands/devel.py +++ b/src/antsibull_docs/cli/doc_commands/devel.py @@ -121,7 +121,7 @@ def generate_docs() -> int: retrieve( collections, tmp_dir, - galaxy_server=app_ctx.galaxy_url, + galaxy_server=str(app_ctx.galaxy_url), ansible_core_source=app_ctx.extra["ansible_core_source"], collection_cache=app_ctx.collection_cache, use_installed_ansible_core=use_installed_ansible_core, diff --git a/src/antsibull_docs/cli/doc_commands/stable.py b/src/antsibull_docs/cli/doc_commands/stable.py index a5e55b7a..bde7df92 100644 --- a/src/antsibull_docs/cli/doc_commands/stable.py +++ b/src/antsibull_docs/cli/doc_commands/stable.py @@ -123,7 +123,7 @@ def generate_docs() -> int: ansible_core_version, collections, tmp_dir, - galaxy_server=app_ctx.galaxy_url, + galaxy_server=str(app_ctx.galaxy_url), ansible_core_source=app_ctx.extra["ansible_core_source"], collection_cache=app_ctx.collection_cache, use_installed_ansible_core=use_installed_ansible_core, diff --git a/src/antsibull_docs/collection_config.py b/src/antsibull_docs/collection_config.py index 118d90f2..0e99afc4 100644 --- a/src/antsibull_docs/collection_config.py +++ b/src/antsibull_docs/collection_config.py @@ -15,8 +15,8 @@ from antsibull_core import app_context from antsibull_core.logging import log from antsibull_core.yaml import load_yaml_file -from pydantic import Extra -from pydantic.error_wrappers import ValidationError, display_errors + +from antsibull_docs._pydantic_compat import v1 from .schemas.collection_config import CollectionConfig @@ -51,7 +51,7 @@ async def load_collection_config( if os.path.isfile(config_path): try: return CollectionConfig.parse_obj(load_yaml_file(config_path)) - except ValidationError: + except v1.ValidationError: pass return CollectionConfig.parse_obj({}) finally: @@ -100,7 +100,7 @@ def lint_collection_config(collection_path: str) -> list[tuple[str, int, int, st result: list[tuple[str, int, int, str]] = [] for cls in (CollectionConfig,): - cls.__config__.extra = Extra.forbid # type: ignore[attr-defined] + cls.__config__.extra = v1.Extra.forbid # type: ignore[attr-defined] try: config_path = os.path.join(collection_path, "docs", "docsite", "config.yml") @@ -110,10 +110,15 @@ def lint_collection_config(collection_path: str) -> list[tuple[str, int, int, st config_data = load_yaml_file(config_path) try: CollectionConfig.parse_obj(config_data) - except ValidationError as exc: + except v1.ValidationError as exc: for error in exc.errors(): result.append( - (config_path, 0, 0, display_errors([error]).replace("\n ", ":")) + ( + config_path, + 0, + 0, + v1.error_wrappers.display_errors([error]).replace("\n ", ":"), + ) ) return result diff --git a/src/antsibull_docs/collection_links.py b/src/antsibull_docs/collection_links.py index 272cfcc0..30fd8f4e 100644 --- a/src/antsibull_docs/collection_links.py +++ b/src/antsibull_docs/collection_links.py @@ -16,8 +16,8 @@ from antsibull_core import app_context from antsibull_core.logging import log from antsibull_core.yaml import load_yaml_file -from pydantic import Extra -from pydantic.error_wrappers import ValidationError, display_errors + +from antsibull_docs._pydantic_compat import v1 from .schemas.collection_links import ( CollectionEditOnGitHub, @@ -131,7 +131,7 @@ def load( ld = {} try: result = CollectionLinks.parse_obj(ld) - except ValidationError: + except v1.ValidationError: result = CollectionLinks.parse_obj({}) # Parse MANIFEST or galaxy data @@ -249,7 +249,7 @@ def lint_collection_links(collection_path: str) -> list[tuple[str, int, int, str Communication, CollectionLinks, ): - cls.__config__.extra = Extra.forbid # type: ignore[attr-defined] + cls.__config__.extra = v1.Extra.forbid # type: ignore[attr-defined] try: index_path = os.path.join(collection_path, "docs", "docsite", "links.yml") @@ -264,10 +264,15 @@ def lint_collection_links(collection_path: str) -> list[tuple[str, int, int, str ) try: CollectionLinks.parse_obj(links_data) - except ValidationError as exc: + except v1.ValidationError as exc: for error in exc.errors(): result.append( - (index_path, 0, 0, display_errors([error]).replace("\n ", ":")) + ( + index_path, + 0, + 0, + v1.error_wrappers.display_errors([error]).replace("\n ", ":"), + ) ) return result diff --git a/src/antsibull_docs/process_docs.py b/src/antsibull_docs/process_docs.py index 6fbfc412..15a0b0e3 100644 --- a/src/antsibull_docs/process_docs.py +++ b/src/antsibull_docs/process_docs.py @@ -12,7 +12,8 @@ from concurrent.futures import ProcessPoolExecutor from antsibull_core.logging import log -from pydantic import ValidationError + +from antsibull_docs._pydantic_compat import v1 from . import app_context from .docs_parsing.fqcn import get_fqcn_parts @@ -64,7 +65,7 @@ def normalize_plugin_info( try: parsed = DOCS_SCHEMAS[plugin_type].parse_obj(plugin_info) # type: ignore[attr-defined] return parsed.dict(by_alias=True), errors - except ValidationError as e: + except v1.ValidationError as e: raise ValueError(str(e)) # pylint:disable=raise-missing-from new_info: dict[str, t.Any] = {} @@ -73,7 +74,7 @@ def normalize_plugin_info( try: schema = DOCS_SCHEMAS[plugin_type][field] # type: ignore[index] field_model = schema.parse_obj({field: plugin_info.get(field)}) - except ValidationError as e: + except v1.ValidationError as e: if field == "doc": # We can't recover if there's not a doc field # pydantic exceptions are not picklable (probably due to bugs in the pickle module) diff --git a/src/antsibull_docs/schemas/app_context.py b/src/antsibull_docs/schemas/app_context.py index 54d4a487..10d631ba 100644 --- a/src/antsibull_docs/schemas/app_context.py +++ b/src/antsibull_docs/schemas/app_context.py @@ -9,13 +9,14 @@ # to initialize the attributes when data is loaded into them. # pyre-ignore-all-errors[13] - import pydantic as p from antsibull_core.schemas.context import AppContext as CoreAppContext from antsibull_core.schemas.validators import convert_bool +from antsibull_docs._pydantic_compat import Field + #: Valid choices for the docs parsing backend -DOC_PARSING_BACKEND_CHOICES_F = p.Field("auto", regex="^(auto|ansible-core-2\\.13)$") +DOC_PARSING_BACKEND_CHOICES_F = Field("auto", regex="^(auto|ansible-core-2\\.13)$") DEFAULT_COLLECTION_URL_TRANSFORM = ( @@ -42,6 +43,6 @@ class DocsAppContext(CoreAppContext): } # pylint: disable-next=unused-private-member - __convert_docs_bools = p.validator( + __convert_docs_bools = p.validator( # type: ignore "breadcrumbs", "indexes", "use_html_blobs", pre=True, allow_reuse=True )(convert_bool) diff --git a/src/antsibull_docs/schemas/collection_config.py b/src/antsibull_docs/schemas/collection_config.py index 4f368e46..5f27a03e 100644 --- a/src/antsibull_docs/schemas/collection_config.py +++ b/src/antsibull_docs/schemas/collection_config.py @@ -9,7 +9,7 @@ # to initialize the attributes when data is loaded into them. # pyre-ignore-all-errors[13] -import pydantic as p +from antsibull_docs._pydantic_compat import v1 as p class CollectionConfig(p.BaseModel): diff --git a/src/antsibull_docs/schemas/collection_links.py b/src/antsibull_docs/schemas/collection_links.py index 5439b938..b8d73eee 100644 --- a/src/antsibull_docs/schemas/collection_links.py +++ b/src/antsibull_docs/schemas/collection_links.py @@ -11,7 +11,7 @@ import typing as t -import pydantic as p +from antsibull_docs._pydantic_compat import v1 as p _SENTINEL = object() diff --git a/src/antsibull_docs/schemas/docs/base.py b/src/antsibull_docs/schemas/docs/base.py index 5012c247..c8c35f59 100644 --- a/src/antsibull_docs/schemas/docs/base.py +++ b/src/antsibull_docs/schemas/docs/base.py @@ -125,9 +125,9 @@ def handle_renamed_attribute(cls, values): import typing as t from collections.abc import Mapping -import pydantic as p from antsibull_core.yaml import load_yaml_bytes +from antsibull_docs._pydantic_compat import v1 as p from antsibull_docs.vendored.ansible import ( # type: ignore[import] check_type_bits, check_type_bool, diff --git a/src/antsibull_docs/schemas/docs/callback.py b/src/antsibull_docs/schemas/docs/callback.py index 6584ac64..afc5d733 100644 --- a/src/antsibull_docs/schemas/docs/callback.py +++ b/src/antsibull_docs/schemas/docs/callback.py @@ -5,7 +5,7 @@ # SPDX-FileCopyrightText: 2020, Ansible Project """Schemas for the plugin DOCUMENTATION data.""" -import pydantic as p +from antsibull_docs._pydantic_compat import v1 as p from .base import BaseModel from .plugin import ( diff --git a/src/antsibull_docs/schemas/docs/module.py b/src/antsibull_docs/schemas/docs/module.py index b8dcf535..f193db47 100644 --- a/src/antsibull_docs/schemas/docs/module.py +++ b/src/antsibull_docs/schemas/docs/module.py @@ -6,7 +6,7 @@ """Schemas for the plugin DOCUMENTATION data.""" -import pydantic as p +from antsibull_docs._pydantic_compat import v1 as p from .base import BaseModel, DocSchema, OptionsSchema from .plugin import PluginExamplesSchema, PluginMetadataSchema, PluginReturnSchema diff --git a/src/antsibull_docs/schemas/docs/plugin.py b/src/antsibull_docs/schemas/docs/plugin.py index 729a489b..5107949e 100644 --- a/src/antsibull_docs/schemas/docs/plugin.py +++ b/src/antsibull_docs/schemas/docs/plugin.py @@ -12,7 +12,7 @@ import re import typing as t -import pydantic as p +from antsibull_docs._pydantic_compat import v1 as p from .base import ( COLLECTION_NAME_F, diff --git a/src/antsibull_docs/schemas/docs/positional.py b/src/antsibull_docs/schemas/docs/positional.py index 59c97de4..6a20635a 100644 --- a/src/antsibull_docs/schemas/docs/positional.py +++ b/src/antsibull_docs/schemas/docs/positional.py @@ -7,7 +7,7 @@ """Schemas for the plugin DOCUMENTATION data.""" -import pydantic as p +from antsibull_docs._pydantic_compat import v1 as p from .base import BaseModel from .plugin import ( diff --git a/src/antsibull_docs/schemas/docs/role.py b/src/antsibull_docs/schemas/docs/role.py index 8b800063..9d021241 100644 --- a/src/antsibull_docs/schemas/docs/role.py +++ b/src/antsibull_docs/schemas/docs/role.py @@ -13,7 +13,7 @@ import typing as t from collections.abc import Mapping -import pydantic as p +from antsibull_docs._pydantic_compat import v1 as p from .base import ( COLLECTION_NAME_F, diff --git a/src/sphinx_antsibull_ext/directive_helper.py b/src/sphinx_antsibull_ext/directive_helper.py index d4e542ad..26523752 100644 --- a/src/sphinx_antsibull_ext/directive_helper.py +++ b/src/sphinx_antsibull_ext/directive_helper.py @@ -15,10 +15,10 @@ from antsibull_core.yaml import load_yaml_bytes from docutils import nodes from docutils.parsers.rst import Directive -from pydantic import BaseModel -from pydantic.error_wrappers import ValidationError -SchemaT = t.TypeVar("SchemaT", bound=BaseModel) +from antsibull_docs._pydantic_compat import v1 + +SchemaT = t.TypeVar("SchemaT", bound=v1.BaseModel) class YAMLDirective(Directive, t.Generic[SchemaT], metaclass=abc.ABCMeta): @@ -46,7 +46,7 @@ def run(self) -> list[nodes.Node]: } try: content_obj = self.schema.parse_obj(content) - except ValidationError as exc: + except v1.ValidationError as exc: raise self.error( f"Error while parsing content of {self.name}: {exc}" ) from exc diff --git a/src/sphinx_antsibull_ext/schemas/ansible_links.py b/src/sphinx_antsibull_ext/schemas/ansible_links.py index 5f09010a..e53c9819 100644 --- a/src/sphinx_antsibull_ext/schemas/ansible_links.py +++ b/src/sphinx_antsibull_ext/schemas/ansible_links.py @@ -9,7 +9,7 @@ import typing as t -import pydantic as p +from antsibull_docs._pydantic_compat import v1 as p # Ignore Unitialized attribute errors because BaseModel works some magic # to initialize the attributes when data is loaded into them.