add compatibility for pydantic v2

This uses the `pydantic.v1` compat module (well really, it's just an
in-tree copy of the latest pydantic v1 release) so we can also support
pydantic v2. Major changes to root models make it infeasible to support
pydantic v2 natively without dropping support for v1.

The `_pydantic_compat` module simplifies imports of the `pydantic.v1`
compat module with pydantic v2. The pydantic.v1 compat module is always
used when type checking. We have to force one of them when type
checking, as type checkers cannot understand `try-except` imports.

.pylintrc.automated

@@ -449,7 +449,7 @@ max-branches=12
 max-locals=15
 
 # Maximum number of parents for a class (see R0901).
-max-parents=7
+max-parents=8
 
 # Maximum number of public methods for a class (see R0904).
 max-public-methods=20

pyproject.toml

@@ -30,11 +30,13 @@ dependencies = [
     "asyncio-pool",
     "docutils",
     "jinja2 >= 3.0",
-    "packaging",
+    # Support for Version.major
+    "packaging >= 20.0",
     "rstcheck >= 3.0.0, < 7.0.0",
     "sphinx",
-    # pydantic v2 is a major rewrite
-    "pydantic >= 1.0.0, < 2.0.0",
+    # pydantic v2 contains deprecated features that we need for pydantic v1
+    # compat
+    "pydantic >= 1.0.0, < 3.0.0",
     "semantic_version",
     "aiohttp >= 3.0.0",
     "twiggy",

src/antsibull_docs/__init__.py

@@ -4,4 +4,27 @@
 # SPDX-FileCopyrightText: 2020, Ansible Project
 """The main antsibull-docs module. Contains versioning information."""
 
+import os
+import warnings
+
+import pydantic
+
 __version__ = "2.6.1.post0"
+
+
+def _filter_pydantic_v2_warnings() -> None:
+    """
+    Filter DeprecationWarnings from Pydantic v2. We cannot fix these without
+    dropping support for v1 entirely, and we don't want to break setups with
+    PYTHONWARNINGS=error.
+    """
+
+    typ: type[DeprecationWarning] | None
+    if typ := getattr(pydantic, "PydanticDeprecatedSince20", None):
+        warnings.simplefilter(action="ignore", category=typ)
+
+
+if "_ANTSIBULL_SHOW_PYDANTIC_WARNINGS" not in os.environ:
+    _filter_pydantic_v2_warnings()
+
+__all__ = ("__version__",)

src/antsibull_docs/_pydantic_compat.py

@@ -0,0 +1,57 @@
+# Copyright (C) 2023 Maxwell G <[email protected]>
+# SPDX-License-Identifier: GPL-3.0-or-later
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
+# https://www.gnu.org/licenses/gpl-3.0.txt)
+
+"""
+Utilities to help maintain compatibility between Pydantic v1 and Pydantic v2
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+import pydantic as p
+import pydantic.version
+from packaging.version import Version
+
+pydantic_version = Version(p.version.VERSION)
+HAS_PYDANTIC_V2 = pydantic_version.major == 2
+
+if TYPE_CHECKING or HAS_PYDANTIC_V2:
+    # These pragmas are only applicable when running linters with pydantic v1
+    # installed. pylint and mypy will work correctly when run against pydantic
+    # v2.
+
+    # pylint: disable-next=no-name-in-module,useless-suppression
+    from pydantic import v1  # type: ignore
+
+else:
+    v1 = p
+
+_PYDANTIC_FIELD_RENAMES: dict[str, tuple[str, Callable | None]] = {
+    "min_items": ("min_length", None),
+    "max_items": ("max_length", None),
+    "regex": ("pattern", None),
+    "allow_mutation": ("frozen", lambda v: not v),
+}
+
+
+def Field(*args: Any, **kwargs: Any) -> Any:
+    """
+    Compatibility shim between pydantic v1 and pydantic v2's `Field`.
+    """
+
+    if HAS_PYDANTIC_V2:
+        for key, value in tuple(kwargs.items()):
+            if key in _PYDANTIC_FIELD_RENAMES:
+                new_key, transform = _PYDANTIC_FIELD_RENAMES[key]
+                if transform:
+                    value = transform(value)
+                kwargs[new_key] = value
+                del kwargs[key]
+    return pydantic.Field(*args, **kwargs)
+
+
+__all__ = ("pydantic_version", "HAS_PYDANTIC_V2", "Field", "v1")

src/antsibull_docs/cli/doc_commands/collection.py

@@ -136,7 +136,7 @@ def generate_docs() -> int:
                 app_ctx.extra["collections"],
                 collection_version,
                 tmp_dir,
-                galaxy_server=app_ctx.galaxy_url,
+                galaxy_server=str(app_ctx.galaxy_url),
                 collection_cache=app_ctx.collection_cache,
             )
         )

src/antsibull_docs/cli/doc_commands/collection_plugins.py

@@ -89,7 +89,7 @@ def generate_docs() -> int:
                 list(app_ctx.extra["collection"]),
                 collection_version,
                 tmp_dir,
-                galaxy_server=app_ctx.galaxy_url,
+                galaxy_server=str(app_ctx.galaxy_url),
                 collection_cache=app_ctx.collection_cache,
             )
         )

src/antsibull_docs/cli/doc_commands/devel.py

@@ -121,7 +121,7 @@ def generate_docs() -> int:
             retrieve(
                 collections,
                 tmp_dir,
-                galaxy_server=app_ctx.galaxy_url,
+                galaxy_server=str(app_ctx.galaxy_url),
                 ansible_core_source=app_ctx.extra["ansible_core_source"],
                 collection_cache=app_ctx.collection_cache,
                 use_installed_ansible_core=use_installed_ansible_core,

src/antsibull_docs/cli/doc_commands/stable.py

@@ -123,7 +123,7 @@ def generate_docs() -> int:
                 ansible_core_version,
                 collections,
                 tmp_dir,
-                galaxy_server=app_ctx.galaxy_url,
+                galaxy_server=str(app_ctx.galaxy_url),
                 ansible_core_source=app_ctx.extra["ansible_core_source"],
                 collection_cache=app_ctx.collection_cache,
                 use_installed_ansible_core=use_installed_ansible_core,

src/antsibull_docs/collection_config.py

@@ -15,8 +15,8 @@
 from antsibull_core import app_context
 from antsibull_core.logging import log
 from antsibull_core.yaml import load_yaml_file
-from pydantic import Extra
-from pydantic.error_wrappers import ValidationError, display_errors
+
+from antsibull_docs._pydantic_compat import v1
 
 from .schemas.collection_config import CollectionConfig
 
@@ -51,7 +51,7 @@ async def load_collection_config(
         if os.path.isfile(config_path):
             try:
                 return CollectionConfig.parse_obj(load_yaml_file(config_path))
-            except ValidationError:
+            except v1.ValidationError:
                 pass
         return CollectionConfig.parse_obj({})
     finally:
@@ -100,7 +100,7 @@ def lint_collection_config(collection_path: str) -> list[tuple[str, int, int, st
     result: list[tuple[str, int, int, str]] = []
 
     for cls in (CollectionConfig,):
-        cls.__config__.extra = Extra.forbid  # type: ignore[attr-defined]
+        cls.__config__.extra = v1.Extra.forbid  # type: ignore[attr-defined]
 
     try:
         config_path = os.path.join(collection_path, "docs", "docsite", "config.yml")
@@ -110,10 +110,15 @@ def lint_collection_config(collection_path: str) -> list[tuple[str, int, int, st
         config_data = load_yaml_file(config_path)
         try:
             CollectionConfig.parse_obj(config_data)
-        except ValidationError as exc:
+        except v1.ValidationError as exc:
             for error in exc.errors():
                 result.append(
-                    (config_path, 0, 0, display_errors([error]).replace("\n ", ":"))
+                    (
+                        config_path,
+                        0,
+                        0,
+                        v1.error_wrappers.display_errors([error]).replace("\n ", ":"),
+                    )
                 )
 
         return result

src/antsibull_docs/collection_links.py

@@ -16,8 +16,8 @@
 from antsibull_core import app_context
 from antsibull_core.logging import log
 from antsibull_core.yaml import load_yaml_file
-from pydantic import Extra
-from pydantic.error_wrappers import ValidationError, display_errors
+
+from antsibull_docs._pydantic_compat import v1
 
 from .schemas.collection_links import (
     CollectionEditOnGitHub,
@@ -131,7 +131,7 @@ def load(
         ld = {}
     try:
         result = CollectionLinks.parse_obj(ld)
-    except ValidationError:
+    except v1.ValidationError:
         result = CollectionLinks.parse_obj({})
 
     # Parse MANIFEST or galaxy data
@@ -249,7 +249,7 @@ def lint_collection_links(collection_path: str) -> list[tuple[str, int, int, str
         Communication,
         CollectionLinks,
     ):
-        cls.__config__.extra = Extra.forbid  # type: ignore[attr-defined]
+        cls.__config__.extra = v1.Extra.forbid  # type: ignore[attr-defined]
 
     try:
         index_path = os.path.join(collection_path, "docs", "docsite", "links.yml")
@@ -264,10 +264,15 @@ def lint_collection_links(collection_path: str) -> list[tuple[str, int, int, str
                 )
         try:
             CollectionLinks.parse_obj(links_data)
-        except ValidationError as exc:
+        except v1.ValidationError as exc:
             for error in exc.errors():
                 result.append(
-                    (index_path, 0, 0, display_errors([error]).replace("\n ", ":"))
+                    (
+                        index_path,
+                        0,
+                        0,
+                        v1.error_wrappers.display_errors([error]).replace("\n ", ":"),
+                    )
                 )
 
         return result

src/antsibull_docs/process_docs.py

@@ -12,7 +12,8 @@
 from concurrent.futures import ProcessPoolExecutor
 
 from antsibull_core.logging import log
-from pydantic import ValidationError
+
+from antsibull_docs._pydantic_compat import v1
 
 from . import app_context
 from .docs_parsing.fqcn import get_fqcn_parts
@@ -64,7 +65,7 @@ def normalize_plugin_info(
         try:
             parsed = DOCS_SCHEMAS[plugin_type].parse_obj(plugin_info)  # type: ignore[attr-defined]
             return parsed.dict(by_alias=True), errors
-        except ValidationError as e:
+        except v1.ValidationError as e:
             raise ValueError(str(e))  # pylint:disable=raise-missing-from
 
     new_info: dict[str, t.Any] = {}
@@ -73,7 +74,7 @@ def normalize_plugin_info(
         try:
             schema = DOCS_SCHEMAS[plugin_type][field]  # type: ignore[index]
             field_model = schema.parse_obj({field: plugin_info.get(field)})
-        except ValidationError as e:
+        except v1.ValidationError as e:
             if field == "doc":
                 # We can't recover if there's not a doc field
                 # pydantic exceptions are not picklable (probably due to bugs in the pickle module)

src/antsibull_docs/schemas/app_context.py

@@ -9,13 +9,14 @@
 # to initialize the attributes when data is loaded into them.
 # pyre-ignore-all-errors[13]
 
-
 import pydantic as p
 from antsibull_core.schemas.context import AppContext as CoreAppContext
 from antsibull_core.schemas.validators import convert_bool
 
+from antsibull_docs._pydantic_compat import Field
+
 #: Valid choices for the docs parsing backend
-DOC_PARSING_BACKEND_CHOICES_F = p.Field("auto", regex="^(auto|ansible-core-2\\.13)$")
+DOC_PARSING_BACKEND_CHOICES_F = Field("auto", regex="^(auto|ansible-core-2\\.13)$")
 
 
 DEFAULT_COLLECTION_URL_TRANSFORM = (
@@ -42,6 +43,6 @@ class DocsAppContext(CoreAppContext):
     }
 
     # pylint: disable-next=unused-private-member
-    __convert_docs_bools = p.validator(
+    __convert_docs_bools = p.validator(  # type: ignore
         "breadcrumbs", "indexes", "use_html_blobs", pre=True, allow_reuse=True
     )(convert_bool)

src/antsibull_docs/schemas/collection_config.py

@@ -9,7 +9,7 @@
 # to initialize the attributes when data is loaded into them.
 # pyre-ignore-all-errors[13]
 
-import pydantic as p
+from antsibull_docs._pydantic_compat import v1 as p
 
 
 class CollectionConfig(p.BaseModel):

src/antsibull_docs/schemas/collection_links.py

@@ -11,7 +11,7 @@
 
 import typing as t
 
-import pydantic as p
+from antsibull_docs._pydantic_compat import v1 as p
 
 _SENTINEL = object()
 

src/antsibull_docs/schemas/docs/base.py

@@ -125,9 +125,9 @@ def handle_renamed_attribute(cls, values):
 import typing as t
 from collections.abc import Mapping
 
-import pydantic as p
 from antsibull_core.yaml import load_yaml_bytes
 
+from antsibull_docs._pydantic_compat import v1 as p
 from antsibull_docs.vendored.ansible import (  # type: ignore[import]
     check_type_bits,
     check_type_bool,

src/antsibull_docs/schemas/docs/callback.py

@@ -5,7 +5,7 @@
 # SPDX-FileCopyrightText: 2020, Ansible Project
 """Schemas for the plugin DOCUMENTATION data."""
 
-import pydantic as p
+from antsibull_docs._pydantic_compat import v1 as p
 
 from .base import BaseModel
 from .plugin import (

src/antsibull_docs/schemas/docs/module.py

@@ -6,7 +6,7 @@
 """Schemas for the plugin DOCUMENTATION data."""
 
 
-import pydantic as p
+from antsibull_docs._pydantic_compat import v1 as p
 
 from .base import BaseModel, DocSchema, OptionsSchema
 from .plugin import PluginExamplesSchema, PluginMetadataSchema, PluginReturnSchema

src/antsibull_docs/schemas/docs/plugin.py

@@ -12,7 +12,7 @@
 import re
 import typing as t
 
-import pydantic as p
+from antsibull_docs._pydantic_compat import v1 as p
 
 from .base import (
     COLLECTION_NAME_F,

src/antsibull_docs/schemas/docs/positional.py

@@ -7,7 +7,7 @@
 """Schemas for the plugin DOCUMENTATION data."""
 
 
-import pydantic as p
+from antsibull_docs._pydantic_compat import v1 as p
 
 from .base import BaseModel
 from .plugin import (

src/antsibull_docs/schemas/docs/role.py

@@ -13,7 +13,7 @@
 import typing as t
 from collections.abc import Mapping
 
-import pydantic as p
+from antsibull_docs._pydantic_compat import v1 as p
 
 from .base import (
     COLLECTION_NAME_F,

src/sphinx_antsibull_ext/directive_helper.py

@@ -15,10 +15,10 @@
 from antsibull_core.yaml import load_yaml_bytes
 from docutils import nodes
 from docutils.parsers.rst import Directive
-from pydantic import BaseModel
-from pydantic.error_wrappers import ValidationError
 
-SchemaT = t.TypeVar("SchemaT", bound=BaseModel)
+from antsibull_docs._pydantic_compat import v1
+
+SchemaT = t.TypeVar("SchemaT", bound=v1.BaseModel)
 
 
 class YAMLDirective(Directive, t.Generic[SchemaT], metaclass=abc.ABCMeta):
@@ -46,7 +46,7 @@ def run(self) -> list[nodes.Node]:
             }
         try:
             content_obj = self.schema.parse_obj(content)
-        except ValidationError as exc:
+        except v1.ValidationError as exc:
             raise self.error(
                 f"Error while parsing content of {self.name}: {exc}"
             ) from exc