Add tombstones for removed collections

changelogs/fragments/341-removed-collections.yml

@@ -0,0 +1,3 @@
+minor_changes:
+  - "When rendering the Ansible docsite with the ``stable`` and ``devel`` subcommands, stub pages for removed collections are added
+     (https://github.com/ansible-community/ansible-build-data/pull/459, https://github.com/ansible-community/antsibull-docs/pull/341)."

src/antsibull_docs/cli/doc_commands/_build.py

@@ -55,7 +55,11 @@
 from ...utils.collection_name_transformer import CollectionNameTransformer
 from ...write_docs import CollectionInfoT, _get_collection_dir
 from ...write_docs.changelog import output_changelogs
-from ...write_docs.collections import output_extra_docs, output_indexes
+from ...write_docs.collections import (
+    output_collection_indexes,
+    output_collection_tombstones,
+    output_extra_docs,
+)
 from ...write_docs.hierarchy import (
     output_collection_index,
     output_collection_namespace_indexes,
@@ -464,7 +468,9 @@ def generate_docs_for_all_collections(  # noqa: C901
         referenced_env_vars, core_env_vars, collection_metadata
     )
 
-    collection_namespaces = get_collection_namespaces(collection_to_plugin_info.keys())
+    collection_namespaces = get_collection_namespaces(
+        collection_to_plugin_info.keys(), collection_meta=collection_meta
+    )
 
     collection_url = CollectionNameTransformer(
         app_ctx.collection_url, DEFAULT_COLLECTION_URL_TRANSFORM
@@ -550,7 +556,7 @@ def generate_docs_for_all_collections(  # noqa: C901
 
     if create_collection_indexes:
         asyncio.run(
-            output_indexes(
+            output_collection_indexes(
                 collection_to_plugin_info,
                 output,
                 collection_url=collection_url,
@@ -567,7 +573,23 @@ def generate_docs_for_all_collections(  # noqa: C901
                 add_version=add_antsibull_docs_version,
             )
         )
-        flog.notice("Finished writing indexes")
+        flog.notice("Finished writing collection indexes")
+
+        asyncio.run(
+            output_collection_tombstones(
+                collection_meta,
+                output,
+                collection_url=collection_url,
+                collection_install=collection_install,
+                squash_hierarchy=squash_hierarchy,
+                output_format=output_format,
+                filename_generator=filename_generator,
+                breadcrumbs=breadcrumbs,
+                for_official_docsite=for_official_docsite,
+                add_version=add_antsibull_docs_version,
+            )
+        )
+        flog.notice("Finished writing collection tombstones")
 
         asyncio.run(
             output_changelogs(

src/antsibull_docs/data/docsite/ansible-docsite/collection-tombstone.rst.j2

@@ -0,0 +1,23 @@
+{#
+  Copyright (c) Ansible Project
+  GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+  SPDX-License-Identifier: GPL-3.0-or-later
+#}
+
+:orphan:
+
+{% if antsibull_docs_version %}
+.. meta::
+  :antsibull-docs: @{ antsibull_docs_version }@
+
+{% endif %}
+
+.. _plugins_in_@{collection_name}@:
+
+@{collection_name.title()}@
+@{ '=' * (collection_name | column_width) }@
+
+This collection has been removed from Ansible @{ collection_removal_version.major }@.
+
+If you want to continue using this collection, you can install it manually using
+@{ collection_name | collection_install | rst_code }@.

src/antsibull_docs/data/docsite/ansible-docsite/list_of_collections_by_namespace.rst.j2

@@ -30,6 +30,8 @@ These are the collections documented here in the **@{ namespace }@** namespace.
 
 {% for name in collections | sort %}
 * :ref:`@{ namespace }@.@{ name }@ <plugins_in_@{ namespace }@.@{ name }@>` @{ collection_deprecation_marker(collection_metadata[namespace ~ '.' ~ name]) }@
+{% else %}
+There is no collection in this namespace.
 {% endfor %}
 
 {% if breadcrumbs %}

src/antsibull_docs/data/docsite/simplified-rst/collection-tombstone.rst.j2

@@ -0,0 +1,19 @@
+{#
+  Copyright (c) Ansible Project
+  GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+  SPDX-License-Identifier: GPL-3.0-or-later
+#}
+
+{% if antsibull_docs_version %}
+.. Created with antsibull-docs @{ antsibull_docs_version }@
+{% else %}
+.. Created with antsibull-docs
+{% endif %}
+
+@{collection_name.title()}@
+@{ '=' * (collection_name | column_width) }@
+
+This collection has been removed from Ansible @{ collection_removal_version.major }@.
+
+If you want to continue using this collection, you can install it manually using
+@{ collection_name | collection_install | rst_code }@.

src/antsibull_docs/data/docsite/simplified-rst/list_of_collections_by_namespace.rst.j2

@@ -29,4 +29,6 @@ These are the collections documented here in the **@{ namespace }@** namespace.
 
 {% for name in collections | sort %}
 * `@{ namespace }@.@{ name }@ <namespace/index.rst>`_
+{% else %}
+There is no collection in this namespace.
 {% endfor %}

src/antsibull_docs/process_docs.py

@@ -17,6 +17,7 @@
 import pydantic as p
 import pydantic_core
 from antsibull_core.logging import log
+from antsibull_core.schemas.collection_meta import CollectionsMetadata
 
 from . import app_context
 from .docs_parsing.fqcn import get_fqcn_parts
@@ -30,17 +31,27 @@
 PluginErrorsRT = defaultdict[str, defaultdict[str, list[str]]]
 
 
-def get_collection_namespaces(collection_names: Iterable[str]) -> dict[str, list[str]]:
+def get_collection_namespaces(
+    collection_names: Iterable[str], *, collection_meta: CollectionsMetadata | None
+) -> dict[str, list[str]]:
     """
     Return the plugins which are in each collection.
 
     :arg collection_names: An iterable of collection names.
+    :kwarg collection_meta: Optional collection metadata. If provided, will
+        ensure that namespaces that only contain removed collections are also
+        present (with an empty collection list).
     :returns: Mapping from collection namespaces to list of collection names.
     """
     namespaces = defaultdict(list)
     for collection_name in collection_names:
         namespace, name = collection_name.split(".", 1)
         namespaces[namespace].append(name)
+    if collection_meta:
+        for collection_name in collection_meta.removed_collections:
+            namespace, name = collection_name.split(".", 1)
+            # Simply make sure that there's an entry for the namespace:
+            namespaces[namespace]  # pylint:disable=pointless-statement
     return namespaces
 
 

src/antsibull_docs/write_docs/collections.py

@@ -14,6 +14,10 @@
 import asyncio_pool  # type: ignore[import]
 from antsibull_core import app_context
 from antsibull_core.logging import log
+from antsibull_core.schemas.collection_meta import (
+    CollectionsMetadata,
+    RemovedCollectionMetadata,
+)
 from jinja2 import Template
 from packaging.specifiers import SpecifierSet
 
@@ -54,7 +58,7 @@ def _parse_required_ansible(requires_ansible: str) -> list[str]:
     return result
 
 
-async def write_plugin_lists(
+async def write_collection_index(
     collection_name: str,
     plugin_maps: Mapping[str, Mapping[str, BasicPluginInfo]],
     template: Template,
@@ -90,7 +94,7 @@ async def write_plugin_lists(
     :kwarg add_version: If set to ``False``, will not insert antsibull-docs' version into
         the generated files.
     """
-    flog = mlog.fields(func="write_plugin_lists")
+    flog = mlog.fields(func="write_collection_index")
     flog.debug("Enter")
 
     requires_ansible = []
@@ -132,7 +136,55 @@ async def write_plugin_lists(
     flog.debug("Leave")
 
 
-async def output_indexes(
+async def write_collection_tombstone(
+    collection_name: str,
+    template: Template,
+    output: Output,
+    collection_dir: str,
+    collection_metadata: RemovedCollectionMetadata,
+    output_format: OutputFormat,
+    filename_generator: FilenameGenerator,  # pylint: disable=unused-argument
+    breadcrumbs: bool = True,
+    for_official_docsite: bool = False,
+    squash_hierarchy: bool = False,
+    add_version: bool = True,
+) -> None:
+    """
+    Write a tombstone page for a collection.
+
+    :arg template: A template to render the collection tombstone.
+    :arg output: Output helper for writing output.
+    :arg collection_metadata: Removal metadata for the collection.
+    :kwarg breadcrumbs: Default True.  Set to False if breadcrumbs for collections should be
+        disabled.  This will disable breadcrumbs but save on memory usage.
+    :kwarg for_official_docsite: Default False.  Set to True to use wording specific for the
+        official docsite on docs.ansible.com.
+    :kwarg squash_hierarchy: If set to ``True``, no directory hierarchy will be used.
+        Undefined behavior if documentation for multiple collections are created.
+    :kwarg add_version: If set to ``False``, will not insert antsibull-docs' version into
+        the generated files.
+    """
+    flog = mlog.fields(func="write_collection_tombstone")
+    flog.debug("Enter")
+
+    index_file = os.path.join(collection_dir, f"index{output_format.output_extension}")
+    index_contents = _render_template(
+        template,
+        index_file,
+        collection_name=collection_name,
+        collection_removal_version=collection_metadata.removal.version,
+        breadcrumbs=breadcrumbs,
+        for_official_docsite=for_official_docsite,
+        squash_hierarchy=squash_hierarchy,
+        add_version=add_version,
+    )
+
+    await output.write_file(index_file, index_contents)
+
+    flog.debug("Leave")
+
+
+async def output_collection_indexes(
     collection_to_plugin_info: CollectionInfoT,
     output: Output,
     collection_metadata: Mapping[str, AnsibleCollectionMetadata],
@@ -168,7 +220,7 @@ async def output_indexes(
     :kwarg add_version: If set to ``False``, will not insert antsibull-docs' version into
         the generated files.
     """
-    flog = mlog.fields(func="output_indexes")
+    flog = mlog.fields(func="output_collection_indexes")
     flog.debug("Enter")
 
     if collection_metadata is None:
@@ -201,7 +253,7 @@ async def output_indexes(
             )
             writers.append(
                 await pool.spawn(
-                    write_plugin_lists(
+                    write_collection_index(
                         collection_name,
                         plugin_maps,
                         collection_plugins_tmpl,
@@ -225,6 +277,90 @@ async def output_indexes(
     flog.debug("Leave")
 
 
+async def output_collection_tombstones(
+    collections_metadata: CollectionsMetadata | None,
+    output: Output,
+    collection_url: CollectionNameTransformer,
+    collection_install: CollectionNameTransformer,
+    output_format: OutputFormat,
+    filename_generator: FilenameGenerator,
+    squash_hierarchy: bool = False,
+    breadcrumbs: bool = True,
+    for_official_docsite: bool = False,
+    add_version: bool = True,
+) -> None:
+    """
+    Generate collection-level index pages for the collections.
+
+    :arg collections_metadata: Metadata on collections.
+    :arg output: Output helper for writing output.
+    :kwarg squash_hierarchy: If set to ``True``, no directory hierarchy will be used.
+        Undefined behavior if documentation for multiple collections are created.
+    :kwarg breadcrumbs: Default True.  Set to False if breadcrumbs for collections should be
+        disabled.  This will disable breadcrumbs but save on memory usage.
+    :kwarg for_official_docsite: Default False.  Set to True to use wording specific for the
+        official docsite on docs.ansible.com.
+    :kwarg output_format: The output format to use.
+    :kwarg add_version: If set to ``False``, will not insert antsibull-docs' version into
+        the generated files.
+    """
+    flog = mlog.fields(func="output_collection_tombstones")
+    flog.debug("Enter")
+
+    if collections_metadata is None:
+        return
+
+    env = doc_environment(
+        collection_url=collection_url,
+        collection_install=collection_install,
+        referable_envvars=None,
+        output_format=output_format,
+        filename_generator=filename_generator,
+    )
+    # Get the templates
+    collection_tombstone_tmpl = env.get_template(
+        get_template_filename("collection-tombstone", output_format)
+    )
+
+    writers = []
+    lib_ctx = app_context.lib_ctx.get()
+
+    async with asyncio_pool.AioPool(size=lib_ctx.thread_max) as pool:
+        for (
+            collection_name,
+            collection_meta,
+        ) in collections_metadata.removed_collections.items():
+            namespace, collection = collection_name.split(".", 1)
+            collection_dir = _get_collection_dir(
+                output,
+                namespace,
+                collection,
+                squash_hierarchy=squash_hierarchy,
+                create_if_not_exists=True,
+            )
+            writers.append(
+                await pool.spawn(
+                    write_collection_tombstone(
+                        collection_name,
+                        collection_tombstone_tmpl,
+                        output,
+                        collection_dir,
+                        collection_meta,
+                        output_format,
+                        filename_generator,
+                        breadcrumbs=breadcrumbs,
+                        for_official_docsite=for_official_docsite,
+                        squash_hierarchy=squash_hierarchy,
+                        add_version=add_version,
+                    )
+                )
+            )
+
+        await asyncio.gather(*writers)
+
+    flog.debug("Leave")
+
+
 async def output_extra_docs(
     output: Output,
     extra_docs_data: Mapping[str, CollectionExtraDocsInfoT],