Merge pull request #494 from DiamondLightSource/api-docs-via-autosummary

Fix most docs build warnings and generate API docs with `autosummary` extension

.github/workflows/httomo_docs.yml

@@ -31,10 +31,7 @@ jobs:
             activate-environment: httomo-docs
             environment-file: ./docs/source/doc-conda-requirements.yml
 
-        - name: Build api docs
-          run: sphinx-apidoc -feT -t=./docs/source/_templates -o ./docs/source/api ./httomo
-
-        - name: Build html
+        - name: Build docs
           run: sphinx-build -a -E -b html ./docs/source/ ./docs/build/
 
         - name: Publish docs

.gitignore

@@ -48,3 +48,5 @@ output_dir/
 temp.yaml
 cufile.log
 
+# Sphinx generated API docs
+docs/source/developers/generated/

docs/source/_templates/autosummary/class.rst

@@ -0,0 +1,32 @@
+{{ fullname | escape | underline}}
+
+.. currentmodule:: {{ module }}
+
+.. autoclass:: {{ objname }}
+   :members:
+
+   {% block methods %}
+   {% if methods %}
+   .. rubric:: {{ _('Methods') }}
+
+   .. autosummary::
+   {% for item in methods %}
+      {% if item not in inherited_members %}
+      ~{{ name }}.{{ item }}
+      {% endif %}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block attributes %}
+   {% if attributes %}
+   .. rubric:: {{ _('Attributes') }}
+
+   .. autosummary::
+   {% for item in attributes %}
+      {% if item not in inherited_members %}
+      ~{{ name }}.{{ item }}
+      {% endif %}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}

docs/source/_templates/autosummary/module.rst

@@ -0,0 +1,36 @@
+{{ fullname | escape | underline}}
+
+{% if modules %}
+.. rubric:: Modules
+
+.. autosummary::
+   :toctree:
+   :recursive:
+{% for item in modules %}
+   {{ fullname }}.{{ item }}
+{%- endfor %}
+{% endif %}
+
+{% if classes %}
+.. currentmodule:: {{ fullname }}
+.. rubric:: {{ _('Classes') }}
+
+.. autosummary::
+   :toctree:
+{% for item in classes %}
+   {{ item }}
+{%- endfor %}
+{% endif %}
+
+{% if functions %}
+.. currentmodule:: {{ fullname }}
+.. rubric:: {{ _('Functions') }}
+
+.. autosummary::
+   :toctree:
+{% for item in functions %}
+   {{ item }}
+{%- endfor %}
+{% endif %}
+
+.. automodule:: {{ fullname }}

docs/source/conf.py

@@ -23,7 +23,6 @@
 import os
 import sys
 from datetime import date
-from unittest import mock
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -33,44 +32,15 @@
 
 # -- Mock imports -------------------------------------------------------------
 
-
-# Mock imports instead of full environment in readthedocs
-MOCK_MODULES = [
+autodoc_mock_imports = [
     "click",
     "mpi4py",
     "cupy",
     "h5py",
-    "yaml",
-    "skimage",
-    "skimage.exposure",
-    "nvtx",
-    "mpi4py.MPI",
-    "scipy",
-    "scipy.ndimage",
+    "hdf5plugin",
+    "httomo_backends",
 ]
 
-for mod_name in MOCK_MODULES:
-    sys.modules[mod_name] = mock.Mock()
-
-
-class CustomMock(mock.Mock):
-    def __repr__(self):
-        return "<cp.ndarray>"
-
-
-sys.modules["cupy"] = CustomMock()
-sys.modules["numpy"] = CustomMock()
-sys.modules["cupyx.scipy.interpolate"] = CustomMock()
-sys.modules["cupyx.scipy.ndimage"] = CustomMock()
-sys.modules["cupyx.scipy.fft"] = CustomMock()
-sys.modules["cupyx.scipy.fftpack"] = CustomMock()
-sys.modules["scipy.fftpack"] = CustomMock()
-sys.modules["tomobar.methodsDIR_CuPy"] = CustomMock()
-sys.modules["tomobar.methodsIR_CuPy "] = CustomMock()
-sys.modules["skimage.registration"] = CustomMock()
-sys.modules["httomolibgpu.cuda_kernels"] = CustomMock()
-
-
 # ------------------------------------------------------------------------------
 
 project = "HTTomo"
@@ -109,7 +79,7 @@ def __repr__(self):
 
 autosummary_generate = True
 numfig = True
-template_patterns = ["_templates"]
+templates_path = ["_templates"]
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = "sphinx"

docs/source/developers/api.rst

@@ -0,0 +1,28 @@
+API
+===
+
+.. autosummary::
+    :recursive:
+    :toctree: generated
+
+    httomo.base_block
+    httomo.block_interfaces
+    httomo.cli
+    httomo.cli_utils
+    httomo.darks_flats
+    httomo.data
+    httomo.globals
+    httomo.loaders
+    httomo.logger
+    httomo.methods
+    httomo.method_wrappers
+    httomo.monitors
+    httomo.preview
+    httomo.runner
+    httomo.sweep_runner
+    httomo.transform_layer
+    httomo.transform_loader_params
+    httomo.types
+    httomo.ui_layer
+    httomo.utils
+    httomo.yaml_checker

docs/source/index.rst

@@ -67,3 +67,4 @@
 
     developers/how_to_contribute
     developers/memory_calculation
+    developers/api

docs/sphinx-build.sh

@@ -10,23 +10,11 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 # members will document all modules
 # undoc keeps modules without docstrings
 # show-inheritance displays a list of base classes below the class signature
-export SPHINX_APIDOC_OPTIONS='members,show-inheritance,undoc-members'
 
 # Remove directory for api so that there are no obsolete files
-rm -rf $DIR/source/api/
+rm -rf $DIR/source/generated/
 rm -rf $DIR/build/
 
-# sphinx-apidoc [options] -o <outputdir> <sourcedir> [pathnames to exclude]
-# -f, --force. Usually, apidoc does not overwrite files, unless this option is given.
-# -e, --separate. Put each module file in its own page.
-# -T, --no-toc. Do not create a table of contents file.
-# -E, --no-headings. Do not create headings for the modules/packages. This is useful, for example, when docstrings already contain headings.
-# -t template directory
-# add -Q to suppress warnings
-
-# sphinx-apidoc generates source files that use sphinx.ext.autodoc to document all found modules
-sphinx-apidoc -feT -t=$DIR/source/_templates -o $DIR/source/api $DIR/../httomo
-
 # sphinx-build [options] <sourcedir> <outputdir> [filenames]
 # -a Write all output files. The default is to only write output files for new and changed source files. (This may not apply to all builders.)
 # -E Don’t use a saved environment (the structure caching all cross-references), but rebuild it completely. The default is to only read and parse source files that are new or have changed since the last run.

httomo/block_interfaces.py

@@ -1,12 +1,9 @@
-from typing import Literal, Protocol, Tuple, TypeAlias, TypeVar
+from typing import Literal, Protocol, Tuple, TypeVar
 
 import numpy as np
 
+from httomo.types import generic_array
 from httomo.runner.auxiliary_data import AuxiliaryData
-from httomo.utils import xp
-
-
-generic_array: TypeAlias = np.ndarray | xp.ndarray
 
 
 class BlockTransfer(Protocol):

httomo/data/padding.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Optional, Union
 
 import h5py
 import numpy as np
@@ -10,7 +10,7 @@
 
 
 def extrapolate_before(
-    global_data: h5py.Dataset | np.ndarray,
+    global_data: Union[h5py.Dataset, np.ndarray],
     block_data: np.ndarray,
     slices: int,
     dim: int,
@@ -50,7 +50,7 @@ def extrapolate_before(
 
 
 def extrapolate_after(
-    global_data: h5py.Dataset | np.ndarray,
+    global_data: Union[h5py.Dataset, np.ndarray],
     block_data: np.ndarray,
     slices: int,
     dim: int,

httomo/loaders/standard_tomo_loader.py

@@ -15,6 +15,7 @@
 from httomo.runner.dataset import DataSetBlock
 from httomo.runner.dataset_store_interfaces import DataSetSource
 from httomo.runner.loader import LoaderInterface
+from httomo.types import generic_array
 from httomo.utils import Pattern, log_once, make_3d_shape_from_shape
 
 
@@ -88,11 +89,11 @@ def dtype(self) -> np.dtype:
         return self._data.dtype
 
     @property
-    def flats(self) -> Optional[AuxiliaryData.generic_array]:
+    def flats(self) -> Optional[generic_array]:
         return self._aux_data.get_flats()
 
     @property
-    def darks(self) -> Optional[AuxiliaryData.generic_array]:
+    def darks(self) -> Optional[generic_array]:
         return self._aux_data.get_darks()
 
     @property

httomo/runner/auxiliary_data.py

@@ -1,6 +1,8 @@
-from typing import Optional, Tuple, Union
-from typing_extensions import TypeAlias
+from typing import Optional, Tuple
+
 import numpy as np
+
+from httomo.types import generic_array
 from httomo.utils import xp, gpu_enabled
 
 
@@ -12,16 +14,14 @@ class AuxiliaryData:
     including the GPU/CPU transfers if needed.
     """
 
-    generic_array: TypeAlias = Union[xp.ndarray, np.ndarray]
-
     def __init__(
         self,
         angles: np.ndarray,
         darks: Optional[np.ndarray] = None,
         flats: Optional[np.ndarray] = None,
     ):
-        self._darks: Optional[AuxiliaryData.generic_array] = darks
-        self._flats: Optional[AuxiliaryData.generic_array] = flats
+        self._darks: Optional[generic_array] = darks
+        self._flats: Optional[generic_array] = flats
         self._angles: np.ndarray = angles
 
     @property

httomo/types.py

@@ -0,0 +1,8 @@
+from typing import TypeAlias, Union
+
+import numpy as np
+
+from httomo.utils import xp
+
+
+generic_array: TypeAlias = Union[np.ndarray, xp.ndarray]