From 188637b744f957641758ff04ff98394d7e50e8d1 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 16 Oct 2024 11:10:40 +0100 Subject: [PATCH 01/10] Move `generic_array` type alias to separate module --- httomo/block_interfaces.py | 7 ++----- httomo/loaders/standard_tomo_loader.py | 5 +++-- httomo/runner/auxiliary_data.py | 12 ++++++------ httomo/types.py | 8 ++++++++ 4 files changed, 19 insertions(+), 13 deletions(-) create mode 100644 httomo/types.py diff --git a/httomo/block_interfaces.py b/httomo/block_interfaces.py index 064870777..606252b2b 100644 --- a/httomo/block_interfaces.py +++ b/httomo/block_interfaces.py @@ -1,12 +1,9 @@ -from typing import Literal, Protocol, Tuple, TypeAlias, TypeVar +from typing import Literal, Protocol, Tuple, TypeVar import numpy as np +from httomo.types import generic_array from httomo.runner.auxiliary_data import AuxiliaryData -from httomo.utils import xp - - -generic_array: TypeAlias = np.ndarray | xp.ndarray class BlockTransfer(Protocol): diff --git a/httomo/loaders/standard_tomo_loader.py b/httomo/loaders/standard_tomo_loader.py index eba2a892f..36541eec6 100644 --- a/httomo/loaders/standard_tomo_loader.py +++ b/httomo/loaders/standard_tomo_loader.py @@ -15,6 +15,7 @@ from httomo.runner.dataset import DataSetBlock from httomo.runner.dataset_store_interfaces import DataSetSource from httomo.runner.loader import LoaderInterface +from httomo.types import generic_array from httomo.utils import Pattern, log_once, make_3d_shape_from_shape @@ -88,11 +89,11 @@ def dtype(self) -> np.dtype: return self._data.dtype @property - def flats(self) -> Optional[AuxiliaryData.generic_array]: + def flats(self) -> Optional[generic_array]: return self._aux_data.get_flats() @property - def darks(self) -> Optional[AuxiliaryData.generic_array]: + def darks(self) -> Optional[generic_array]: return self._aux_data.get_darks() @property diff --git a/httomo/runner/auxiliary_data.py b/httomo/runner/auxiliary_data.py index 6dd134af1..edc7782c3 100644 --- a/httomo/runner/auxiliary_data.py +++ b/httomo/runner/auxiliary_data.py @@ -1,6 +1,8 @@ -from typing import Optional, Tuple, Union -from typing_extensions import TypeAlias +from typing import Optional, Tuple + import numpy as np + +from httomo.types import generic_array from httomo.utils import xp, gpu_enabled @@ -12,16 +14,14 @@ class AuxiliaryData: including the GPU/CPU transfers if needed. """ - generic_array: TypeAlias = Union[xp.ndarray, np.ndarray] - def __init__( self, angles: np.ndarray, darks: Optional[np.ndarray] = None, flats: Optional[np.ndarray] = None, ): - self._darks: Optional[AuxiliaryData.generic_array] = darks - self._flats: Optional[AuxiliaryData.generic_array] = flats + self._darks: Optional[generic_array] = darks + self._flats: Optional[generic_array] = flats self._angles: np.ndarray = angles @property diff --git a/httomo/types.py b/httomo/types.py new file mode 100644 index 000000000..d29387510 --- /dev/null +++ b/httomo/types.py @@ -0,0 +1,8 @@ +from typing import TypeAlias + +import numpy as np + +from httomo.utils import xp + + +generic_array: TypeAlias = np.ndarray | xp.ndarray From 79e4b77ee25e4265f1422c9468924f5cf8772fdd Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 16 Oct 2024 11:11:35 +0100 Subject: [PATCH 02/10] Use `Union` in `generic_array` type alias See #492 for more details --- httomo/types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/httomo/types.py b/httomo/types.py index d29387510..25aa7a8ae 100644 --- a/httomo/types.py +++ b/httomo/types.py @@ -1,8 +1,8 @@ -from typing import TypeAlias +from typing import TypeAlias, Union import numpy as np from httomo.utils import xp -generic_array: TypeAlias = np.ndarray | xp.ndarray +generic_array: TypeAlias = Union[np.ndarray, xp.ndarray] From 52e2a7a85cc50452ecca226ee35441fdcbfeb8e6 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 16 Oct 2024 11:44:46 +0100 Subject: [PATCH 03/10] Mock imports in docs build with `autodoc_mock_imports` See #490 for more details. Maybe there's differences between `autodoc_mock_imports` and `autosummary_mock_imports`, but for now, `autodoc_mock_imports` seems to do what is needed, so will carry on using `autodoc_mock_imports` untill there's a reason to distinguish between the two. --- docs/source/conf.py | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index c0ef590be..76c7ff992 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -23,7 +23,6 @@ import os import sys from datetime import date -from unittest import mock # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -33,9 +32,7 @@ # -- Mock imports ------------------------------------------------------------- - -# Mock imports instead of full environment in readthedocs -MOCK_MODULES = [ +autodoc_mock_imports = [ "click", "mpi4py", "cupy", @@ -49,28 +46,6 @@ "scipy.ndimage", ] -for mod_name in MOCK_MODULES: - sys.modules[mod_name] = mock.Mock() - - -class CustomMock(mock.Mock): - def __repr__(self): - return "" - - -sys.modules["cupy"] = CustomMock() -sys.modules["numpy"] = CustomMock() -sys.modules["cupyx.scipy.interpolate"] = CustomMock() -sys.modules["cupyx.scipy.ndimage"] = CustomMock() -sys.modules["cupyx.scipy.fft"] = CustomMock() -sys.modules["cupyx.scipy.fftpack"] = CustomMock() -sys.modules["scipy.fftpack"] = CustomMock() -sys.modules["tomobar.methodsDIR_CuPy"] = CustomMock() -sys.modules["tomobar.methodsIR_CuPy "] = CustomMock() -sys.modules["skimage.registration"] = CustomMock() -sys.modules["httomolibgpu.cuda_kernels"] = CustomMock() - - # ------------------------------------------------------------------------------ project = "HTTomo" From 07f4759fd404cbb4610937febfcf7e59fa37bd66 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 16 Oct 2024 11:45:49 +0100 Subject: [PATCH 04/10] Add `hdf5plugin` to sphinx mock imports --- docs/source/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 76c7ff992..8099f28b8 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -37,6 +37,7 @@ "mpi4py", "cupy", "h5py", + "hdf5plugin", "yaml", "skimage", "skimage.exposure", From c43e1360c7ef7d401e13058f265eecd25ac19d85 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 16 Oct 2024 11:54:25 +0100 Subject: [PATCH 05/10] Remove unnecessary mocks in sphinx docs config The modules removed are either no longer used in httomo, or are in the docs environment now, so the mocking of them by sphinx can safely be removed without any issues. --- docs/source/conf.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 8099f28b8..0a29b19be 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -38,13 +38,6 @@ "cupy", "h5py", "hdf5plugin", - "yaml", - "skimage", - "skimage.exposure", - "nvtx", - "mpi4py.MPI", - "scipy", - "scipy.ndimage", ] # ------------------------------------------------------------------------------ From 252bacb89d2a9b65ce176ca290c6b03518471c9b Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 16 Oct 2024 12:09:48 +0100 Subject: [PATCH 06/10] Use `Union` for a union type involving `h5py` See #492 for more details --- httomo/data/padding.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/httomo/data/padding.py b/httomo/data/padding.py index 678651814..fe3e28532 100644 --- a/httomo/data/padding.py +++ b/httomo/data/padding.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, Union import h5py import numpy as np @@ -10,7 +10,7 @@ def extrapolate_before( - global_data: h5py.Dataset | np.ndarray, + global_data: Union[h5py.Dataset, np.ndarray], block_data: np.ndarray, slices: int, dim: int, @@ -50,7 +50,7 @@ def extrapolate_before( def extrapolate_after( - global_data: h5py.Dataset | np.ndarray, + global_data: Union[h5py.Dataset, np.ndarray], block_data: np.ndarray, slices: int, dim: int, From 090ac1df7bb55f000eaee2549e4befe771db2edd Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 16 Oct 2024 12:25:18 +0100 Subject: [PATCH 07/10] Add `httomo_backends` to sphinx mock list The YAML checker needs `httomo_backends` purely for getting a path to the YAML templates at runtime (ie, for functionality), which isn't needed for just building the docs. --- docs/source/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 0a29b19be..84bdc7a00 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -38,6 +38,7 @@ "cupy", "h5py", "hdf5plugin", + "httomo_backends", ] # ------------------------------------------------------------------------------ From 1fe92a149801291e26d5ba81734fa15bd536e186 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Thu, 17 Oct 2024 12:05:42 +0100 Subject: [PATCH 08/10] Generate API docs with sphinx `autosummary` extension --- .gitignore | 2 ++ docs/source/_templates/autosummary/module.rst | 36 +++++++++++++++++++ docs/source/_templates/module.rst_t | 7 ---- docs/source/conf.py | 2 +- docs/source/developers/api.rst | 28 +++++++++++++++ docs/source/index.rst | 1 + docs/sphinx-build.sh | 14 +------- 7 files changed, 69 insertions(+), 21 deletions(-) create mode 100644 docs/source/_templates/autosummary/module.rst delete mode 100644 docs/source/_templates/module.rst_t create mode 100644 docs/source/developers/api.rst diff --git a/.gitignore b/.gitignore index 0e5c67a38..5701967cc 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,5 @@ output_dir/ temp.yaml cufile.log +# Sphinx generated API docs +docs/source/developers/generated/ diff --git a/docs/source/_templates/autosummary/module.rst b/docs/source/_templates/autosummary/module.rst new file mode 100644 index 000000000..1516f105a --- /dev/null +++ b/docs/source/_templates/autosummary/module.rst @@ -0,0 +1,36 @@ +{{ fullname | escape | underline}} + +{% if modules %} +.. rubric:: Modules + +.. autosummary:: + :toctree: + :recursive: +{% for item in modules %} + {{ fullname }}.{{ item }} +{%- endfor %} +{% endif %} + +{% if classes %} +.. currentmodule:: {{ fullname }} +.. rubric:: {{ _('Classes') }} + +.. autosummary:: + :toctree: +{% for item in classes %} + {{ item }} +{%- endfor %} +{% endif %} + +{% if functions %} +.. currentmodule:: {{ fullname }} +.. rubric:: {{ _('Functions') }} + +.. autosummary:: + :toctree: +{% for item in functions %} + {{ item }} +{%- endfor %} +{% endif %} + +.. automodule:: {{ fullname }} diff --git a/docs/source/_templates/module.rst_t b/docs/source/_templates/module.rst_t deleted file mode 100644 index bd7adc13a..000000000 --- a/docs/source/_templates/module.rst_t +++ /dev/null @@ -1,7 +0,0 @@ -{%- if show_headings %}:mod:`{{ basename }}` -============================================ -{%- endif %} -.. automodule:: {{ qualname }} -{%- for option in automodule_options %} - :{{ option }}: -{%- endfor %} diff --git a/docs/source/conf.py b/docs/source/conf.py index 84bdc7a00..e7cb07a6e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -79,7 +79,7 @@ autosummary_generate = True numfig = True -template_patterns = ["_templates"] +templates_path = ["_templates"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" diff --git a/docs/source/developers/api.rst b/docs/source/developers/api.rst new file mode 100644 index 000000000..1a244ab8b --- /dev/null +++ b/docs/source/developers/api.rst @@ -0,0 +1,28 @@ +API +=== + +.. autosummary:: + :recursive: + :toctree: generated + + httomo.base_block + httomo.block_interfaces + httomo.cli + httomo.cli_utils + httomo.darks_flats + httomo.data + httomo.globals + httomo.loaders + httomo.logger + httomo.methods + httomo.method_wrappers + httomo.monitors + httomo.preview + httomo.runner + httomo.sweep_runner + httomo.transform_layer + httomo.transform_loader_params + httomo.types + httomo.ui_layer + httomo.utils + httomo.yaml_checker diff --git a/docs/source/index.rst b/docs/source/index.rst index 9f87d795a..e1fc25491 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -67,3 +67,4 @@ developers/how_to_contribute developers/memory_calculation + developers/api diff --git a/docs/sphinx-build.sh b/docs/sphinx-build.sh index 36a4c8695..051ea1a2f 100644 --- a/docs/sphinx-build.sh +++ b/docs/sphinx-build.sh @@ -10,23 +10,11 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # members will document all modules # undoc keeps modules without docstrings # show-inheritance displays a list of base classes below the class signature -export SPHINX_APIDOC_OPTIONS='members,show-inheritance,undoc-members' # Remove directory for api so that there are no obsolete files -rm -rf $DIR/source/api/ +rm -rf $DIR/source/generated/ rm -rf $DIR/build/ -# sphinx-apidoc [options] -o [pathnames to exclude] -# -f, --force. Usually, apidoc does not overwrite files, unless this option is given. -# -e, --separate. Put each module file in its own page. -# -T, --no-toc. Do not create a table of contents file. -# -E, --no-headings. Do not create headings for the modules/packages. This is useful, for example, when docstrings already contain headings. -# -t template directory -# add -Q to suppress warnings - -# sphinx-apidoc generates source files that use sphinx.ext.autodoc to document all found modules -sphinx-apidoc -feT -t=$DIR/source/_templates -o $DIR/source/api $DIR/../httomo - # sphinx-build [options] [filenames] # -a Write all output files. The default is to only write output files for new and changed source files. (This may not apply to all builders.) # -E Don’t use a saved environment (the structure caching all cross-references), but rebuild it completely. The default is to only read and parse source files that are new or have changed since the last run. From 5ade8db1b504fbd264fa26ff75bf2b33d42e7194 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Thu, 17 Oct 2024 12:08:35 +0100 Subject: [PATCH 09/10] Add rst template for class API docs --- docs/source/_templates/autosummary/class.rst | 32 ++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 docs/source/_templates/autosummary/class.rst diff --git a/docs/source/_templates/autosummary/class.rst b/docs/source/_templates/autosummary/class.rst new file mode 100644 index 000000000..660caa5be --- /dev/null +++ b/docs/source/_templates/autosummary/class.rst @@ -0,0 +1,32 @@ +{{ fullname | escape | underline}} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :members: + + {% block methods %} + {% if methods %} + .. rubric:: {{ _('Methods') }} + + .. autosummary:: + {% for item in methods %} + {% if item not in inherited_members %} + ~{{ name }}.{{ item }} + {% endif %} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Attributes') }} + + .. autosummary:: + {% for item in attributes %} + {% if item not in inherited_members %} + ~{{ name }}.{{ item }} + {% endif %} + {%- endfor %} + {% endif %} + {% endblock %} From ab99a46355e2832000d2f0cd9ab25825156dc036 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Thu, 17 Oct 2024 12:34:57 +0100 Subject: [PATCH 10/10] Remove unnecessary `sphinx-apidoc` step in docs build CI --- .github/workflows/httomo_docs.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/httomo_docs.yml b/.github/workflows/httomo_docs.yml index 3620bfb06..4bda85575 100644 --- a/.github/workflows/httomo_docs.yml +++ b/.github/workflows/httomo_docs.yml @@ -31,10 +31,7 @@ jobs: activate-environment: httomo-docs environment-file: ./docs/source/doc-conda-requirements.yml - - name: Build api docs - run: sphinx-apidoc -feT -t=./docs/source/_templates -o ./docs/source/api ./httomo - - - name: Build html + - name: Build docs run: sphinx-build -a -E -b html ./docs/source/ ./docs/build/ - name: Publish docs