Various changes, primarily testing-related (#492)

* Add discussion on conda-forge installation

* Update license year range

* Add note on miniconda scripts

* Update license year range

* Move --level option config to separate file

* Resolve doctest failures

* Resolve mypy complaint

* Switch to Python 3.10 for CI

* Add CI doctest in docs rst files

* No doctest on macos

* Typo fix

* Change --level option usage

* Add pytest-split and remove pytest-runner

* Update submodule

* Attempt to resolve CI pytest failure

* Update submodule

* Change command line argument option style

* Update submodule

* Update installation docs

* Move tifffile back to main requirements

* Bug fix

* Update imageio usage

* Docstring fix

* Switch to imageio v3

* Update submodule

* Update copyright year

* Switch to imageio v3

* Resolve mypy error

.github/workflows/lint.yml

@@ -15,7 +15,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.9"
+          python-version: "3.10"
       - name: Black code formatter
         uses: psf/black@stable
         with:

.github/workflows/mypy.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Install Python 3
         uses: actions/setup-python@v4
         with:
-          python-version: "3.9"
+          python-version: "3.10"
       - name: Install dependencies
         run: |
           pip install mypy

.github/workflows/pypi_upload.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Install Python 3
         uses: actions/setup-python@v4
         with:
-          python-version: "3.9"
+          python-version: "3.10"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/pytest_latest.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Install Python 3
         uses: actions/setup-python@v4
         with:
-          python-version: "3.9"
+          python-version: "3.10"
       - name: Install lastversion
         run: |
           python -m pip install --upgrade pip

.github/workflows/pytest_macos.yml

@@ -35,7 +35,7 @@ jobs:
             miniforge-version: latest
             activate-environment: test-env
             use-mamba: true
-            python-version: "3.9"
+            python-version: "3.10"
       # Configure conda environment cache
       - name: Set up conda environment cache
         uses: actions/cache@v3
@@ -75,9 +75,4 @@ jobs:
         run: |
           DURATIONS_FILE=$(mktemp)
           bzcat data/pytest/durations_macos.bz2 > $DURATIONS_FILE
-          pytest -x --durations-path $DURATIONS_FILE --splits 5 --group ${{ matrix.group }} --level 1
-      # Run doc tests
-      - name: Run doc tests
-        if: matrix.group == 1
-        run: |
-          pytest --ignore-glob="*test_*.py" --doctest-modules scico
+          pytest -x --level=1 --durations-path=$DURATIONS_FILE --splits=5 --group=${{ matrix.group }} --pyargs scico

.github/workflows/pytest_ubuntu.yml

@@ -35,7 +35,7 @@ jobs:
             miniforge-version: latest
             activate-environment: test-env
             use-mamba: true
-            python-version: "3.9"
+            python-version: "3.10"
       # Configure conda environment cache
       - name: Set up conda environment cache
         uses: actions/cache@v3
@@ -78,7 +78,7 @@ jobs:
         run: |
           DURATIONS_FILE=$(mktemp)
           bzcat data/pytest/durations_ubuntu.bz2 > $DURATIONS_FILE
-          pytest -x --cov --durations-path $DURATIONS_FILE --splits 5 --group ${{ matrix.group }} --level 2
+          pytest -x --cov --level=2 --durations-path=$DURATIONS_FILE --splits=5 --group=${{ matrix.group }} --pyargs scico
       # Upload coverage data
       - name: Upload coverage
         uses: actions/upload-artifact@v3
@@ -90,16 +90,17 @@ jobs:
         if: matrix.group == 1
         run: |
           pytest --ignore-glob="*test_*.py" --doctest-modules scico
+          pytest --doctest-glob="*.rst" docs
 
   coverage:
     needs: test
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.9
+      - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: "3.9"
+          python-version: "3.10"
       - name: Install deps
         run: |
           python -m pip install --upgrade pip

.github/workflows/test_examples.yml

@@ -33,7 +33,7 @@ jobs:
             miniforge-version: latest
             activate-environment: test-env
             use-mamba: true
-            python-version: "3.9"
+            python-version: "3.10"
       # Configure conda environment cache
       - name: Set up conda environment cache
         uses: actions/cache@v3

LICENSE

@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) 2021-2023, Los Alamos National Laboratory
+Copyright (c) 2021-2024, Los Alamos National Laboratory
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

README.rst

@@ -38,6 +38,10 @@
    :target: https://pepy.tech/project/scico
    :alt: PyPI download statistics
 
+.. image:: https://img.shields.io/conda/vn/conda-forge/scico.svg
+    :target: https://anaconda.org/conda-forge/scico
+    :alt: Conda Forge Release
+
 .. image:: https://raw.githubusercontent.com/jupyter/design/master/logos/Badges/nbviewer_badge.svg
    :target: https://nbviewer.jupyter.org/github/lanl/scico-data/tree/main/notebooks/index.ipynb
    :alt: View notebooks at nbviewer
@@ -83,5 +87,5 @@ SCICO is distributed as open-source software under a BSD 3-Clause License (see t
 
 LANL open source approval reference C20091.
 
-(c) 2020-2023. Triad National Security, LLC. All rights reserved.
+(c) 2020-2024. Triad National Security, LLC. All rights reserved.
 This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department of Energy/National Nuclear Security Administration. All rights in the program are reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear Security Administration. The Government has granted for itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare derivative works, distribute copies to the public, perform publicly and display publicly, and to permit others to do so.

conftest.py

@@ -19,33 +19,3 @@ def add_modules(doctest_namespace):
     """Add common modules for use in docstring examples."""
     doctest_namespace["np"] = np
     doctest_namespace["snp"] = snp
-
-
-def pytest_addoption(parser, pluginmanager):
-    """Add --level pytest option.
-
-    Level definitions:
-      1  Critical tests only
-      2  Skip tests that do have a significant impact on coverage
-      3  All standard tests
-      4  Run all tests, including those marked as slow to run
-    """
-    parser.addoption(
-        "--level", action="store", default=3, type=int, help="Set test level to be run"
-    )
-
-
-def pytest_configure(config):
-    """Add marker description."""
-    config.addinivalue_line("markers", "slow: mark test as slow to run")
-
-
-def pytest_collection_modifyitems(config, items):
-    """Skip slow tests depending on selected testing level."""
-    if config.getoption("--level") >= 4:
-        # don't skip tests at level 4 or higher
-        return
-    level_skip = pytest.mark.skip(reason="test not appropriate for selected level")
-    for item in items:
-        if "slow" in item.keywords:
-            item.add_marker(level_skip)

dev_requirements.txt

@@ -1,7 +1,7 @@
 -r requirements.txt
 pylint
 pytest>=7.3.0
-pytest-runner
+pytest-split
 packaging
 pre-commit
 black>=22.3,<23

docs/source/conf/10-project.py

@@ -2,7 +2,7 @@
 
 # General information about the project.
 project = "SCICO"
-copyright = "2020-2023, SCICO Developers"
+copyright = "2020-2024, SCICO Developers"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

docs/source/include/blockarray.rst

@@ -5,11 +5,12 @@ BlockArray
 
 .. testsetup::
 
+   >>> import numpy as np
    >>> import scico
+   >>> import scico.random
+   >>> import scico.linop
    >>> import scico.numpy as snp
    >>> from scico.numpy import BlockArray
-   >>> import numpy as np
-   >>> import jax.numpy
 
 The class :class:`.BlockArray` provides a way to combine arrays of
 different shapes into a single object for use with other SCICO classes.

docs/source/install.rst

@@ -3,9 +3,9 @@
 Installing SCICO
 ================
 
-SCICO requires Python version 3.8 or later. (Version 3.9 is
+SCICO requires Python version 3.8 or later. (Version 3.10 is
 recommended as it is the version under which SCICO has been most
-thoroughly tested.) It is supported on both Linux and macOS, but is
+thoroughly tested.) It is supported on both Linux and MacOS, but is
 not currently supported on Windows due to the limited support for
 ``jaxlib`` on Windows. However, Windows users can use SCICO via the
 `Windows Subsystem for Linux
@@ -15,31 +15,86 @@ exist for using WSL with `CPU only
 with `GPU support
 <https://docs.microsoft.com/en-us/windows/win32/direct3d12/gpu-cuda-in-wsl>`_.
 
+While not required, installation of SCICO and its dependencies within a `Conda <https://conda.io/projects/conda/en/latest/user-guide/index.html>`_ environment
+is recommended. `Scripts <https://github.com/lanl/scico/tree/main/misc/conda>`_
+are provided for creating a `miniconda <https://docs.conda.io/en/latest/miniconda.html>`_ installation and an environment including all primary SCICO dependencies as well as dependencies
+for usage example, testing, and building the documentation.
+
 
 From PyPI
 ---------
 
 The simplest way to install the most recent release of SCICO from
 `PyPI <https://pypi.python.org/pypi/scico/>`_ is
-
 ::
 
    pip install scico
 
+which will install SCICO and its primary dependencies. If the additional
+dependencies for the example scripts are also desired, it can instead be
+installed using
+::
+
+   pip install scico[examples]
+
+Note, however, that since the ``astra-toolbox`` package available from
+PyPI is not straightforward to install (it has numerous build requirements
+that are not specified as package dependencies), it is recommended to
+first install this package via conda
+::
+
+   conda install astra-toolbox
+
+
+
+From conda-forge
+----------------
+
+SCICO can also be installed from `conda-forge <https://anaconda.org/conda-forge/scico>`_
+::
+
+  conda install -c conda-forge "scico>0.0.5"
+
+where the version constraint is required to avoid installation of an old
+package with broken dependencies. Note, however, that installation from conda forge is only straightforward for a Python 3.10 environment on Linux x64, due
+to limitations of conda packages for some of the SCICO dependencies:
+
+* There is no conda package for the secondary dependency ``tensorstore``
+  under MacOS.
+* In a Python 3.9 environment, a version of secondary dependency ``etils``
+  that does not support Python 3.9 will be installed. This can be rectified
+  by
+  ::
+
+     conda install etils=1.5.1
+* Conda packages for dependency ``svmbir`` are not currently available for
+  Python versions greater than 3.10. If an attempt is made to install SCICO
+  via conda forge, an older package with some missing dependencies for the
+  example scripts will be installed. If required, these dependencies
+  (including ``svmbir``, which can be installed using ``pip``) will have to
+  be manually installed.
+
+The most recent SCICO conda forge package also includes dependencies for
+the example scripts, except for ``bm3d``, ``bm4d``, and
+``colour_demosaicing``, for which conda packages are not available. These
+can be installed from PyPI
+::
+
+  pip install bm3d bm4d colour_demosaicing
+
+
 
 From GitHub
 -----------
 
 SCICO can be downloaded from the `GitHub repo
 <https://github.com/lanl/scico>`_. Note that, since the SCICO repo has
 a submodule, it should be cloned via the command
-
 ::
 
    git clone --recurse-submodules [email protected]:lanl/scico.git
 
 Install using the commands
-
 ::
 
    cd scico
@@ -59,13 +114,11 @@ a version with GPU support:
 2. Install the version of jaxlib with GPU support, as described in the `JAX installation
    instructions  <https://jax.readthedocs.io/en/latest/installation.html>`_.
    In the simplest case, the appropriate command is
-
    ::
 
       pip install --upgrade "jax[cuda11]"
 
    for CUDA 11, or
-
    ::
 
       pip install --upgrade "jax[cuda12]"
@@ -74,7 +127,8 @@ a version with GPU support:
    ``jaxlib`` version if the most recent release is not yet supported
    by SCICO (as specified in the ``requirements.txt`` file), or if
    using a version of CUDA older than 11.4, or CuDNN older than 8.2,
-   in which case the command would be of the form ::
+   in which case the command would be of the form
+   ::
 
       pip install --upgrade "jaxlib==0.4.2+cuda11.cudnn82" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
 

examples/examples_requirements.txt

@@ -1,5 +1,4 @@
 -r ../requirements.txt
-tifffile
 colour_demosaicing
 svmbir>=0.3.3
 astra-toolbox

examples/scripts/video_rpca_admm.py

@@ -25,7 +25,7 @@
 have potential application in scientific imaging problems.
 """
 
-import imageio
+import imageio.v3 as iio
 
 import scico.numpy as snp
 from scico import functional, linop, loss, plot
@@ -36,12 +36,9 @@
 """
 Load example video.
 """
-reader = imageio.get_reader("imageio:newtonscradle.gif")
-nfrm = reader.get_length()
-frmlst = []
-for i, frm in enumerate(reader):
-    frmlst.append(rgb2gray(frm[..., 0:3].astype(snp.float32) / 255.0))
-vid = snp.stack(frmlst, axis=2)
+vid = rgb2gray(
+    iio.imread("imageio:newtonscradle.gif").transpose((1, 2, 3, 0)).astype(snp.float32) / 255.0
+)
 
 
 """

requirements.txt

@@ -2,6 +2,7 @@ typing_extensions
 numpy>=1.20.0
 scipy>=1.6.0
 imageio>=2.17
+tifffile
 matplotlib
 jaxlib>=0.4.3,<=0.4.23
 jax>=0.4.3,<=0.4.23

scico/data/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2021-2023 by SCICO Developers
+# Copyright (C) 2021-2024 by SCICO Developers
 # All rights reserved. BSD 3-clause License.
 # This file is part of the SCICO package. Details of the copyright and
 # user license can be found in the 'LICENSE' file distributed with the
@@ -10,7 +10,7 @@
 import os.path
 from typing import Optional
 
-from imageio.v2 import imread
+from imageio.v3 import imread
 
 import scico.numpy as snp