pip install, CI/CD enhancements (Project-MONAI#1033)

* CI CD enhancements Signed-off-by: Wenqi Li <[email protected]> * remove temp tests Signed-off-by: Wenqi Li <[email protected]> * improves error msg for python versions Signed-off-by: Wenqi Li <[email protected]>
thibaultdvx · Sep 14, 2020 · 1a41750 · 1a41750
1 parent fa521a9
commit 1a41750
Show file tree

Hide file tree

Showing 10 changed files with 163 additions and 83 deletions.
diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml
@@ -35,7 +35,7 @@ jobs:
         echo $CUDA_VISIBLE_DEVICES
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
-        ./runtests.sh --coverage
+        BUILD_MONAI=1 ./runtests.sh --coverage
         coverage xml
     - name: Upload coverage
       uses: codecov/codecov-action@v1
@@ -58,7 +58,7 @@ jobs:
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
         ngc --version
-        ./runtests.sh --coverage --pytype
+        BUILD_MONAI=1 ./runtests.sh --coverage --pytype
         coverage xml
     - name: Upload coverage
       uses: codecov/codecov-action@v1

diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml
@@ -84,9 +84,9 @@ jobs:
         cat "requirements-dev.txt"
         python -m pip install -r requirements-dev.txt
         python -m pip list
-        SKIP_MONAI_BUILD=1 python setup.py develop
+        python setup.py develop
         python setup.py develop --uninstall
-        python setup.py develop  # compile the cpp extensions
+        BUILD_MONAI=1 python setup.py develop  # compile the cpp extensions
       shell: bash
     - name: Run quick tests (CPU ${{ runner.os }})
       run: |
@@ -95,6 +95,56 @@ jobs:
       env:
         QUICKTEST: True
 
+  min-dep-py3:  # min dependencies installed
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest, macOS-latest, ubuntu-latest]
+    timeout-minutes: 20
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.x
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.x'
+    - name: Prepare pip wheel
+      run: |
+        which python
+        python -m pip install --upgrade pip wheel
+    - name: cache weekly timestamp
+      id: pip-cache
+      run: |
+        echo "::set-output name=datew::$(date '+%Y-%V')"
+        echo "::set-output name=dir::$(pip cache dir)"
+      shell: bash
+    - name: cache for pip
+      uses: actions/cache@v2
+      id: cache
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: ${{ matrix.os }}-latest-pip-${{ steps.pip-cache.outputs.datew }}
+    - if: runner.os == 'windows'
+      name: Install torch cpu from pytorch.org (Windows only)
+      run: |
+        python -m pip install torch==1.4 -f https://download.pytorch.org/whl/cpu/torch_stable.html
+    - name: Install the dependencies
+      run: |
+        # min. requirements for windows instances
+        python -m pip install torch==1.4
+        python -c "f=open('requirements-dev.txt', 'r'); txt=f.readlines(); f.close(); print(txt); f=open('requirements-dev.txt', 'w'); f.writelines(txt[1:5]); f.close()"
+        cat "requirements-dev.txt"
+        python -m pip install -r requirements-dev.txt
+        python -m pip list
+        BUILD_MONAI=0 python setup.py develop  # no compile of extensions
+      shell: bash
+    - name: Run quick tests (CPU ${{ runner.os }})
+      run: |
+        python -c 'import torch; print(torch.__version__); print(torch.rand(5,3))'
+        python -m tests.min_tests
+      env:
+        QUICKTEST: True
+
   GPU-quick-py3:  # GPU with full dependencies
     strategy:
       matrix:
@@ -170,11 +220,11 @@ jobs:
       run: |
         python -m pip list
         nvidia-smi
-        export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
+        export CUDA_VISIBLE_DEVICES=$(coverage run -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
-        ./runtests.sh --quick
+        BUILD_MONAI=1 ./runtests.sh --quick
         if [ ${{ matrix.environment }} == "PT16+CUDA110" ]; then
           # test the clang-format tool downloading once
           coverage run -m tests.clang_format_utils
@@ -274,4 +324,7 @@ jobs:
     - name: Make html
       run: |
         cd docs/
-        make html
+        make clean
+        make html 2>&1 | tee tmp_log
+        if [[ $(grep -c "^WARNING:" tmp_log) != 0 ]]; then echo "found warnings"; exit 1; fi
+      shell: bash
diff --git a/.github/workflows/setupapp.yml b/.github/workflows/setupapp.yml
@@ -43,7 +43,7 @@ jobs:
         echo $CUDA_VISIBLE_DEVICES
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
-        ./runtests.sh --coverage
+        BUILD_MONAI=1 ./runtests.sh --coverage
         coverage xml
     - name: Upload coverage
       uses: codecov/codecov-action@v1
@@ -83,65 +83,15 @@ jobs:
       run: |
         python -m pip list
         python -c 'import torch; print(torch.__version__); print(torch.rand(5,3))'
-        ./runtests.sh --quick
+        BUILD_MONAI=1 ./runtests.sh --quick
         coverage xml
     - name: Upload coverage
       uses: codecov/codecov-action@v1
       with:
         fail_ci_if_error: false
         file: ./coverage.xml
 
-  min-dep-py3:  # min dependencies installed
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [windows-latest, macOS-latest, ubuntu-latest]
-    timeout-minutes: 20
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.x
-      uses: actions/setup-python@v1
-      with:
-        python-version: '3.x'
-    - name: Prepare pip wheel
-      run: |
-        which python
-        python -m pip install --upgrade pip wheel
-    - name: cache weekly timestamp
-      id: pip-cache
-      run: |
-        echo "::set-output name=datew::$(date '+%Y-%V')"
-        echo "::set-output name=dir::$(pip cache dir)"
-      shell: bash
-    - name: cache for pip
-      uses: actions/cache@v2
-      id: cache
-      with:
-        path: ${{ steps.pip-cache.outputs.dir }}
-        key: ${{ matrix.os }}-latest-pip-${{ steps.pip-cache.outputs.datew }}
-    - if: runner.os == 'windows'
-      name: Install torch cpu from pytorch.org (Windows only)
-      run: |
-        python -m pip install torch==1.4 -f https://download.pytorch.org/whl/cpu/torch_stable.html
-    - name: Install the dependencies
-      run: |
-        # min. requirements for windows instances
-        python -m pip install torch==1.4
-        python -c "f=open('requirements-dev.txt', 'r'); txt=f.readlines(); f.close(); print(txt); f=open('requirements-dev.txt', 'w'); f.writelines(txt[1:5]); f.close()"
-        cat "requirements-dev.txt"
-        python -m pip install -r requirements-dev.txt
-        python -m pip list
-        SKIP_MONAI_BUILD=1 python setup.py develop  # no compile of extensions
-      shell: bash
-    - name: Run quick tests (CPU ${{ runner.os }})
-      run: |
-        python -c 'import torch; print(torch.__version__); print(torch.rand(5,3))'
-        python -m tests.min_tests
-      env:
-        QUICKTEST: True
-
-  install:
+  install:  # pip install from github url
     runs-on: ubuntu-latest
     steps:
     - name: Set up Python 3.7
@@ -158,14 +108,19 @@ jobs:
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ steps.pip-cache.outputs.datew }}
-    - name: Install the default branch
-      run: |
-        pip install git+https://github.com/Project-MONAI/MONAI#egg=MONAI
-    - name: Import
+    - name: Install the default branch no build
       run: |
+        BUILD_MONAI=0 pip install git+https://github.com/Project-MONAI/MONAI#egg=MONAI
         python -c 'import monai; monai.config.print_config()'
-    - name: Uninstall
+        cd $(python -c 'import monai; import os; print(os.path.dirname(monai.__file__))')
+        ls .
+        pip uninstall -y monai
+    - name: Install the default branch with build
       run: |
+        BUILD_MONAI=1 pip install git+https://github.com/Project-MONAI/MONAI#egg=MONAI
+        python -c 'import monai; monai.config.print_config()'
+        cd $(python -c 'import monai; import os; print(os.path.dirname(monai.__file__))')
+        ls .
         pip uninstall -y monai
 
   docker:

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -3,6 +3,3 @@ include monai/_version.py
 
 include README.md
 include LICENSE
-
-recursive-exclude * __pycache__
-recursive-exclude * *.py[co]
diff --git a/docs/source/installation.md b/docs/source/installation.md
@@ -40,6 +40,10 @@ for the latest features:
 ```bash
 pip install git+https://github.com/Project-MONAI/MONAI#egg=MONAI
 ```
+or, to build with MONAI Cpp/CUDA extensions:
+```bash
+BUILD_MONAI=1 pip install git+https://github.com/Project-MONAI/MONAI#egg=MONAI
+```
 this command will download and install the current master branch of [MONAI from
 GitHub](https://github.com/Project-MONAI/MONAI).
 
@@ -55,17 +59,25 @@ You can install it by running:
 ```bash
 cd MONAI/
 python setup.py develop
+```
+or, to build with MONAI Cpp/CUDA extensions:
+```bash
+cd MONAI/
+BUILD_MONAI=1 python setup.py develop
+# for MacOS
+BUILD_MONAI=1 CC=clang CXX=clang++ python setup.py develop
+```
 
-# For MacOS:
-# CC=clang CXX=clang++ python setup.py develop
-
-# To install without build
-# SKIP_MONAI_BUILD=1 python setup.py develop
-
-# To uninstall the package please run:
+To uninstall the package please run:
+```bash
+cd MONAI/
 python setup.py develop --uninstall
+
+# to further clean up the MONAI/ folder (Bash script)
+./runtests.sh --clean
 ```
-or simply adding the root directory of the cloned source code (e.g., ``/workspace/Documents/MONAI``) to your ``$PYTHONPATH``
+
+Alternatively, simply adding the root directory of the cloned source code (e.g., ``/workspace/Documents/MONAI``) to your ``$PYTHONPATH``
 and the codebase is ready to use (without the additional features of MONAI C++/CUDA extensions).
 
 

diff --git a/monai/__init__.py b/monai/__init__.py
@@ -13,7 +13,9 @@
 import sys
 
 from ._version import get_versions
-from .utils.module import load_submodules
+
+PY_REQUIRED_MAJOR = 3
+PY_REQUIRED_MINOR = 6
 
 __version__ = get_versions()["version"]
 del get_versions
@@ -22,6 +24,15 @@
 
 __basedir__ = os.path.dirname(__file__)
 
+if not (sys.version_info.major == PY_REQUIRED_MAJOR and sys.version_info.minor >= PY_REQUIRED_MINOR):
+    raise RuntimeError(
+        "MONAI requires Python {}.{} or higher. But the current Python is: {}".format(
+            PY_REQUIRED_MAJOR, PY_REQUIRED_MINOR, sys.version
+        ),
+    )
+
+from .utils.module import load_submodules  # noqa: E402
+
 # handlers_* have some external decorators the users may not have installed
 # *.so files and folder "_C" may not exist when the cpp extensions are not compiled
 excludes = "(^(monai.handlers))|((\\.so)$)|(^(monai._C))"

diff --git a/runtests.sh b/runtests.sh
@@ -95,6 +95,11 @@ function print_usage {
     exit 1
 }
 
+function check_import {
+    echo "PYTHON: $(which python)"
+    ${cmdPrefix}python -c "import monai"
+}
+
 function print_version {
     ${cmdPrefix}python -c 'import monai; monai.config.print_config()'
 }
@@ -106,6 +111,8 @@ function install_deps {
 
 function compile_cpp {
     echo "Compiling and installing MONAI cpp extensions..."
+    # depends on setup.py behaviour for building
+    # currently setup.py uses environment variables: BUILD_MONAI and FORCE_CUDA
     ${cmdPrefix}python setup.py -v develop --uninstall
     if [[ "$OSTYPE" == "darwin"* ]];
     then  # clang for mac os
@@ -138,6 +145,7 @@ function clean_py {
 
     find ${TO_CLEAN} -type f -name "*.py[co]" -delete
     find ${TO_CLEAN} -type f -name ".coverage" -delete
+    find ${TO_CLEAN} -type f -name "*.so" -delete
     find ${TO_CLEAN} -type d -name "__pycache__" -delete
 
     find ${TO_CLEAN} -depth -type d -name ".eggs" -exec rm -r "{}" +
@@ -248,7 +256,7 @@ cd "$homedir"
 
 # python path
 export PYTHONPATH="$homedir:$PYTHONPATH"
-echo "$PYTHONPATH"
+echo "PYTHONPATH: $PYTHONPATH"
 
 # by default do nothing
 cmdPrefix=""
@@ -260,8 +268,11 @@ then
     # commands are echoed instead of ran
     cmdPrefix="dryrun "
     function dryrun { echo "    " "$@"; }
+else
+    check_import
 fi
 
+
 if [ $doCleanup = true ]
 then
     echo "${separator}${blue}clean${noColor}"

diff --git a/setup.cfg b/setup.cfg
@@ -63,7 +63,7 @@ exclude = *.pyi,.git,.eggs,monai/_version.py,versioneer.py,venv,_version.py
 known_first_party = monai
 profile = black
 line_length = 120
-skip = .git, .eggs, venv, versioneer.py, _version.py, conf.py
+skip = .git, .eggs, venv, versioneer.py, _version.py, conf.py, monai/__init__.py
 skip_glob = *.pyi
 
 [versioneer]