Organize anomalib dependencies (#32)

* format the txt files

* Fixed request version conflicts

* bumped up the openvino version

* use setuptools in pyproject.toml

* remove ote-sdk from dockerfile

* Read anomalib version via a new function

* Added MANIFEST.in file to add requirements to sdist

* Removed flit metadata from pyproject.toml file

* Temporarily hardcode version in setup.py

* Address tox issues

* adress mypy issues

* Update MANIFEST.in

* added issue to hard-coded version in setup.py

* Mypy typing

* Fix fixed tmp path in tests

* Update base.txt

* version is 0.2.2

.github/workflows/docs.yml

@@ -14,9 +14,9 @@ jobs:
         uses: actions/checkout@v2
       - name: Install requirements
         run: |
-          pip install -r requirements/requirements.txt
-          pip install -r requirements/requirements_dev.txt
-          pip install -r requirements/requirements_openvino_mo.txt
+          pip install -r requirements/base.txt
+          pip install -r requirements/dev.txt
+          pip install -r requirements/openvino.txt
       - name: Build and Commit Docs
         run: |
           cd docs

Dockerfile

@@ -32,17 +32,12 @@ RUN wget ftp://guest:[email protected]/mvtec_anomaly_detection/mvtec_
     tar -xf mvtec_anomaly_detection.tar.xz -C /tmp/anomalib/datasets/MVTec
 
 # Install all anomalib requirements
-COPY ./requirements/requirements.txt /tmp/anomalib/requirements/requirements.txt
-RUN pip install -r /tmp/anomalib/requirements/requirements.txt
+COPY ./requirements/base.txt /tmp/anomalib/requirements/base.txt
+RUN pip install -r /tmp/anomalib/requirements/base.txt
 
-COPY ./requirements/requirements_openvino_mo.txt /tmp/anomalib/requirements/requirements_openvino_mo.txt
-RUN pip install -r /tmp/anomalib/requirements/requirements_openvino_mo.txt
+COPY ./requirements/openvino.txt /tmp/anomalib/requirements/openvino.txt
+RUN pip install -r /tmp/anomalib/requirements/openvino.txt
 
 # Install other requirements related to development
-COPY ./requirements/requirements_dev.txt /tmp/anomalib/requirements/requirements_dev.txt
-RUN pip install -r /tmp/anomalib/requirements/requirements_dev.txt
-
-# Install ote_sdk
-COPY ./impt/src/ote_sdk/ /tmp/impt/src/ote_sdk/
-RUN cd /tmp/impt/src/ote_sdk && \
-    pip install -e .
+COPY ./requirements/dev.txt /tmp/anomalib/requirements/dev.txt
+RUN pip install -r /tmp/anomalib/requirements/dev.txt

MANIFEST.in

@@ -0,0 +1 @@
+recursive-include requirements *

README.md

@@ -43,7 +43,7 @@ To perform an installation, run the following:
 ```
 yes | conda create -n anomalib python=3.8
 conda activate anomalib
-pip install -r requirements/requirements.txt
+pip install -r requirements/base.txt
 ```
 
 ## Training

anomalib/__init__.py

@@ -14,4 +14,4 @@
 # See the License for the specific language governing permissions
 # and limitations under the License.
 
-__version__ = "0.2.1"
+__version__ = "0.2.2"

anomalib/core/callbacks/visualizer_callback.py

@@ -100,14 +100,14 @@ def on_test_batch_end(
             self._add_images(visualizer, pl_module, Path(filename))
             visualizer.close()
 
-    def on_test_end(self, trainer: pl.Trainer, pl_module: pl.LightningModule) -> None:
+    def on_test_end(self, _trainer: pl.Trainer, pl_module: pl.LightningModule) -> None:
         """Sync logs.
 
         Currently only ``AnomalibWandbLogger`` is called from this method. This is because logging as a single batch
         ensures that all images appear as part of the same step.
 
         Args:
-            trainer (pl.Trainer): Pytorch Lightning trainer
+            _trainer (pl.Trainer): Pytorch Lightning trainer
             pl_module (pl.LightningModule): Anomaly module
         """
         if pl_module.logger is not None and isinstance(pl_module.logger, AnomalibWandbLogger):

anomalib/data/tiler.py

@@ -196,11 +196,11 @@ def __init__(
         self.num_patches_w: int
 
     @staticmethod
-    def __validate_size_type(parameter) -> Tuple:
+    def __validate_size_type(parameter) -> Tuple[int, int]:
         if isinstance(parameter, int):
-            output = (parameter,) * 2
+            output = (parameter, parameter)
         elif isinstance(parameter, Sequence):
-            output = tuple(parameter)
+            output = (parameter[0], parameter[1])
         else:
             raise ValueError(f"Unknown type {type(parameter)} for tile or stride size. Could be int or Sequence type.")
 

pyproject.toml

@@ -1,13 +1,6 @@
 [build-system]
-requires = ["flit_core >=2,<4"]
-build-backend = "flit_core.buildapi"
-
-[tool.flit.metadata]
-module = "anomalib"
-author = "Intel OpenVINO"
-author-email = "[email protected]"
-description-file = "README.md"
-requires-python = ">=3.7"
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
 
 [tool.isort]
 profile = "black"

requirements/base.txt

@@ -0,0 +1,17 @@
+albumentations==1.1.0
+attrdict==2.0.1
+defusedxml==0.7.1
+kornia==0.5.6
+lxml==4.6.3
+matplotlib==3.4.3
+networkx==2.5.1
+nncf==2.0.0
+numpy~=1.19.5
+omegaconf==2.1.1
+pillow==8.3.2
+pytorch-lightning==1.3.6
+torch==1.8.1
+torchvision==0.9.1
+scikit-image>=0.17.2
+scikit-learn==0.24.2
+wandb==0.12.9

requirements/requirements_dev.txt → requirements/dev.txt

@@ -1,13 +1,13 @@
-pre-commit>=2.15.0
-tox>=3.24.3
-mypy
+black
+coverage
 flake8
+flaky
+isort
+mypy
 pytest
 pylint
-black
-isort
-coverage
+pre-commit>=2.15.0
+setuptools
+tox>=3.24.3
 types-PyYAML
 types-setuptools
-flaky
-setuptools

docs/requirements.txt → requirements/docs.txt

@@ -1,5 +1,5 @@
-sphinx>=4.1.2
-sphinxemoji==0.1.8
 furo==2021.7.31b41
 myst-parser
+sphinx>=4.1.2
 sphinx-autoapi
+sphinxemoji==0.1.8

requirements/openvino.txt

@@ -0,0 +1,6 @@
+defusedxml==0.7.1
+requests==2.26.0
+networkx~=2.5
+onnx==1.10.1
+opencv-python==4.5.3.56
+openvino-dev==2021.4.2

setup.py

@@ -13,27 +13,90 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions
 # and limitations under the License.
+from pathlib import Path
+from typing import List
 
 from setuptools import find_packages, setup
 
-import anomalib
 
-install_requires = []
-with open("requirements/requirements.txt", "r", encoding="utf8") as f:
-    for line in f:
-        line = line.strip()
-        if line and not line.startswith("#"):
-            install_requires.append(line)
+def get_version() -> str:
+    """Get version from `anomalib.__init__`.
+
+    Version is stored in the main __init__ module in `anomalib`.
+    The varible storing the version is `__version__`. This function
+    reads `__init__` file, checks `__version__ variable and return
+    the value assigned to it.
+
+    Example:
+        >>> # Assume that __version__ = "0.2.1"
+        >>> get_version()
+        "0.2.1"
+
+    Returns:
+        str: Version number of `anomalib` package.
+    """
+
+    with open(Path.cwd() / "anomalib" / "__init__.py", "r", encoding="utf8") as file:
+        lines = file.readlines()
+        for line in lines:
+            line = line.strip()
+            if line.startswith("__version__"):
+                version = line.replace("__version__ = ", "")
+
+    return version
+
+
+def get_required_packages(requirement_files: List[str]) -> List[str]:
+    """Get packages from requirements.txt file.
+
+    This function returns list of required packages from requirement files.
+
+    Args:
+        requirement_files (List[str]): txt files that contains list of required
+            packages.
+
+    Example:
+        >>> get_required_packages(requirement_files=["openvino"])
+        ['onnx>=1.8.1', 'networkx~=2.5', 'openvino-dev==2021.4.1', ...]
+
+    Returns:
+        List[str]: List of required packages
+    """
+
+    required_packages: List[str] = []
+
+    for requirement_file in requirement_files:
+        with open(f"requirements/{requirement_file}.txt", "r", encoding="utf8") as file:
+            for line in file:
+                package = line.strip()
+                if package and not package.startswith(("#", "-f")):
+                    required_packages.append(package)
+
+    return required_packages
+
+
+VERSION = get_version()
+INSTALL_REQUIRES = get_required_packages(requirement_files=["base"])
+EXTRAS_REQUIRE = {
+    "dev": get_required_packages(requirement_files=["dev", "docs"]),
+    "openvino": get_required_packages(requirement_files=["openvino"]),
+    "full": get_required_packages(requirement_files=["dev", "docs", "openvino"]),
+}
 
 setup(
     name="anomalib",
-    version=anomalib.__version__,
-    packages=find_packages(include=["anomalib", "anomalib.*"]),
+    # TODO: https://github.com/openvinotoolkit/anomalib/issues/36
+    version="0.2.2",
+    author="Intel OpenVINO",
+    author_email="[email protected]",
+    description="anomalib - Anomaly Detection Library",
     url="",
     license="Copyright (c) Intel - All Rights Reserved. "
     'Licensed under the Apache License, Version 2.0 (the "License")'
     "See LICENSE file for more details.",
-    install_requires=install_requires,
-    author="Intel",
-    description="anomalib - Anomaly Detection Library",
+    python_requires=">=3.8",
+    packages=find_packages("."),
+    install_requires=INSTALL_REQUIRES,
+    extras_require=EXTRAS_REQUIRE,
+    package_data={"": ["config.yaml"]},
 )

tests/datasets/test_transforms.py

@@ -7,6 +7,9 @@
         deserializes the transformations.
 """
 
+import tempfile
+from pathlib import Path
+
 import albumentations as A
 import numpy as np
 import pytest
@@ -39,6 +42,8 @@ def test_load_transforms_from_string():
     should work with either string or A.Compose and return a ValueError
     otherwise."""
 
+    config_path = tempfile.NamedTemporaryFile(suffix=".yaml").name
+
     # Create a dummy transformation.
     transforms = A.Compose(
         [
@@ -47,10 +52,10 @@ def test_load_transforms_from_string():
             A.Resize(224, 224, always_apply=True),
         ]
     )
-    A.save(transform=transforms, filepath="/tmp/transforms.yaml", data_format="yaml")
+    A.save(transform=transforms, filepath=config_path, data_format="yaml")
 
     # Pass a path to config
-    pre_processor = PreProcessor(config="/tmp/transforms.yaml")
+    pre_processor = PreProcessor(config=config_path)
     assert isinstance(pre_processor.transforms, A.Compose)
 
     # Pass a config of type A.Compose