Refactor (#87)

* First round of refactor

* 🔨 fix merge issues

* 🚚 Move components to model

* 🔥 remove artifacts from merge

* Rename logger name

* 🔨 Add wandb to pyproject config

* Address PR comments

* Fix black version

* Fix license + minor refactor

* isort

* Properly import AnomalyModule

* Fix circular import

* address pre-commit

* Rebase development

* fix cyclic imports reported by pylint

* absolute import of AnomalyModule

* Moved anomaly module and dynamic module out of base.anomaly_models

* import AnomalyModule and DynamicBufferModule from anomalib.models.components.base

* reduced coverag percentage

* Refactor import

Co-authored-by: Ashwin Vaidya <[email protected]>
Co-authored-by: Samet Akcay <[email protected]>

.gitignore

@@ -87,6 +87,10 @@ instance/
 # Scrapy stuff:
 .scrapy
 
+# Sphinx documentation
+docs/build/
+docs/source/_build/
+
 # PyBuilder
 .pybuilder/
 target/

CONTRIBUTING.md

@@ -61,4 +61,4 @@ You accept that your contributions will be licensed under the [Apache-2.0 Licens
 
 
 ## References
-This document was adapted from [here](https://gist.github.com/briandk/3d2e8b3ec8daf5a27a62).
+This document was adapted from [here](https://gist.github.com/briandk/3d2e8b3ec8daf5a27a62).

anomalib/config/__init__.py

@@ -1,4 +1,4 @@
-"""Utilities to get configurable parameters."""
+"""Utilities for parsing model configuration."""
 
 # Copyright (C) 2020 Intel Corporation
 #
@@ -14,10 +14,6 @@
 # See the License for the specific language governing permissions
 # and limitations under the License.
 
-from .config import (
-    get_configurable_parameters,
-    update_input_size_config,
-    update_nncf_config,
-)
+from .config import get_configurable_parameters, update_nncf_config
 
-__all__ = ["get_configurable_parameters", "update_input_size_config", "update_nncf_config"]
+__all__ = ["get_configurable_parameters", "update_nncf_config"]

anomalib/data/inference.py

@@ -20,8 +20,8 @@
 import albumentations as A
 from torch.utils.data.dataset import Dataset
 
-from anomalib.data.transforms import PreProcessor
 from anomalib.data.utils import get_image_filenames, read_image
+from anomalib.pre_processing import PreProcessor
 
 
 class InferenceDataset(Dataset):

anomalib/data/mvtec.py

@@ -41,9 +41,8 @@
 from torchvision.datasets.folder import VisionDataset
 
 from anomalib.data.inference import InferenceDataset
-from anomalib.data.transforms import PreProcessor
-from anomalib.data.utils import read_image
-from anomalib.utils.download_progress_bar import DownloadProgressBar
+from anomalib.data.utils import DownloadProgressBar, read_image
+from anomalib.pre_processing import PreProcessor
 
 logger = logging.getLogger(name="Dataset: MVTec")
 logger.setLevel(logging.DEBUG)

anomalib/data/utils/__init__.py

@@ -0,0 +1,20 @@
+"""Helper utilities for data."""
+
+# Copyright (C) 2020 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+
+from .download_progress_bar import DownloadProgressBar
+from .image import get_image_filenames, read_image
+
+__all__ = ["get_image_filenames", "read_image", "DownloadProgressBar"]

anomalib/data/utils.py → anomalib/data/utils/image.py

@@ -1,4 +1,4 @@
-"""Dataset Utils."""
+"""Image Utils."""
 
 # Copyright (C) 2020 Intel Corporation
 #

anomalib/deploy/__init__.py

@@ -1,4 +1,4 @@
-"""Utilities for inference and deployment."""
+"""Functions for Inference and model deployment."""
 
 # Copyright (C) 2020 Intel Corporation
 #
@@ -13,3 +13,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions
 # and limitations under the License.
+
+from .inferencers import OpenVINOInferencer, TorchInferencer
+from .optimize import export_convert, get_model_metadata
+
+__all__ = ["OpenVINOInferencer", "TorchInferencer", "export_convert", "get_model_metadata"]

anomalib/deploy/inferencers/base.py

@@ -23,10 +23,12 @@
 from torch import Tensor
 
 from anomalib.data.utils import read_image
-from anomalib.utils.normalization.cdf import normalize as normalize_cdf
-from anomalib.utils.normalization.cdf import standardize
-from anomalib.utils.normalization.min_max import normalize as normalize_min_max
-from anomalib.utils.post_process import superimpose_anomaly_map
+from anomalib.post_processing import superimpose_anomaly_map
+from anomalib.post_processing.normalization.cdf import normalize as normalize_cdf
+from anomalib.post_processing.normalization.cdf import standardize
+from anomalib.post_processing.normalization.min_max import (
+    normalize as normalize_min_max,
+)
 
 
 class Inferencer(ABC):

anomalib/deploy/inferencers/openvino.py

@@ -22,7 +22,7 @@
 from omegaconf import DictConfig, ListConfig
 from openvino.inference_engine import IECore  # pylint: disable=no-name-in-module
 
-from anomalib.data.transforms.pre_process import PreProcessor
+from anomalib.pre_processing import PreProcessor
 
 from .base import Inferencer
 

anomalib/deploy/inferencers/torch.py

@@ -23,10 +23,10 @@
 from omegaconf import DictConfig, ListConfig
 from torch import Tensor
 
-from anomalib.core.model import AnomalyModule
-from anomalib.data.transforms.pre_process import PreProcessor
 from anomalib.deploy.optimize import get_model_metadata
 from anomalib.models import get_model
+from anomalib.models.components import AnomalyModule
+from anomalib.pre_processing import PreProcessor
 
 from .base import Inferencer
 

anomalib/deploy/optimize.py

@@ -24,7 +24,7 @@
 import torch
 from torch import Tensor
 
-from anomalib.core.model.anomaly_module import AnomalyModule
+from anomalib.models.components import AnomalyModule
 
 
 def get_model_metadata(model: AnomalyModule) -> Dict[str, Tensor]:

anomalib/models/__init__.py

@@ -21,7 +21,7 @@
 from omegaconf import DictConfig, ListConfig
 from torch import load
 
-from anomalib.core.model import AnomalyModule
+from anomalib.models.components import AnomalyModule
 
 
 def get_model(config: Union[DictConfig, ListConfig]) -> AnomalyModule:

anomalib/models/cflow/model.py

@@ -28,9 +28,8 @@
 from pytorch_lightning.callbacks import EarlyStopping
 from torch import Tensor, nn, optim
 
-from anomalib.core.model import AnomalyModule
-from anomalib.core.model.feature_extractor import FeatureExtractor
 from anomalib.models.cflow.backbone import cflow_head, positional_encoding_2d
+from anomalib.models.components import AnomalyModule, FeatureExtractor
 
 __all__ = ["AnomalyMapGenerator", "CflowModel", "CflowLightning"]
 

anomalib/models/components/__init__.py

@@ -0,0 +1,32 @@
+"""Components used within the models."""
+
+# Copyright (C) 2020 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+
+from .base import AnomalyModule, DynamicBufferModule
+from .dimensionality_reduction import PCA, SparseRandomProjection
+from .feature_extractors import FeatureExtractor
+from .sampling import KCenterGreedy
+from .stats import GaussianKDE, MultiVariateGaussian
+
+__all__ = [
+    "AnomalyModule",
+    "DynamicBufferModule",
+    "PCA",
+    "SparseRandomProjection",
+    "FeatureExtractor",
+    "KCenterGreedy",
+    "GaussianKDE",
+    "MultiVariateGaussian",
+]

anomalib/models/components/base/__init__.py

@@ -0,0 +1,20 @@
+"""Base classes for all anomaly components."""
+
+# Copyright (C) 2020 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+
+from .anomaly_module import AnomalyModule
+from .dynamic_module import DynamicBufferModule
+
+__all__ = ["AnomalyModule", "DynamicBufferModule"]

anomalib/core/model/anomaly_module.py → anomalib/models/components/base/anomaly_module.py

@@ -23,7 +23,7 @@
 from torch import Tensor, nn
 from torchmetrics import F1, MetricCollection
 
-from anomalib.core.metrics import (
+from anomalib.utils.metrics import (
     AUROC,
     AdaptiveThreshold,
     AnomalyScoreDistribution,

anomalib/models/components/dimensionality_reduction/__init__.py

@@ -0,0 +1,20 @@
+"""Algorithms for decomposition and dimensionality reduction."""
+
+# Copyright (C) 2020 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+
+from .pca import PCA
+from .random_projection import SparseRandomProjection
+
+__all__ = ["PCA", "SparseRandomProjection"]

anomalib/core/model/pca.py → anomalib/models/components/dimensionality_reduction/pca.py

@@ -19,7 +19,7 @@
 import torch
 from torch import Tensor
 
-from anomalib.core.model.dynamic_module import DynamicBufferModule
+from anomalib.models.components.base import DynamicBufferModule
 
 
 class PCA(DynamicBufferModule):

anomalib/core/model/__init__.py → anomalib/models/components/feature_extractors/__init__.py

@@ -1,4 +1,4 @@
-"""Anomalib Core Model Entities."""
+"""Feature extractors."""
 
 # Copyright (C) 2020 Intel Corporation
 #
@@ -14,6 +14,6 @@
 # See the License for the specific language governing permissions
 # and limitations under the License.
 
-from .anomaly_module import AnomalyModule
+from .feature_extractor import FeatureExtractor
 
-__all__ = ["AnomalyModule"]
+__all__ = ["FeatureExtractor"]

anomalib/models/components/sampling/__init__.py

@@ -0,0 +1,19 @@
+"""Sampling methods."""
+
+# Copyright (C) 2020 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+
+from .k_center_greedy import KCenterGreedy
+
+__all__ = ["KCenterGreedy"]

anomalib/core/model/k_center_greedy.py → anomalib/models/components/sampling/k_center_greedy.py

@@ -11,7 +11,7 @@
 import torch.nn.functional as F
 from torch import Tensor
 
-from anomalib.core.model.random_projection import SparseRandomProjection
+from anomalib.models.components.dimensionality_reduction import SparseRandomProjection
 
 
 class KCenterGreedy:

anomalib/models/components/stats/__init__.py

@@ -0,0 +1,20 @@
+"""Statistical functions."""
+
+# Copyright (C) 2020 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+
+from .kde import GaussianKDE
+from .multi_variate_gaussian import MultiVariateGaussian
+
+__all__ = ["GaussianKDE", "MultiVariateGaussian"]

anomalib/core/model/kde.py → anomalib/models/components/stats/kde.py

@@ -20,7 +20,7 @@
 import torch
 from torch import Tensor
 
-from anomalib.core.model.dynamic_module import DynamicBufferModule
+from anomalib.models.components.base import DynamicBufferModule
 
 
 class GaussianKDE(DynamicBufferModule):

anomalib/models/dfkde/model.py

@@ -22,9 +22,9 @@
 from omegaconf.listconfig import ListConfig
 from torch import Tensor
 
-from anomalib.core.model import AnomalyModule
-from anomalib.core.model.feature_extractor import FeatureExtractor
-from anomalib.models.dfkde.normality_model import NormalityModel
+from anomalib.models.components import AnomalyModule, FeatureExtractor
+
+from .normality_model import NormalityModel
 
 
 class DfkdeLightning(AnomalyModule):

anomalib/models/dfkde/normality_model.py

@@ -20,8 +20,7 @@
 import torch
 from torch import Tensor, nn
 
-from anomalib.core.model.kde import GaussianKDE
-from anomalib.core.model.pca import PCA
+from anomalib.models.components import PCA, GaussianKDE
 
 
 class NormalityModel(nn.Module):

anomalib/models/dfm/dfm_model.py

@@ -19,8 +19,7 @@
 import torch
 from torch import Tensor, nn
 
-from anomalib.core.model.dynamic_module import DynamicBufferModule
-from anomalib.core.model.pca import PCA
+from anomalib.models.components import PCA, DynamicBufferModule
 
 
 class SingleClassGaussian(DynamicBufferModule):

anomalib/models/dfm/model.py

@@ -21,9 +21,9 @@
 from omegaconf import DictConfig, ListConfig
 from torch import Tensor
 
-from anomalib.core.model import AnomalyModule
-from anomalib.core.model.feature_extractor import FeatureExtractor
-from anomalib.models.dfm.dfm_model import DFMModel
+from anomalib.models.components import AnomalyModule, FeatureExtractor
+
+from .dfm_model import DFMModel
 
 
 class DfmLightning(AnomalyModule):

anomalib/models/ganomaly/model.py

@@ -24,7 +24,7 @@
 from pytorch_lightning.callbacks import EarlyStopping
 from torch import Tensor, nn, optim
 
-from anomalib.core.model import AnomalyModule
+from anomalib.models.components import AnomalyModule
 
 from .torch_model import Discriminator, Generator