From f7770b8f015539d518fdfb52fd0fe7b96daf08af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Potrykus?= <52150545+potipot@users.noreply.github.com> Date: Thu, 31 Oct 2024 12:39:03 +0100 Subject: [PATCH] Fix icevision master branch (#1174) * squashed chhanges from other PRs * add importorskip on fiftyone * fix tests * update ci to python 3.9, dev extras includes all * black formatting * fix build-pkg workflow * fix build-pkg python version * fix fridge class map and expected results * update expected results * use ubuntu-20.04 * use updated model checkpoint * update dependencies * use pre-release model link * add skips for mmcv and mmseg tests * update workflows * update to yolov5==6.2.0 * use yaml.safe_load in yolo cfg loading * use python native types casting * use register in detection matching * black formatting * disable soft dependency check on mmseg and mmdet * add swap action to fix failing tests * update mk-docs ci to use python 3.9 * reenable mmseg tests * reenable mmdet tests * update and use installation script * unskip torchvision backbones tests * update mmseg and mmdet configs according to the installed versions * fix yolov5 disabling notebook img display * update dependencies * black formatting * add tests * remove unused code --- .github/workflows/build-pkg.yml | 3 +- .github/workflows/ci-all-testing.yml | 12 +- .github/workflows/mk-docs-build.yml | 10 +- icevision/core/bbox.py | 3 + icevision/core/record_components.py | 1 + .../data/convert_records_to_coco_style.py | 134 ---------- icevision/data/data_splitter.py | 27 ++ icevision/data/dataset.py | 15 +- .../confusion_matrix/confusion_matrix.py | 21 +- .../confusion_matrix_utils.py | 252 +++++++++++++++--- .../binary_dice_coefficient.py | 4 +- .../jaccard_index/binary_jaccard_index.py | 4 +- icevision/models/__init__.py | 6 + icevision/models/mmdet/download_configs.py | 4 +- icevision/models/mmseg/download_configs.py | 16 +- icevision/models/ultralytics/yolov5/model.py | 11 +- icevision/parsers/coco_parser.py | 2 +- icevision/visualize/draw_data.py | 2 +- icevision/visualize/utils.py | 2 +- icevision_install.sh | 25 +- setup.cfg | 25 +- tests/conftest.py | 4 +- tests/data/test_convert_records_to_fo.py | 28 +- tests/data/test_record_collection.py | 30 ++- tests/metrics/expected_register_p2t.pkl | Bin 0 -> 607 bytes tests/metrics/expected_register_t2p.pkl | Bin 0 -> 830 bytes tests/metrics/test_confusion_matrix.py | 62 +---- tests/models/efficient_det/conftest.py | 1 + tests/models/efficient_det/test_metrics.py | 27 +- tests/models/efficient_det/test_prediction.py | 6 +- tests/models/fastai/unet/test_backbones.py | 2 - tests/models/mmseg/fastai/test_train.py | 1 + tests/models/mmseg/test_show_results.py | 1 + .../keypoints_rcnn/test_backbones.py | 2 - .../mask_rcnn/test_backbones.py | 2 - .../retinanet/test_backbones.py | 2 - .../batch_tfms/test_img_pad_stack.py | 4 +- 37 files changed, 407 insertions(+), 344 deletions(-) create mode 100644 tests/metrics/expected_register_p2t.pkl create mode 100644 tests/metrics/expected_register_t2p.pkl diff --git a/.github/workflows/build-pkg.yml b/.github/workflows/build-pkg.yml index e0a50562c..773c392dd 100644 --- a/.github/workflows/build-pkg.yml +++ b/.github/workflows/build-pkg.yml @@ -16,13 +16,12 @@ jobs: - uses: actions/checkout@master - uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.9 - name: Build package run: python setup.py sdist - name: Install package run: | -# pip install numpy pip install -e . python -c "from icevision.all import *" diff --git a/.github/workflows/ci-all-testing.yml b/.github/workflows/ci-all-testing.yml index 266df5de2..093143be2 100644 --- a/.github/workflows/ci-all-testing.yml +++ b/.github/workflows/ci-all-testing.yml @@ -12,8 +12,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-18.04] - python-version: [3.7, 3.8] + os: [ubuntu-20.04] + python-version: [3.9] steps: - uses: actions/checkout@v2 @@ -25,7 +25,7 @@ jobs: - name: Install package run: | sh ./icevision_install.sh cpu - pip install -e ".[all,dev]" + pip install -e .[dev] pip install fiftyone - name: Lint with flake8 @@ -34,6 +34,12 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Set Swap Space + uses: pierotofy/set-swap-space@master + with: + swap-size-gb: 10 + - name: Unit tests run: pytest tests -m "not cuda" --cov=icevision --cov-report=xml --color=yes diff --git a/.github/workflows/mk-docs-build.yml b/.github/workflows/mk-docs-build.yml index 5fa85bd42..9adb36bfe 100644 --- a/.github/workflows/mk-docs-build.yml +++ b/.github/workflows/mk-docs-build.yml @@ -13,7 +13,7 @@ env: jobs: build-docs: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 with: @@ -22,14 +22,10 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Install package - run: | - sh ./icevision_install.sh cpu - pip install -e ".[all,dev]" - - + run: pip install -e .[dev] - name: Prepare the docs run: | cd docs diff --git a/icevision/core/bbox.py b/icevision/core/bbox.py index 4c0cc21b6..17cb72fbf 100644 --- a/icevision/core/bbox.py +++ b/icevision/core/bbox.py @@ -36,6 +36,9 @@ def __eq__(self, other) -> bool: return self.xyxy == other.xyxy return False + def __hash__(self): + return hash(self.xyxy) + @property def width(self): return self.xmax - self.xmin diff --git a/icevision/core/record_components.py b/icevision/core/record_components.py index 91fa0bdbf..239bbf4ea 100644 --- a/icevision/core/record_components.py +++ b/icevision/core/record_components.py @@ -269,6 +269,7 @@ def _autofix(self) -> Dict[str, bool]: def _remove_annotation(self, i): self.label_ids.pop(i) + self.labels.pop(i) def _aggregate_objects(self) -> Dict[str, List[dict]]: return {**super()._aggregate_objects(), "labels": self.label_ids} diff --git a/icevision/data/convert_records_to_coco_style.py b/icevision/data/convert_records_to_coco_style.py index 784834494..b1d2aa303 100644 --- a/icevision/data/convert_records_to_coco_style.py +++ b/icevision/data/convert_records_to_coco_style.py @@ -6,7 +6,6 @@ "coco_api_from_records", "coco_api_from_preds", "create_coco_eval", - "export_batch_inferences_as_coco_annotations", ] from icevision.imports import * @@ -14,139 +13,6 @@ from icevision.core import * from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval -import json -import numpy as np -import PIL - - -class NpEncoder(json.JSONEncoder): - """ - Smooths out datatype conversions for IceVision preds to JSON export - """ - - def default(self, obj): - if isinstance(obj, np.integer): - return int(obj) - if isinstance(obj, np.floating): - return float(obj) - if isinstance(obj, np.ndarray): - return obj.tolist() - return json.JSONEncoder.default(self, obj) - - -def export_batch_inferences_as_coco_annotations( - preds, - img_files, - transforms, - class_map, - output_filepath="inference_results_as_coco_annotations.json", - info=None, - licenses=None, -): - """ - For converting object detection predictions to COCO annotation format. - Useful for e.g. leveraging partly-trained models to help annotate - unlabeled data and make round trips back into annotation programs. - - Parameters - ---------- - preds : List[Prediction] - The result of predict_from_dl() - img_files : fastcore.foundation.L (i.e. 'Paths') - References to the original image filepaths in array-like form. - transforms : Albumentations Adapter - Transforms that were applied to the original images (to be reversed) - class_map : icevision.core.class_map.ClassMap - The map of classes your model is familiar with - output_filepath : str, optional - The filepath (including filename) where you want the json results - to be serialized, by default - "new_pseudo_labels_for_further_training.json" - info: dict, optional - Option to manually create the info dict containing annotation metadata - including year, version, description, contributor, url, and date created - For example: - "info": { - "year": "2022", - "version": "1", - "description": "Exported from IceVision", - "contributor": "Awesome contributor", - "url": "https://lazyannotator.fun", - "date_created": "2022-08-05T20:13:09+00:00" - } - licenses: List[dict], optional - Option to manually create the license metadata for the annotations, e.g. - licenses = [ - { - "name": "Creative Commons Attribution 4.0", - "id": 0, - "url": "https://creativecommons.org/licenses/by/4.0/legalcode", - } - ] - - Returns - ------- - None - This just spits out a serialized .json file and returns nothing. - """ - object_category_list = [ - {"id": v, "name": k, "supercategory": ""} - for k, v in class_map._class2id.items() - ] - - if info is None: - # Then automatically generate COCO annotation metadata: - info = { - "contributor": "", - "date_created": "", - "description": "", - "url": "", - "version": "", - "year": "", - } - - if licenses is None: - licenses = [ - { - "name": "", - "id": 0, - "url": "", - } - ] - - addl_info = { - "licenses": licenses, - "info": info, - "categories": object_category_list, - } - - # Each entry needs a filepath - [pred.add_component(FilepathRecordComponent()) for pred in preds] - [preds[_].set_filepath(img_files[_]) for _ in range(len(preds))] - - # process_bbox_predictions happens inplace, thus no new variable - for p in preds: - process_bbox_predictions( - p, PIL.Image.open(Path(p.pred.filepath)), transforms.tfms_list - ) - - coco_style_preds = convert_preds_to_coco_style(preds) - imgs_array = [PIL.Image.open(Path(fname)) for fname in img_files] - - sizes = [{"x": img._size[0], "y": img._size[1]} for img in imgs_array] - - for idx, image in enumerate(coco_style_preds["images"]): - coco_style_preds["images"][idx]["width"] = sizes[idx]["x"] - coco_style_preds["images"][idx]["height"] = sizes[idx]["y"] - - finalized_pseudo_labels = {**addl_info, **coco_style_preds} - - # Serialize - with open(output_filepath, "w") as jfile: - json.dump(finalized_pseudo_labels, jfile, cls=NpEncoder) - - # Print confirmation message - print(f"New COCO annotation file saved to {output_filepath}") def create_coco_api(coco_records) -> COCO: diff --git a/icevision/data/data_splitter.py b/icevision/data/data_splitter.py index 515222827..463387b8f 100644 --- a/icevision/data/data_splitter.py +++ b/icevision/data/data_splitter.py @@ -4,6 +4,7 @@ "RandomSplitter", "FixedSplitter", "FuncSplitter", + "FolderSplitter", ] from icevision.imports import * @@ -132,3 +133,29 @@ def __init__(self, func): def split(self, records: Sequence[BaseRecord]): return self.func(records) + + +class FolderSplitter(DataSplitter): + """ + Split items into subsets based on provided keywords. + Set of items not containing any of the keywords is returned as first. + """ + + def __init__(self, keywords=Collection[str]): + self.keywords = keywords + + def split(self, records: Sequence[BaseRecord]): + """ + Splits records based on the provided keywords by their filepaths. + If some records don't match any keywords, they are returned as an additional split. + """ + remainder_set = {record.record_id for record in records} + keyword_sets = [set() for _ in self.keywords] + for record in records: + for keyword, other_set in zip(self.keywords, keyword_sets): + if keyword in record.filepath.as_posix(): + other_set.add(record.record_id) + remainder_set -= set.union(*keyword_sets) + if remainder_set: + keyword_sets.append(remainder_set) + return keyword_sets diff --git a/icevision/data/dataset.py b/icevision/data/dataset.py index 260b8f9b2..6f67600e1 100644 --- a/icevision/data/dataset.py +++ b/icevision/data/dataset.py @@ -32,13 +32,16 @@ def __len__(self): return len(self.records) def __getitem__(self, i): - record = self.records[i].load() - if self.tfm is not None: - record = self.tfm(record) + if isinstance(i, slice): + return self.__class__(self.records[i], self.tfm) else: - # HACK FIXME - record.set_img(np.array(record.img)) - return record + record = self.records[i].load() + if self.tfm is not None: + record = self.tfm(record) + else: + # HACK FIXME + record.set_img(np.array(record.img)) + return record def __repr__(self): return f"<{self.__class__.__name__} with {len(self.records)} items>" diff --git a/icevision/metrics/confusion_matrix/confusion_matrix.py b/icevision/metrics/confusion_matrix/confusion_matrix.py index c949f7a0e..d65ee5bf7 100644 --- a/icevision/metrics/confusion_matrix/confusion_matrix.py +++ b/icevision/metrics/confusion_matrix/confusion_matrix.py @@ -10,6 +10,7 @@ class MatchingPolicy(Enum): BEST_SCORE = 1 BEST_IOU = 2 + ALL = 3 class SimpleConfusionMatrix(Metric): @@ -41,7 +42,7 @@ def accumulate(self, preds: Collection[Prediction]): # if not target_record.detection.bboxes: # continue # create matches based on iou - matches = match_records( + register = match_predictions_to_targets( target=target_record, prediction=prediction_record, iou_threshold=self._iou_threshold, @@ -49,10 +50,10 @@ def accumulate(self, preds: Collection[Prediction]): target_labels, predicted_labels = [], [] # iterate over multiple targets and preds in a record - for target_item, prediction_items in matches: + for target_item in register: if self._policy == MatchingPolicy.BEST_SCORE: - predicted_item = get_best_score_item( - prediction_items=prediction_items, + match, iou = get_best_score_match( + prediction_items=register[target_item] ) elif self._policy == MatchingPolicy.BEST_IOU: raise NotImplementedError @@ -60,8 +61,8 @@ def accumulate(self, preds: Collection[Prediction]): raise RuntimeError(f"policy must be one of {list(MatchingPolicy)}") # using label_id instead of named label to save memory - target_label = target_item["target_label_id"] - predicted_label = predicted_item["predicted_label_id"] + target_label = target_item.label_id + predicted_label = match.label_id target_labels.append(target_label) predicted_labels.append(predicted_label) @@ -144,8 +145,8 @@ def _maybe_normalize(self, cm, normalize): def log(self, logger_object) -> None: # TODO: Disabled for now, need to design for metric logging for this to work + pl dependency - # if isinstance(logger_object, pl_loggers.WandbLogger): - # fig = self.plot() - # image = self._fig2img(fig) - # logger_object.experiment.log({"Confusion Matrix": wandb.Image(image)}) + if isinstance(logger_object, pl_loggers.WandbLogger): + fig = self.plot() + image = self._fig2img(fig) + logger_object.experiment.log({"Confusion Matrix": wandb.Image(image)}) return diff --git a/icevision/metrics/confusion_matrix/confusion_matrix_utils.py b/icevision/metrics/confusion_matrix/confusion_matrix_utils.py index 5835c8968..3d3b35e3b 100644 --- a/icevision/metrics/confusion_matrix/confusion_matrix_utils.py +++ b/icevision/metrics/confusion_matrix/confusion_matrix_utils.py @@ -1,16 +1,112 @@ +import collections + from icevision.imports import * from icevision import BBox, BaseRecord -def get_best_score_item(prediction_items: Collection[Dict]): +@dataclass(frozen=True) +class ObjectDetectionItem: + bbox: BBox + label: str + label_id: int + record_id: int + item_id: int + + # def __eq__(self, other): + # return ( + # self.record_id == other.record_id + # and self.item_id == other.item_id + # and self.__class__ == other.__class__ + # ) + + +@dataclass(frozen=True) +class ObjectDetectionTarget(ObjectDetectionItem): + # matches: Collection[ObjectDetectionItem] = dataclasses.field(default_factory=list) + pass + + +@dataclass(frozen=True) +class ObjectDetectionPrediction(ObjectDetectionItem): + score: float = None + + +@dataclass(frozen=True) +class ObjectDetectionMatch(ObjectDetectionItem): + item: ObjectDetectionItem + iou_score: float + + +class Register(collections.UserDict): + def __init__(self, allow_duplicates=False): + super().__init__() + self.allow_duplicates = allow_duplicates + # self.data = defaultdict(dict) + + def update(self, other=(), /, **kwds): + self.data.update(other, **kwds) + + def __setitem__(self, key, value: Dict): + # TODO: change self.data to defaultdict with duplicates or not and move this logic to the inner dict + duplicate_items = set() + for item, iou in value.items(): + if not self.allow_duplicates: + for existing_key, matches_dict in self.data.items(): + if item in matches_dict.keys(): + existing_iou = matches_dict[item] + if existing_iou >= iou: + duplicate_items.add(item) + else: + del self.data[existing_key][item] + for item in duplicate_items: + del value[item] + # allow multiple matches per prediction + self[key].update(value) + # single, best iou match per prediction: + if len(self.data[key].values()) > 1: + self.data[key] = dict([max(self[key].items(), key=lambda item: item[1])]) + + def filter_by_label_id(self, label_id: int): + register_view = Register(allow_duplicates=self.allow_duplicates) + for key, matches_dict in self.data.items(): + if key.label_id == label_id: + register_view.data[key] = {} + for inner_key, iou in matches_dict.items(): + if inner_key.label_id == label_id: + register_view.update({key: {inner_key: iou}}) + return register_view + + def register_keys(self, prediction_list): + self.data = {prediction: {} for prediction in prediction_list} + + +def default_item(): + return ( + ObjectDetectionPrediction( + bbox=BBox.from_xyxy(0, 0, 0, 0), + score=0.0, + label_id=0, + label="background", + record_id=-1, + item_id=-1, + ), + 0.0, + ) + + +def get_best_score_match(prediction_items: Dict[ObjectDetectionItem, float]): # fill with dummy if list of prediction_items is empty - dummy = dict( - predicted_bbox=BBox.from_xyxy(0, 0, 0, 0), - score=1.0, - iou_score=1.0, - predicted_label_id=0, + best_item = max( + prediction_items.items(), key=lambda x: x[0].score, default=default_item() + ) + return best_item + + +def get_best_iou_match(prediction_items: Dict[ObjectDetectionItem, float]): + # fill with dummy if list of prediction_items is empty + best_item = max( + prediction_items.items(), key=lambda x: x[1], default=default_item() ) - best_item = max(prediction_items, key=lambda x: x["score"], default=dummy) return best_item @@ -28,45 +124,135 @@ def pairwise_iou_record_record(target: BaseRecord, prediction: BaseRecord): return torchvision.ops.box_iou(stacked_preds, stacked_targets) -def match_records( - target: BaseRecord, prediction: BaseRecord, iou_threshold: float = 0.5 -) -> Collection: +def pairwise_iou_list_list( + target_list: List[ObjectDetectionItem], prediction_list: List[ObjectDetectionItem] +) -> torch.Tensor: """ - matches bboxes, labels from targets with their predictions by iou threshold + Calculates pairwise iou on prediction and target BaseRecord. Uses torchvision implementation of `box_iou`. """ - # here we get a tensor of indices that match iou criteria (order is (pred_id, target_id)) - iou_table = pairwise_iou_record_record(target=target, prediction=prediction) - pairs_indices = torch.nonzero(iou_table > iou_threshold) + stacked_preds = [prediction.bbox.to_tensor() for prediction in prediction_list] + stacked_preds = torch.stack(stacked_preds) if stacked_preds else torch.empty(0, 4) + + stacked_targets = [target.bbox.to_tensor() for target in target_list] + stacked_targets = ( + torch.stack(stacked_targets) if stacked_targets else torch.empty(0, 4) + ) + return torchvision.ops.box_iou(stacked_preds, stacked_targets) + - # creating a list of [target, matching_predictions] +def build_target_list(target: BaseRecord) -> List: + record_id = target.record_id target_list = [ - [dict(target_bbox=bbox, target_label=label, target_label_id=label_id), []] - for bbox, label, label_id in zip( - target.detection.bboxes, target.detection.labels, target.detection.label_ids + ObjectDetectionTarget( + record_id=record_id, + item_id=item_id, + bbox=bbox, + label=label, + label_id=label_id, + ) + for item_id, (bbox, label, label_id) in enumerate( + zip( + target.detection.bboxes, + target.detection.labels, + target.detection.label_ids, + ) ) ] + + return target_list + + +def build_prediction_list(prediction: BaseRecord) -> List: + record_id = prediction.record_id prediction_list = [ - dict( - predicted_bbox=bbox, - predicted_label=label, - predicted_label_id=label_id, + ObjectDetectionPrediction( + record_id=record_id, + item_id=item_id, + bbox=bbox, + label=label, + label_id=label_id, score=score, ) - for bbox, label, label_id, score in zip( - prediction.detection.bboxes, - prediction.detection.labels, - prediction.detection.label_ids, - prediction.detection.scores, + for item_id, (bbox, label, label_id, score) in enumerate( + zip( + prediction.detection.bboxes, + prediction.detection.labels, + prediction.detection.label_ids, + prediction.detection.scores, + ) ) ] - # appending matches to targets + return prediction_list + + +def match_predictions_to_targets( + target: BaseRecord, prediction: BaseRecord, iou_threshold: float = 0.5 +) -> Collection: + """ + matches bboxes, labels from targets with their predictions by iou threshold + """ + # here we get a tensor of indices that match iou criteria (order is (pred_id, target_id)) + iou_table = pairwise_iou_record_record(target=target, prediction=prediction) + pairs_indices = torch.nonzero(iou_table > iou_threshold) + + target_list = build_target_list(target) + prediction_list = build_prediction_list(prediction) + register = Register(allow_duplicates=True) + register.register_keys(target_list) + + # appending matching predictions to targets for pred_id, target_id in pairs_indices: - single_prediction = deepcopy(prediction_list[pred_id]) + single_prediction = prediction_list[pred_id] + single_target = target_list[target_id] # python value casting needs rounding cause otherwise there are 0.69999991 values iou_score = round(iou_table[pred_id, target_id].item(), 4) - single_prediction["iou_score"] = iou_score - # seems like a magic number, but we want to append to the list of target's matching_predictions - target_list[target_id][1].append(single_prediction) + register[single_target] = {single_prediction: iou_score} - return target_list + return register + + +def match_targets_to_predictions( + target: BaseRecord, + prediction: BaseRecord, + iou_threshold: float = 0.5, + use_coco_matching=True, +) -> Collection: + """ + matches bboxes, labels from targets with their predictions by iou threshold + """ + + target_list = build_target_list(target) + prediction_list = sorted( + build_prediction_list(prediction), key=lambda item: item.score, reverse=True + ) + # here we get a tensor of indices that match iou criteria (order is (pred_id, target_id)) + iou_table = pairwise_iou_list_list( + target_list=target_list, prediction_list=prediction_list + ) + pairs_indices = torch.nonzero(iou_table > iou_threshold) + register = Register(allow_duplicates=False) + register.register_keys(prediction_list) + # appending matching targets to predictions + for pred_id, target_id in pairs_indices: + single_target = target_list[target_id] + single_prediction = prediction_list[pred_id] + if use_coco_matching: + # match only same class bboxes + if single_target.label != single_prediction.label: + continue + + # python value casting needs rounding cause otherwise there are 0.69999991 values + iou_score = round(iou_table[pred_id, target_id].item(), 4) + register[single_prediction] = {single_target: iou_score} + + return register + + +class NoCopyRepeat(nn.Module): + def __init__(self, out_channels=3): + super().__init__() + self.out_channels = out_channels + + def forward(self, x): + return x.expand(self.out_channels, -1, -1) diff --git a/icevision/metrics/dice_coefficient/binary_dice_coefficient.py b/icevision/metrics/dice_coefficient/binary_dice_coefficient.py index 200183da0..cf0592c98 100644 --- a/icevision/metrics/dice_coefficient/binary_dice_coefficient.py +++ b/icevision/metrics/dice_coefficient/binary_dice_coefficient.py @@ -24,13 +24,13 @@ def accumulate(self, preds): pred = ( np.stack([x.pred.segmentation.mask_array.data for x in preds]) - .astype(np.bool) + .astype(bool) .flatten() ) target = ( np.stack([x.ground_truth.segmentation.mask_array.data for x in preds]) - .astype(np.bool) + .astype(bool) .flatten() ) diff --git a/icevision/metrics/jaccard_index/binary_jaccard_index.py b/icevision/metrics/jaccard_index/binary_jaccard_index.py index ddc6a1668..441eadce8 100644 --- a/icevision/metrics/jaccard_index/binary_jaccard_index.py +++ b/icevision/metrics/jaccard_index/binary_jaccard_index.py @@ -22,13 +22,13 @@ def accumulate(self, preds): pred = ( np.stack([x.pred.segmentation.mask_array.data for x in preds]) - .astype(np.bool) + .astype(bool) .flatten() ) target = ( np.stack([x.ground_truth.segmentation.mask_array.data for x in preds]) - .astype(np.bool) + .astype(bool) .flatten() ) diff --git a/icevision/models/__init__.py b/icevision/models/__init__.py index cee7c8240..0a6fa66f4 100644 --- a/icevision/models/__init__.py +++ b/icevision/models/__init__.py @@ -18,12 +18,18 @@ if SoftDependencies.yolov5: # HACK: yolov5 changes matplotlib backend here: https://github.com/ultralytics/yolov5/blob/77415a42e5975ea356393c9f1d5cff0ae8acae2c/utils/plots.py#L26 import matplotlib + from IPython import get_ipython backend = matplotlib.get_backend() from icevision.models import ultralytics matplotlib.use(backend) matplotlib.rcdefaults() + session = get_ipython() + shell = session.__class__.__module__ + # HACK: yolov5 breaks automatic setting of backend setting by notebook + if shell in ["google.colab._shell", "ipykernel.zmqshell"]: + session.run_line_magic("matplotlib", "inline") if SoftDependencies.mmseg: from icevision.models import mmseg diff --git a/icevision/models/mmdet/download_configs.py b/icevision/models/mmdet/download_configs.py index e273e4138..d1ad88678 100644 --- a/icevision/models/mmdet/download_configs.py +++ b/icevision/models/mmdet/download_configs.py @@ -3,7 +3,7 @@ from icevision.imports import * from icevision.utils import * -VERSION = "v2.20.1" +VERSION = "v2.25.0" BASE_URL = "https://github.com/airctic/mmdetection_configs/archive/refs/tags" @@ -15,7 +15,7 @@ def download_mmdet_configs() -> Path: if mmdet_config_path.exists(): logger.info( - f"The mmdet config folder already exists. No need to downloaded it. Path : {mmdet_config_path}" + f"The mmdet config folder already exists. No need to download it. Path : {mmdet_config_path}" ) elif download_path.exists(): # The zip file was downloaded by not extracted yet diff --git a/icevision/models/mmseg/download_configs.py b/icevision/models/mmseg/download_configs.py index 9e9ee0ae9..16f361e93 100644 --- a/icevision/models/mmseg/download_configs.py +++ b/icevision/models/mmseg/download_configs.py @@ -3,34 +3,32 @@ from icevision.imports import * from icevision.utils import * -# VERSION = "v2.10.0" # TODO: Update -VERSION = "v0.17.0" -BASE_URL = "https://codeload.github.com/Orbis-International/mmsegmentation_configs/zip/refs/tags" +VERSION = "0.29.1" +BASE_URL = f"https://github.com/potipot/icevision/releases/download/0.13.0/mmsegmentation_configs-{VERSION}.zip" def download_mmseg_configs() -> Path: save_dir = get_root_dir() / f"mmsegmentation_configs" - mmseg_config_path = save_dir / f"mmsegmentation_configs-{VERSION[1:]}/configs" - download_path = save_dir / f"{VERSION}.zip" + mmseg_config_path = save_dir / Path(BASE_URL).stem / "configs" + download_path = save_dir / f"{Path(BASE_URL).stem}.zip" if mmseg_config_path.exists(): logger.info( - f"The mmseg config folder already exists. No need to downloaded it. Path : {mmseg_config_path}" + f"The mmseg config folder already exists. No need to download it. Path : {mmseg_config_path}" ) elif download_path.exists(): # The zip file was downloaded by not extracted yet # Extract zip file - logger.info(f"Extracting the {VERSION}.zip file.") + logger.info(f"Extracting the {download_path.name} file.") save_dir = Path(download_path).parent shutil.unpack_archive(filename=str(download_path), extract_dir=str(save_dir)) else: save_dir.mkdir(parents=True, exist_ok=True) - download_path = save_dir / f"{VERSION}.zip" if not download_path.exists(): logger.info("Downloading mmseg configs") - download_and_extract(f"{BASE_URL}/{VERSION}", download_path) + download_and_extract(BASE_URL, download_path) return mmseg_config_path diff --git a/icevision/models/ultralytics/yolov5/model.py b/icevision/models/ultralytics/yolov5/model.py index 693bbe26e..8ac43ecc8 100644 --- a/icevision/models/ultralytics/yolov5/model.py +++ b/icevision/models/ultralytics/yolov5/model.py @@ -7,8 +7,7 @@ import yolov5 from yolov5.models.yolo import Model from yolov5.utils.downloads import attempt_download -from yolov5.utils.torch_utils import intersect_dicts -from yolov5.utils.general import check_img_size +from yolov5.utils.general import check_img_size, intersect_dicts from icevision.models.ultralytics.yolov5.utils import * from icevision.models.ultralytics.yolov5.backbones import * @@ -45,8 +44,8 @@ def model( if pretrained: weights_path = yolo_dir / f"{model_name}.pt" - with open(Path(yolov5.__file__).parent / "data/hyps/hyp.finetune.yaml") as f: - hyp = yaml.load(f, Loader=yaml.SafeLoader) + with open(Path(yolov5.__file__).parent / "data/hyps/hyp.VOC.yaml") as f: + hyp = yaml.safe_load(f) attempt_download(weights_path) # download if not found locally sys.path.insert(0, str(Path(yolov5.__file__).parent)) @@ -64,8 +63,8 @@ def model( ) # intersect model.load_state_dict(state_dict, strict=False) # load else: - with open(Path(yolov5.__file__).parent / "data/hyps/hyp.scratch.yaml") as f: - hyp = yaml.load(f, Loader=yaml.SafeLoader) + with open(Path(yolov5.__file__).parent / "data/hyps/hyp.scratch-med.yaml") as f: + hyp = yaml.safe_load(f) model = Model( cfg_filepath, ch=3, nc=num_classes, anchors=hyp.get("anchors") diff --git a/icevision/parsers/coco_parser.py b/icevision/parsers/coco_parser.py index c5cd91789..b19daede1 100644 --- a/icevision/parsers/coco_parser.py +++ b/icevision/parsers/coco_parser.py @@ -79,7 +79,7 @@ def iscrowds(self, o) -> List[bool]: def parse_fields(self, o, record, is_new): if is_new: record.set_filepath(self.filepath(o)) - record.set_img_size(self.img_size(o)) + record.set_img_size(self.img_size(o), original=True) # TODO: is class_map still a issue here? record.detection.set_class_map(self.class_map) diff --git a/icevision/visualize/draw_data.py b/icevision/visualize/draw_data.py index e6fb6d324..aee2905a4 100644 --- a/icevision/visualize/draw_data.py +++ b/icevision/visualize/draw_data.py @@ -186,7 +186,7 @@ def draw_sample( # then set color accordingly if color_map is not None: color = as_rgb_tuple(color_map[label]) - color = np.array(color).astype(np.float) + color = np.array(color).astype(np.float32) if display_mask and mask is not None: img = draw_mask( img=img, diff --git a/icevision/visualize/utils.py b/icevision/visualize/utils.py index 4a9f3c103..4801e58c8 100644 --- a/icevision/visualize/utils.py +++ b/icevision/visualize/utils.py @@ -44,7 +44,7 @@ def as_rgb_tuple(x: Union[np.ndarray, tuple, list, str]) -> tuple: if not len(x) == 3: raise ValueError(f"Expected 3 (RGB) numbers, got {len(x)}") if isinstance(x, np.ndarray): - return tuple(x.astype(np.int)) + return tuple(x.astype(int)) elif isinstance(x, tuple): return x elif isinstance(x, list): diff --git a/icevision_install.sh b/icevision_install.sh index cf9a5d948..666265d7b 100644 --- a/icevision_install.sh +++ b/icevision_install.sh @@ -9,32 +9,23 @@ # !bash icevision_install.sh cpu target="${1}" -case ${target} in - cuda10) - echo "Installing icevision + dependencices for ${1}" - echo "- Installing torch and its dependencies" - pip install torch==1.10.0+cu102 torchvision==0.11.1+cu102 torchtext==0.11.0 -f https://download.pytorch.org/whl/torch_stable.html --upgrade -q - - echo "- Installing mmcv" - pip install mmcv-full==1.3.17 -f https://download.openmmlab.com/mmcv/dist/cu102/torch1.10.0/index.html --upgrade -q - ;; - +case ${target} in cuda11) echo "Installing icevision + dependencices for ${1}" echo "- Installing torch and its dependencies" - pip install torch==1.10.0+cu111 torchvision==0.11.1+cu111 torchtext==0.11.0 -f https://download.pytorch.org/whl/torch_stable.html --upgrade + pip install torch==1.13.1+cu117 torchvision==0.14.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 --upgrade echo "- Installing mmcv" - pip install mmcv-full==1.3.17 -f https://download.openmmlab.com/mmcv/dist/cu111/torch1.10.0/index.html --upgrade -q - ;; + pip install mmcv-full==1.7.0 -f https://download.openmmlab.com/mmcv/dist/cu117/torch1.13/index.html + ;; cpu) echo "Installing icevision + dependencices for ${1}" echo "- Installing torch and its dependencies" - pip install torch=="1.10.0+cpu" torchvision=="0.11.1+cpu" torchtext==0.11.0 -f https://download.pytorch.org/whl/torch_stable.html + pip install torch==1.13.1+cpu torchvision==0.14.1+cpu --extra-index-url https://download.pytorch.org/whl/cpu --upgrade echo "- Installing mmcv" - pip install mmcv-full=="1.3.17" -f https://download.openmmlab.com/mmcv/dist/cpu/torch1.10.0/index.html --upgrade -q + pip install mmcv-full==1.7.0 -f https://download.openmmlab.com/mmcv/dist/cpu/torch1.13/index.html --upgrade -q ;; *) @@ -45,10 +36,10 @@ esac echo "- Installing mmdet" -pip install mmdet==2.17.0 --upgrade -q +pip install mmdet==2.26.0 --upgrade -q echo "- Installing mmseg" -pip install mmsegmentation==0.20.2 --upgrade -q +pip install mmsegmentation==0.29.1 --upgrade -q icevision_version="${2}" diff --git a/setup.cfg b/setup.cfg index b1fa8aec2..f35b4fca3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,32 +13,27 @@ license = Apache-2.0 classifiers = Development Status :: 4 - Beta Intended Audience :: Developers -# Programming Language :: Python :: 3.6 ## numpy >= 1.20.0 dropped support for python 3.6 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 Topic :: Scientific/Engineering :: Artificial Intelligence Topic :: Scientific/Engineering :: Image Recognition [options] -python_requires = >=3.7,<4 +python_requires = >=3.9,<4 zip_safe = False include_package_data = True packages = find: install_requires = pillow >8.0.0,<9 - torch >=1.9.0,<1.11 - torchvision >=0.10.0,<0.12 - fastcore >=1.3.0,<1.4 + torch == 1.13.1 + torchvision == 0.14.1 + fastcore >=1.3.8,<1.5 tqdm >=4.49.0,<5 opencv-python >=4.1.1,<5 albumentations >=1.0.3,<1.1 resnest >=0.0.6b20201125,<0.0.7 - effdet >=0.2.1,<0.3 + effdet >=0.4.1,<0.5 sahi >=0.11.1,<0.12.0 - yolov5-icevision >=6.0.0 - importlib-metadata >=1;python_version<"3.8" - ipykernel >=4.10.1,<6 - dataclasses ==0.6 + yolov5 ==6.2.0 loguru >=0.5.3 rasterio>=1.3a1, <2 @@ -48,8 +43,8 @@ inference = icevision all = - fastai >=2.5.2,<2.6 - pytorch-lightning >=1.5.0 + fastai >=2.7.10,<2.8 + pytorch-lightning >=1.5.0,<2 wandb >=0.10.7 dev = @@ -60,7 +55,7 @@ dev = mkdocs >=1.1.2,<2 mkdocs-material >=7.3.1,<8 mike >=1.0 - jupyter >=1.0.0,<2 + notebook >=6.0.0,<7 Sphinx >=3.1.0,<4 pytest-cov >=2.10.1,<3 flake8 >=3.8.3,<4 diff --git a/tests/conftest.py b/tests/conftest.py index 64d819c01..46d2caa10 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,7 +19,7 @@ def coco_record_id_map(): @pytest.fixture() def fridge_efficientdet_model() -> nn.Module: - WEIGHTS_URL = "https://github.com/airctic/model_zoo/releases/download/m2/fridge_tf_efficientdet_lite0.pt" + WEIGHTS_URL = "https://github.com/potipot/icevision/releases/download/0.13.0/fridge_tf_efficientdet_lite0.pt" # TODO: HACK 5+1 in num_classes (becaues of change in model.py) backbone = models.ross.efficientdet.backbones.tf_lite0(pretrained=False) model = efficientdet.model(backbone=backbone, num_classes=5, img_size=384) @@ -157,7 +157,7 @@ def voc_class_map(): @pytest.fixture(scope="session") def fridge_class_map(): - classes = sorted({"milk_bottle", "carton", "can", "water_bottle"}) + classes = ["carton", "milk_bottle", "can", "water_bottle"] return ClassMap(classes) diff --git a/tests/data/test_convert_records_to_fo.py b/tests/data/test_convert_records_to_fo.py index 17b57b6a5..f8236df17 100644 --- a/tests/data/test_convert_records_to_fo.py +++ b/tests/data/test_convert_records_to_fo.py @@ -2,7 +2,9 @@ from venv import create import pytest from PIL import Image -from fiftyone import list_datasets, load_dataset, Detection, Sample, Dataset +import pytest + +fo = pytest.importorskip("fiftyone") from icevision import tfms from icevision import data @@ -20,27 +22,27 @@ ) from tests.conftest import object_detection_record + # Fixtures @pytest.fixture(scope="function") def cleanup_fo_dataset(): dataset_test_names = ["_iv_test", "_iv_test_1"] yield for dataset_name in dataset_test_names: - if dataset_name in list_datasets(): - ds = load_dataset(dataset_name) + if dataset_name in fo.list_datasets(): + ds = fo.load_dataset(dataset_name) ds.delete() del ds # Test from low to high abstraction def test_record_to_fo_detections(object_detection_record): - detections = record_to_fo_detections( object_detection_record, lambda bbox: _convert_bbox_to_fo_bbox(bbox, 500, 500) ) for fo_bbox in detections: - assert isinstance(fo_bbox, Detection) + assert isinstance(fo_bbox, fo.Detection) def test_convert_record_to_fo_sample(object_detection_record): @@ -48,16 +50,16 @@ def test_convert_record_to_fo_sample(object_detection_record): test_sample = convert_record_to_fo_sample( object_detection_record, "test", None, False, None, lambda x: x ) - assert isinstance(test_sample, Sample) + assert isinstance(test_sample, fo.Sample) assert hasattr(test_sample, "test") assert len(test_sample["test"]) > 0 # Test add to existing sample and autosave - sample = Sample(object_detection_record.common.filepath) + sample = fo.Sample(object_detection_record.common.filepath) test_sample = convert_record_to_fo_sample( object_detection_record, "test", sample, False, None, lambda x: x ) - assert isinstance(test_sample, Sample) + assert isinstance(test_sample, fo.Sample) assert hasattr(test_sample, "test") assert len(test_sample["test"]) > 0 @@ -69,7 +71,7 @@ def test_convert_record_to_fo_sample(object_detection_record): object_detection_record, "test", None, False, test_tfms, None ) - assert isinstance(test_sample, Sample) + assert isinstance(test_sample, fo.Sample) assert hasattr(test_sample, "test") assert len(test_sample["test"]) > 0 @@ -89,7 +91,7 @@ def test_convert_prediction_to_fo_sample(object_detection_record): object_detection_record.original_img_size ) # Cleanup record, that has scope session - assert isinstance(test_sample, Sample) + assert isinstance(test_sample, fo.Sample) assert hasattr(test_sample, "ground_truth") assert len(test_sample["ground_truth"]) > 0 @@ -106,7 +108,7 @@ def test_create_fo_dataset(object_detection_record, cleanup_fo_dataset): ) # Test for record and with existing dataset - dataset = Dataset("_iv_test") + dataset = fo.Dataset("_iv_test") dataset = create_fo_dataset( [object_detection_prediction], dataset_name="_iv_test", exist_ok=True ) @@ -115,11 +117,11 @@ def test_create_fo_dataset(object_detection_record, cleanup_fo_dataset): object_detection_record.original_img_size ) # Cleanup record, that has scope session - assert isinstance(dataset, Dataset) + assert isinstance(dataset, fo.Dataset) assert len(list(dataset.iter_samples())) > 0 # Test for prediction and create new dataset dataset = create_fo_dataset([object_detection_record], "_iv_test_1") - assert isinstance(dataset, Dataset) + assert isinstance(dataset, fo.Dataset) assert len(list(dataset.iter_samples())) > 0 diff --git a/tests/data/test_record_collection.py b/tests/data/test_record_collection.py index 8025a04be..1751e76a4 100644 --- a/tests/data/test_record_collection.py +++ b/tests/data/test_record_collection.py @@ -5,11 +5,15 @@ @pytest.fixture def records(): def create_record_func(): - return BaseRecord([]) + return BaseRecord([FilepathRecordComponent()]) records = RecordCollection(create_record_func) - for record_id in ["file1", "file2", "file3", "file4"]: - records.get_by_record_id(record_id) + for record_id, filepath in zip( + ["file1", "file2", "file3", "file4"], + ["/train/0", "/train/1", "/valid/0", "/test/0"], + ): + record = records.get_by_record_id(record_id) + record.set_filepath(filepath) return records @@ -33,6 +37,26 @@ def test_fixed_splitter(records): assert splits == presplits +def test_folder_splitter(records): + presplits = [{"file1", "file2"}, {"file3"}, {"file4"}] + + keywords = ["train", "valid", "test"] + data_splitter = FolderSplitter(keywords=keywords) + splits = data_splitter(records) + + assert splits == presplits + + +def test_folder_splitter_with_remainder(records): + presplits = [{"file1", "file2"}, {"file4"}, {"file3"}] + + keywords = ["train", "test"] + data_splitter = FolderSplitter(keywords=keywords) + splits = data_splitter(records) + + assert splits == presplits + + def test_record_collection_adding(records): copy = deepcopy(records) for record_id in ["file5", "file6", "file7", "file8"]: diff --git a/tests/metrics/expected_register_p2t.pkl b/tests/metrics/expected_register_p2t.pkl new file mode 100644 index 0000000000000000000000000000000000000000..8c80b549b90412636af631bdf3037c5c37c182c9 GIT binary patch literal 607 zcmZvYze~eF6o8YMVnqckjFJ-GzDAF|;(~Hgm`+i&)UF`-F$giRw!Hl*nyj*^UOvbu+liNkjZhZS%g6!r)&oW(eZ zm8^^moFF9itZFu!usPz!KI<#XmFD?x68k7t?HfHQWtqUTK!DS-PAG z3DDj2Pm`036Fw;CVWt4Fk)g9^y@mYy>YjUiL&SOL|@sc;Y2u8cQZPkHZm z!DqeLN!MD9w{9AV$*c3ZvwHOWzsaKkv=^A}Y648YvgwL6Bk>S&7I|K9X1n6VV-DgZ JQr804e*lci*(Lx0 literal 0 HcmV?d00001 diff --git a/tests/metrics/expected_register_t2p.pkl b/tests/metrics/expected_register_t2p.pkl new file mode 100644 index 0000000000000000000000000000000000000000..e74109b39d069cd7879071ed81b715e3cc878f65 GIT binary patch literal 830 zcmZ{hv1{8v6o(~SCaw!1iJi7C1TvKbf|pF0OhdrK5n4)zLI-mvoqd-`R*>#mbqFM5 z;D$o4L;J^sjP2^JkRebgbZa1U>ARCsiP(7{@$@8p`hHI*-skPQV@vIDpT6TMk0-r| ziiG=V&yOb~X(q!6!z7y-45i>mo;zLH|#_^^`S>_o{^Rg7CK_D3@qp)Cz{n zo(}rIMz*Zt+AL;&%Plx>YNg4uD#e&w`Pa|a-(Ji&X#t_d+Gxr;XgZUmZY+Lo+p-pO z!k9Izpl`4bYjKnBqiRtzl&gFn_fbINu1FQpFo++OKQ>1MbpX+UESHew(qvIBg8jpB k63a`hhhAF7|5fL`&iIapGngD2b**uJ^%m! literal 0 HcmV?d00001 diff --git a/tests/metrics/test_confusion_matrix.py b/tests/metrics/test_confusion_matrix.py index c9bdc29e2..396934fe0 100644 --- a/tests/metrics/test_confusion_matrix.py +++ b/tests/metrics/test_confusion_matrix.py @@ -104,57 +104,17 @@ def test_pairwise_iou_matching(target, prediction): assert torch.allclose(result, expected_result, atol=1e-4) -def test_match_prediction(target, prediction): - result = match_records(target, prediction, iou_threshold=0.5) - expected_result = [ - [ - { - "target_bbox": BBox.from_xyxy(100, 100, 400, 400), - "target_label": "a", - "target_label_id": 1, - }, - [ - { - "predicted_bbox": BBox.from_xyxy(100, 100, 400, 400), - "predicted_label": "a", - "predicted_label_id": 1, - "score": 0.8, - "iou_score": 1.0, - }, - { - "predicted_bbox": BBox.from_xyxy(190, 100, 510, 400), - "predicted_label": "b", - "predicted_label_id": 2, - "score": 0.7, - "iou_score": 0.5122, - }, - ], - ], - [ - { - "target_bbox": BBox.from_xyxy(300, 100, 600, 400), - "target_label": "b", - "target_label_id": 2, - }, - [ - { - "predicted_bbox": BBox.from_xyxy(190, 100, 510, 400), - "predicted_label": "b", - "predicted_label_id": 2, - "score": 0.7, - "iou_score": 0.5122, - }, - ], - ], - [ - { - "target_bbox": BBox.from_xyxy(700, 200, 900, 500), - "target_label": "b", - "target_label_id": 2, - }, - [], - ], - ] +def test_match_prediction_to_target(target, prediction): + result = match_predictions_to_targets(target, prediction, iou_threshold=0.5) + with open(Path(__file__).parent / "expected_register_p2t.pkl", mode="rb") as infile: + expected_result = pickle.load(infile) + assert result == expected_result + + +def test_match_target_to_prediction(target, prediction): + result = match_targets_to_predictions(target, prediction, iou_threshold=0.5) + with open(Path(__file__).parent / "expected_register_t2p.pkl", mode="rb") as infile: + expected_result = pickle.load(infile) assert result == expected_result diff --git a/tests/models/efficient_det/conftest.py b/tests/models/efficient_det/conftest.py index cfad09097..321632726 100644 --- a/tests/models/efficient_det/conftest.py +++ b/tests/models/efficient_det/conftest.py @@ -1,5 +1,6 @@ import pytest + # TODO: Hacky approach @pytest.fixture def fridge_efficientdet_records(fridge_ds): diff --git a/tests/models/efficient_det/test_metrics.py b/tests/models/efficient_det/test_metrics.py index 8940ff91c..2241915c3 100644 --- a/tests/models/efficient_det/test_metrics.py +++ b/tests/models/efficient_det/test_metrics.py @@ -6,13 +6,13 @@ @pytest.fixture def expected_coco_metric_output(): return [ - " Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.494", - " Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.556", - " Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.556", + " Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.850", + " Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 1.000", + " Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 1.000", " Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000", " Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000", - " Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.507", - " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.450", + " Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.850", + " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.850", " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.850", " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.850", " Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000", @@ -25,18 +25,22 @@ def expected_coco_metric_output(): def expected_confusion_matrix_output(): return [ "[[0 0 0 0 0]", - " [0 0 0 0 0]", - " [0 0 1 0 0]", " [0 1 0 0 0]", + " [0 0 1 0 0]", + " [0 0 0 0 0]", " [0 0 0 0 0]]", ] @pytest.mark.parametrize( - "metric, expected_output", + "metric, expected_output, detection_threshold", [ - (SimpleConfusionMatrix(print_summary=True), "expected_confusion_matrix_output"), - (COCOMetric(print_summary=True), "expected_coco_metric_output"), + ( + SimpleConfusionMatrix(print_summary=True), + "expected_confusion_matrix_output", + 0.5, + ), + (COCOMetric(print_summary=True), "expected_coco_metric_output", 0.0), ], ) def test_efficientdet_metrics( @@ -44,6 +48,7 @@ def test_efficientdet_metrics( fridge_efficientdet_records, metric, expected_output, + detection_threshold, request, ): expected_output = request.getfixturevalue(expected_output) @@ -57,7 +62,7 @@ def test_efficientdet_metrics( batch=batch, raw_preds=raw_preds["detections"], records=fridge_efficientdet_records, - detection_threshold=0.0, + detection_threshold=detection_threshold, ) metric.accumulate(preds) diff --git a/tests/models/efficient_det/test_prediction.py b/tests/models/efficient_det/test_prediction.py index 2b6e0cce2..f4285eb80 100644 --- a/tests/models/efficient_det/test_prediction.py +++ b/tests/models/efficient_det/test_prediction.py @@ -39,12 +39,12 @@ def _test_preds(preds): pred = preds[0].pred assert len(pred.detection.scores) == 2 - np.testing.assert_equal(pred.detection.label_ids, [2, 1]) + np.testing.assert_equal(pred.detection.label_ids, [1, 2]) assert isinstance(pred.detection.bboxes[0], BBox) bboxes_np = np.array([bbox.xyxy for bbox in pred.detection.bboxes]) - bboxes_expected = np.array([[65, 59, 170, 261], [123, 212, 341, 292]]) - np.testing.assert_allclose(bboxes_np, bboxes_expected, atol=1) + bboxes_expected = np.array([[65, 60, 170, 257], [121, 215, 333, 278]]) + np.testing.assert_allclose(bboxes_np, bboxes_expected, atol=5) def test_efficient_det_predict(fridge_efficientdet_model, fridge_efficientdet_records): diff --git a/tests/models/fastai/unet/test_backbones.py b/tests/models/fastai/unet/test_backbones.py index a4406d4ab..fee36e2c4 100644 --- a/tests/models/fastai/unet/test_backbones.py +++ b/tests/models/fastai/unet/test_backbones.py @@ -3,7 +3,6 @@ from icevision.models.fastai import unet -@pytest.mark.skip @pytest.mark.parametrize( "model_name,param_groups_len", ( @@ -20,7 +19,6 @@ def test_unet_fpn_backbones_large(model_name, param_groups_len): assert len(list(model.param_groups())) == param_groups_len -@pytest.mark.skip @pytest.mark.parametrize( "model_name,param_groups_len", ( diff --git a/tests/models/mmseg/fastai/test_train.py b/tests/models/mmseg/fastai/test_train.py index 437b3508d..6a1023bc9 100644 --- a/tests/models/mmseg/fastai/test_train.py +++ b/tests/models/mmseg/fastai/test_train.py @@ -1,5 +1,6 @@ import pytest from icevision.all import * + from icevision.models.mmseg.models.segformer.backbones import * from icevision.models.mmseg.models import * diff --git a/tests/models/mmseg/test_show_results.py b/tests/models/mmseg/test_show_results.py index a1733fced..7aeebf040 100644 --- a/tests/models/mmseg/test_show_results.py +++ b/tests/models/mmseg/test_show_results.py @@ -1,5 +1,6 @@ import pytest from icevision.all import * + from icevision.models.mmseg.models.deeplabv3.backbones import * from icevision.models.mmseg.models import * from icevision.models.interpretation import get_samples_losses diff --git a/tests/models/torchvision_models/keypoints_rcnn/test_backbones.py b/tests/models/torchvision_models/keypoints_rcnn/test_backbones.py index b9ae45879..042382c4f 100644 --- a/tests/models/torchvision_models/keypoints_rcnn/test_backbones.py +++ b/tests/models/torchvision_models/keypoints_rcnn/test_backbones.py @@ -3,7 +3,6 @@ from icevision.models.torchvision import keypoint_rcnn -@pytest.mark.skip @pytest.mark.parametrize( "model_name,param_groups_len", ( @@ -21,7 +20,6 @@ def test_keypoint_rcnn_fpn_backbones_large(model_name, param_groups_len): assert len(model.param_groups()) == param_groups_len -@pytest.mark.skip @pytest.mark.parametrize( "model_name,param_groups_len", ( diff --git a/tests/models/torchvision_models/mask_rcnn/test_backbones.py b/tests/models/torchvision_models/mask_rcnn/test_backbones.py index 0aaa7777b..ef180b309 100644 --- a/tests/models/torchvision_models/mask_rcnn/test_backbones.py +++ b/tests/models/torchvision_models/mask_rcnn/test_backbones.py @@ -3,7 +3,6 @@ from icevision.models.torchvision import mask_rcnn -@pytest.mark.skip @pytest.mark.parametrize( "model_name,param_groups_len", ( @@ -21,7 +20,6 @@ def test_mask_rcnn_fpn_backbones_large(model_name, param_groups_len): assert len(model.param_groups()) == param_groups_len -@pytest.mark.skip @pytest.mark.parametrize( "model_name,param_groups_len", ( diff --git a/tests/models/torchvision_models/retinanet/test_backbones.py b/tests/models/torchvision_models/retinanet/test_backbones.py index b72ac5775..d2030f0e1 100644 --- a/tests/models/torchvision_models/retinanet/test_backbones.py +++ b/tests/models/torchvision_models/retinanet/test_backbones.py @@ -3,7 +3,6 @@ from icevision.models.torchvision import retinanet -@pytest.mark.skip @pytest.mark.parametrize( "model_name", ( @@ -21,7 +20,6 @@ def test_retinanet_fpn_backbones_large(model_name): assert len(model.param_groups()) == 7 -@pytest.mark.skip @pytest.mark.parametrize( "model_name", ( diff --git a/tests/transforms/batch_tfms/test_img_pad_stack.py b/tests/transforms/batch_tfms/test_img_pad_stack.py index 437906d8f..fa89647b5 100644 --- a/tests/transforms/batch_tfms/test_img_pad_stack.py +++ b/tests/transforms/batch_tfms/test_img_pad_stack.py @@ -4,7 +4,7 @@ @pytest.fixture() def records(): - imgs = [np.ones((2, 4, 3), dtype=np.float), np.ones((3, 2, 3), dtype=np.float)] + imgs = [np.ones((2, 4, 3), dtype=np.float32), np.ones((3, 2, 3), dtype=np.float32)] records = [] for img in imgs: @@ -20,7 +20,7 @@ def test_img_pad_stack(records, pad_value): tfmed_records = tfms.batch.ImgPadStack(pad_value=pad_value)(records) imgs = np.asarray([record.img for record in tfmed_records]) - expected = np.ones((2, 3, 4, 3), dtype=np.float) + expected = np.ones((2, 3, 4, 3), dtype=np.float32) expected *= np.array(pad_value).reshape(-1) expected[0, :2, :4, :3] = 1 expected[1, :3, :2, :3] = 1