From e2b0adb881420be7fb175c12d174e07f498ed878 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sat, 24 Dec 2022 11:34:16 +0100 Subject: [PATCH 01/43] First draft --- metrics/panoptic_quality/panoptic_quality.py | 387 +++++++++++++++++++ 1 file changed, 387 insertions(+) create mode 100644 metrics/panoptic_quality/panoptic_quality.py diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py new file mode 100644 index 000000000..ab3d92712 --- /dev/null +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -0,0 +1,387 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Panoptic Quality (PQ) metric.""" + +from collections import defaultdict +import functools +import json +import multiprocessing +import os +import time +import traceback + +import numpy as np +from PIL import Image + +import datasets +import evaluate + + +_DESCRIPTION = """... +""" + +_KWARGS_DESCRIPTION = """ +Args: + predictions (`List[ndarray]`): + List of predicted segmentation maps, each of shape (height, width). Each segmentation map can be of a different size. + references (`List[ndarray]`): + List of ground truth segmentation maps, each of shape (height, width). Each segmentation map can be of a different size. + predicted_annotations (`List[ndarray]`): + List of predicted annotations (segments info). + reference_annotations (`List[ndarray]`): + List of reference annotations (segments info). + output_dir (`str`): + Path to the output directory. + categories (`dict`): + Dictionary mapping category IDs to something like {'name': 'wall', 'id': 0, 'isthing': 0, 'color': [120, 120, 120]}. + Example here: https://github.com/cocodataset/panopticapi/blob/master/panoptic_coco_categories.json. + +Returns: + `Dict[str, float | ndarray]` comprising various elements: + - *panoptic_quality* (`float`): + Panoptic quality score. + ... + +Examples: + + >>> import numpy as np + + >>> panoptic_quality = evaluate.load("panoptic_quality") + + >>> # TODO +""" + +_CITATION = """...""" + + +# The decorator is used to prints an error trhown inside process +def get_traceback(f): + @functools.wraps(f) + def wrapper(*args, **kwargs): + try: + return f(*args, **kwargs) + except Exception as e: + print("Caught exception in worker thread:") + traceback.print_exc() + raise e + + return wrapper + + +def rgb2id(color): + if isinstance(color, np.ndarray) and len(color.shape) == 3: + if color.dtype == np.uint8: + color = color.astype(np.int32) + return color[:, :, 0] + 256 * color[:, :, 1] + 256 * 256 * color[:, :, 2] + return int(color[0] + 256 * color[1] + 256 * 256 * color[2]) + + +OFFSET = 256 * 256 * 256 +VOID = 0 + + +class PQStatCat: + def __init__(self): + self.iou = 0.0 + self.tp = 0 + self.fp = 0 + self.fn = 0 + + def __iadd__(self, pq_stat_cat): + self.iou += pq_stat_cat.iou + self.tp += pq_stat_cat.tp + self.fp += pq_stat_cat.fp + self.fn += pq_stat_cat.fn + return self + + +class PQStat: + def __init__(self): + self.pq_per_cat = defaultdict(PQStatCat) + + def __getitem__(self, i): + return self.pq_per_cat[i] + + def __iadd__(self, pq_stat): + for label, pq_stat_cat in pq_stat.pq_per_cat.items(): + self.pq_per_cat[label] += pq_stat_cat + return self + + def pq_average(self, categories, isthing): + pq, sq, rq, n = 0, 0, 0, 0 + per_class_results = {} + for label, label_info in categories.items(): + if isthing is not None: + cat_isthing = label_info["isthing"] == 1 + if isthing != cat_isthing: + continue + iou = self.pq_per_cat[label].iou + tp = self.pq_per_cat[label].tp + fp = self.pq_per_cat[label].fp + fn = self.pq_per_cat[label].fn + if tp + fp + fn == 0: + per_class_results[label] = {"pq": 0.0, "sq": 0.0, "rq": 0.0} + continue + n += 1 + pq_class = iou / (tp + 0.5 * fp + 0.5 * fn) + sq_class = iou / tp if tp != 0 else 0 + rq_class = tp / (tp + 0.5 * fp + 0.5 * fn) + per_class_results[label] = {"pq": pq_class, "sq": sq_class, "rq": rq_class} + pq += pq_class + sq += sq_class + rq += rq_class + + return {"pq": pq / n, "sq": sq / n, "rq": rq / n, "n": n}, per_class_results + + +@get_traceback +def pq_compute_single_core(proc_id, annotation_set, gt_folder, pred_folder, categories): + pq_stat = PQStat() + + idx = 0 + for gt_ann, pred_ann in annotation_set: + if idx % 100 == 0: + print("Core: {}, {} from {} images processed".format(proc_id, idx, len(annotation_set))) + idx += 1 + + pan_gt = np.array(Image.open(os.path.join(gt_folder, gt_ann["file_name"])), dtype=np.uint32) + pan_gt = rgb2id(pan_gt) + pan_pred = np.array(Image.open(os.path.join(pred_folder, pred_ann["file_name"])), dtype=np.uint32) + pan_pred = rgb2id(pan_pred) + + gt_segms = {el["id"]: el for el in gt_ann["segments_info"]} + pred_segms = {el["id"]: el for el in pred_ann["segments_info"]} + + # predicted segments area calculation + prediction sanity checks + pred_labels_set = set(el["id"] for el in pred_ann["segments_info"]) + labels, labels_cnt = np.unique(pan_pred, return_counts=True) + for label, label_cnt in zip(labels, labels_cnt): + if label not in pred_segms: + if label == VOID: + continue + raise KeyError( + "In the image with ID {} segment with ID {} is presented in PNG and not presented in JSON.".format( + gt_ann["image_id"], label + ) + ) + pred_segms[label]["area"] = label_cnt + pred_labels_set.remove(label) + if pred_segms[label]["category_id"] not in categories: + raise KeyError( + "In the image with ID {} segment with ID {} has unknown category_id {}.".format( + gt_ann["image_id"], label, pred_segms[label]["category_id"] + ) + ) + if len(pred_labels_set) != 0: + raise KeyError( + "In the image with ID {} the following segment IDs {} are presented in JSON and not presented in PNG.".format( + gt_ann["image_id"], list(pred_labels_set) + ) + ) + + # confusion matrix calculation + pan_gt_pred = pan_gt.astype(np.uint64) * OFFSET + pan_pred.astype(np.uint64) + gt_pred_map = {} + labels, labels_cnt = np.unique(pan_gt_pred, return_counts=True) + for label, intersection in zip(labels, labels_cnt): + gt_id = label // OFFSET + pred_id = label % OFFSET + gt_pred_map[(gt_id, pred_id)] = intersection + + # count all matched pairs + gt_matched = set() + pred_matched = set() + for label_tuple, intersection in gt_pred_map.items(): + gt_label, pred_label = label_tuple + if gt_label not in gt_segms: + continue + if pred_label not in pred_segms: + continue + if gt_segms[gt_label]["iscrowd"] == 1: + continue + if gt_segms[gt_label]["category_id"] != pred_segms[pred_label]["category_id"]: + continue + + union = ( + pred_segms[pred_label]["area"] + + gt_segms[gt_label]["area"] + - intersection + - gt_pred_map.get((VOID, pred_label), 0) + ) + iou = intersection / union + if iou > 0.5: + pq_stat[gt_segms[gt_label]["category_id"]].tp += 1 + pq_stat[gt_segms[gt_label]["category_id"]].iou += iou + gt_matched.add(gt_label) + pred_matched.add(pred_label) + + # count false positives + crowd_labels_dict = {} + for gt_label, gt_info in gt_segms.items(): + if gt_label in gt_matched: + continue + # crowd segments are ignored + if gt_info["iscrowd"] == 1: + crowd_labels_dict[gt_info["category_id"]] = gt_label + continue + pq_stat[gt_info["category_id"]].fn += 1 + + # count false positives + for pred_label, pred_info in pred_segms.items(): + if pred_label in pred_matched: + continue + # intersection of the segment with VOID + intersection = gt_pred_map.get((VOID, pred_label), 0) + # plus intersection with corresponding CROWD region if it exists + if pred_info["category_id"] in crowd_labels_dict: + intersection += gt_pred_map.get((crowd_labels_dict[pred_info["category_id"]], pred_label), 0) + # predicted segment is ignored if more than half of the segment correspond to VOID and CROWD regions + if intersection / pred_info["area"] > 0.5: + continue + pq_stat[pred_info["category_id"]].fp += 1 + print("Core: {}, all {} images processed".format(proc_id, len(annotation_set))) + return pq_stat + + +def pq_compute_multi_core(matched_annotations_list, gt_folder, pred_folder, categories): + cpu_num = multiprocessing.cpu_count() + annotations_split = np.array_split(matched_annotations_list, cpu_num) + print("Number of cores: {}, images per core: {}".format(cpu_num, len(annotations_split[0]))) + workers = multiprocessing.Pool(processes=cpu_num) + processes = [] + for proc_id, annotation_set in enumerate(annotations_split): + p = workers.apply_async(pq_compute_single_core, (proc_id, annotation_set, gt_folder, pred_folder, categories)) + processes.append(p) + pq_stat = PQStat() + for p in processes: + pq_stat += p.get() + return pq_stat + + +def pq_compute(gt_json_file, pred_json_file, gt_folder=None, pred_folder=None): + with open(gt_json_file, "r") as f: + gt_json = json.load(f) + with open(pred_json_file, "r") as f: + pred_json = json.load(f) + + if gt_folder is None: + gt_folder = gt_json_file.replace(".json", "") + if pred_folder is None: + pred_folder = pred_json_file.replace(".json", "") + categories = {el["id"]: el for el in gt_json["categories"]} + + print("Evaluation panoptic segmentation metrics:") + print("Ground truth:") + print("\tSegmentation folder: {}".format(gt_folder)) + print("\tJSON file: {}".format(gt_json_file)) + print("Prediction:") + print("\tSegmentation folder: {}".format(pred_folder)) + print("\tJSON file: {}".format(pred_json_file)) + + if not os.path.isdir(gt_folder): + raise Exception("Folder {} with ground truth segmentations doesn't exist".format(gt_folder)) + if not os.path.isdir(pred_folder): + raise Exception("Folder {} with predicted segmentations doesn't exist".format(pred_folder)) + + pred_annotations = {el["image_id"]: el for el in pred_json["annotations"]} + matched_annotations_list = [] + for gt_ann in gt_json["annotations"]: + image_id = gt_ann["image_id"] + if image_id not in pred_annotations: + raise Exception("no prediction for the image with id: {}".format(image_id)) + matched_annotations_list.append((gt_ann, pred_annotations[image_id])) + + pq_stat = pq_compute_multi_core(matched_annotations_list, gt_folder, pred_folder, categories) + + metrics = [("All", None), ("Things", True), ("Stuff", False)] + results = {} + for name, isthing in metrics: + results[name], per_class_results = pq_stat.pq_average(categories, isthing=isthing) + if name == "All": + results["per_class"] = per_class_results + print("{:10s}| {:>5s} {:>5s} {:>5s} {:>5s}".format("", "PQ", "SQ", "RQ", "N")) + print("-" * (10 + 7 * 4)) + + for name, _isthing in metrics: + print( + "{:10s}| {:5.1f} {:5.1f} {:5.1f} {:5d}".format( + name, + 100 * results[name]["pq"], + 100 * results[name]["sq"], + 100 * results[name]["rq"], + results[name]["n"], + ) + ) + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class PanopticQuality(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + # 1st Seq - height dim, 2nd - width dim + { + "predictions": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))), + "references": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))), + } + ), + reference_urls=[ + "https://github.com/open-mmlab/mmsegmentation/blob/71c201b1813267d78764f306a297ca717827c4bf/mmseg/core/evaluation/metrics.py" + ], + ) + + def _compute( + self, + predictions, + references, + predicted_annotations, + reference_annotations, + output_dir, + categories, + ): + if not os.path.exists(output_dir): + os.mkdir(output_dir) + + # step 1: create ground truth JSON file + gt_json_data = {"annotations": reference_annotations, "categories": categories} + gt_json = os.path.join(output_dir, "gt.json") + with open(gt_json, "w") as f: + f.write(json.dumps(gt_json_data)) + + # step 2: create predictions JSON file + predictions_json_data = {"annotations": predicted_annotations} + predictions_json = os.path.join(output_dir, "predictions.json") + with open(predictions_json, "w") as f: + f.write(json.dumps(predictions_json_data)) + + # step 3: dump ground truth PNG files + gt_folder = os.path.join(output_dir, "gt") + if not os.path.exists(gt_folder): + os.mkdir(gt_folder) + for image_id, gt in zip(references.keys(), references.values()): + gt_image = Image.fromarray(gt) + gt_image.save(os.path.join(gt_folder, str(image_id) + ".png")) + + # step 4: dump predictions PNG files + for p in predictions: + with open(os.path.join(output_dir, p["file_name"]), "wb") as f: + f.write(p.pop("png_string")) + + # step 5: compute PQ + result = pq_compute(gt_json, predictions_json, gt_folder=gt_folder, pred_folder=output_dir) + + return result \ No newline at end of file From cc29a2196fce18926019b67f326d940f9efa9230 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sat, 24 Dec 2022 12:30:08 +0100 Subject: [PATCH 02/43] More improvements --- metrics/panoptic_quality/panoptic_quality.py | 97 +++++++++++++------- 1 file changed, 64 insertions(+), 33 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index ab3d92712..a7c39ea09 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -11,14 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Panoptic Quality (PQ) metric.""" +"""Panoptic Quality (PQ) metric. + +Entirely based on https://github.com/cocodataset/panopticapi/blob/master/panopticapi/evaluation.py. +""" from collections import defaultdict import functools import json import multiprocessing +import io import os -import time import traceback import numpy as np @@ -87,6 +90,22 @@ def rgb2id(color): return int(color[0] + 256 * color[1] + 256 * 256 * color[2]) +def id2rgb(id_map): + if isinstance(id_map, np.ndarray): + id_map_copy = id_map.copy() + rgb_shape = tuple(list(id_map.shape) + [3]) + rgb_map = np.zeros(rgb_shape, dtype=np.uint8) + for i in range(3): + rgb_map[..., i] = id_map_copy % 256 + id_map_copy //= 256 + return rgb_map + color = [] + for _ in range(3): + color.append(id_map % 256) + id_map //= 256 + return color + + OFFSET = 256 * 256 * 256 VOID = 0 @@ -335,8 +354,8 @@ def _info(self): features=datasets.Features( # 1st Seq - height dim, 2nd - width dim { - "predictions": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))), - "references": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))), + "predictions": datasets.Sequence(datasets.Image()), + "references": datasets.Sequence(datasets.Image()), } ), reference_urls=[ @@ -346,42 +365,54 @@ def _info(self): def _compute( self, - predictions, - references, - predicted_annotations, - reference_annotations, - output_dir, - categories, + predictions=None, + references=None, + predicted_annotations=None, + image_ids=None, + # references, + # reference_annotations, + output_dir=None, + gt_folder=None, + gt_json=None, ): if not os.path.exists(output_dir): os.mkdir(output_dir) - - # step 1: create ground truth JSON file - gt_json_data = {"annotations": reference_annotations, "categories": categories} - gt_json = os.path.join(output_dir, "gt.json") - with open(gt_json, "w") as f: - f.write(json.dumps(gt_json_data)) + + # step 1: dump predicted segmentations to folder + for seg_img, image_id in zip(predictions, image_ids): + with io.BytesIO() as out: + seg_img = id2rgb(seg_img) + f.write(seg_img.save(out, format="PNG")) + + file_name = f"{image_id:012d}.png" + with open(os.path.join(output_dir, file_name), "wb") as f: + f.write(out.getvalue()) # step 2: create predictions JSON file - predictions_json_data = {"annotations": predicted_annotations} + # predicted_annotations is a list of segments_info + json_data = {"annotations": predicted_annotations} predictions_json = os.path.join(output_dir, "predictions.json") with open(predictions_json, "w") as f: - f.write(json.dumps(predictions_json_data)) - - # step 3: dump ground truth PNG files - gt_folder = os.path.join(output_dir, "gt") - if not os.path.exists(gt_folder): - os.mkdir(gt_folder) - for image_id, gt in zip(references.keys(), references.values()): - gt_image = Image.fromarray(gt) - gt_image.save(os.path.join(gt_folder, str(image_id) + ".png")) - - # step 4: dump predictions PNG files - for p in predictions: - with open(os.path.join(output_dir, p["file_name"]), "wb") as f: - f.write(p.pop("png_string")) - + f.write(json.dumps(json_data)) + # step 5: compute PQ result = pq_compute(gt_json, predictions_json, gt_folder=gt_folder, pred_folder=output_dir) - return result \ No newline at end of file + return result + + +# # step 1: create ground truth JSON file +# gt_json_data = {"annotations": reference_annotations, "categories": categories} +# gt_json = os.path.join(output_dir, "gt.json") +# with open(gt_json, "w") as f: +# f.write(json.dumps(gt_json_data)) + +# # step 4 + # gt_folder = os.path.join(output_dir, "gt") + # if not os.path.exists(gt_folder): + # os.mkdir(gt_folder) + # for ref in reference_annotations: + # image_id = ref["image_id"] + # file_name = f"{image_id:012d}.png" + # with open(os.path.join(gt_folder, file_name), "wb") as f: + # f.write(ref["segmentation"]) \ No newline at end of file From 65136a8e6cd5b4c429306b6409da5642494305a4 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sat, 24 Dec 2022 12:36:33 +0100 Subject: [PATCH 03/43] Update features --- metrics/panoptic_quality/panoptic_quality.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index a7c39ea09..dcb2a5577 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -354,8 +354,8 @@ def _info(self): features=datasets.Features( # 1st Seq - height dim, 2nd - width dim { - "predictions": datasets.Sequence(datasets.Image()), - "references": datasets.Sequence(datasets.Image()), + "predictions": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))), + "references": datasets.Sequence(), } ), reference_urls=[ From 970653da172f5e0ac7c070882e6a91920c49c3e3 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sat, 24 Dec 2022 12:39:42 +0100 Subject: [PATCH 04/43] Remove feature --- metrics/panoptic_quality/panoptic_quality.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index dcb2a5577..6483728a7 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -355,7 +355,6 @@ def _info(self): # 1st Seq - height dim, 2nd - width dim { "predictions": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))), - "references": datasets.Sequence(), } ), reference_urls=[ From 67123ca045659cb128455dfbb54dc3e5a035dd8d Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sat, 24 Dec 2022 12:48:34 +0100 Subject: [PATCH 05/43] Add print statements --- metrics/panoptic_quality/panoptic_quality.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 6483728a7..57df51502 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -380,6 +380,8 @@ def _compute( # step 1: dump predicted segmentations to folder for seg_img, image_id in zip(predictions, image_ids): with io.BytesIO() as out: + print(type(seg_img)) + print(seg_img) seg_img = id2rgb(seg_img) f.write(seg_img.save(out, format="PNG")) From 9c97456257d98d91379efa404b189ed77ae2bc46 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sat, 24 Dec 2022 12:52:18 +0100 Subject: [PATCH 06/43] Use Image feature --- metrics/panoptic_quality/panoptic_quality.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 57df51502..e064b9c7a 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -354,7 +354,7 @@ def _info(self): features=datasets.Features( # 1st Seq - height dim, 2nd - width dim { - "predictions": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))), + "predictions": datasets.Sequence(datasets.Image), } ), reference_urls=[ @@ -380,9 +380,6 @@ def _compute( # step 1: dump predicted segmentations to folder for seg_img, image_id in zip(predictions, image_ids): with io.BytesIO() as out: - print(type(seg_img)) - print(seg_img) - seg_img = id2rgb(seg_img) f.write(seg_img.save(out, format="PNG")) file_name = f"{image_id:012d}.png" From 4d2fdc71c6d7086f44cdf7fb301af99347e12312 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sat, 24 Dec 2022 13:02:46 +0100 Subject: [PATCH 07/43] Update feature --- metrics/panoptic_quality/panoptic_quality.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index e064b9c7a..3a5ecb258 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -354,7 +354,7 @@ def _info(self): features=datasets.Features( # 1st Seq - height dim, 2nd - width dim { - "predictions": datasets.Sequence(datasets.Image), + "predictions": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))), } ), reference_urls=[ @@ -379,6 +379,8 @@ def _compute( # step 1: dump predicted segmentations to folder for seg_img, image_id in zip(predictions, image_ids): + seg_img = Image.fromarray(id2rgb(np.array(seg_img))) + with io.BytesIO() as out: f.write(seg_img.save(out, format="PNG")) From 3d43d5babec2630b6806315c1c12ab3abc5972bd Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sat, 24 Dec 2022 13:08:32 +0100 Subject: [PATCH 08/43] Fix code --- metrics/panoptic_quality/panoptic_quality.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 3a5ecb258..1b957961d 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -379,10 +379,12 @@ def _compute( # step 1: dump predicted segmentations to folder for seg_img, image_id in zip(predictions, image_ids): - seg_img = Image.fromarray(id2rgb(np.array(seg_img))) + seg_img = np.array(seg_img) + print("Shape of seg_img:", seg_img.shape) + seg_img = Image.fromarray(id2rgb(seg_img)) with io.BytesIO() as out: - f.write(seg_img.save(out, format="PNG")) + seg_img.save(out, format="PNG") file_name = f"{image_id:012d}.png" with open(os.path.join(output_dir, file_name), "wb") as f: From 3e6a0db2e6a35c6756d3f37d579dc12cc1e4db31 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sat, 24 Dec 2022 13:12:19 +0100 Subject: [PATCH 09/43] More fixes --- metrics/panoptic_quality/panoptic_quality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 1b957961d..04a406482 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -386,7 +386,7 @@ def _compute( with io.BytesIO() as out: seg_img.save(out, format="PNG") - file_name = f"{image_id:012d}.png" + file_name = f"{image_id}.png" with open(os.path.join(output_dir, file_name), "wb") as f: f.write(out.getvalue()) From 12fe7eed450719ddbd6e80691fd3645505967054 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sat, 24 Dec 2022 13:15:45 +0100 Subject: [PATCH 10/43] Fix more code --- metrics/panoptic_quality/panoptic_quality.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 04a406482..e88772ab4 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -385,10 +385,9 @@ def _compute( with io.BytesIO() as out: seg_img.save(out, format="PNG") - - file_name = f"{image_id}.png" - with open(os.path.join(output_dir, file_name), "wb") as f: - f.write(out.getvalue()) + file_name = f"{image_id}.png" + with open(os.path.join(output_dir, file_name), "wb") as f: + f.write(out.getvalue()) # step 2: create predictions JSON file # predicted_annotations is a list of segments_info From 439c2e3eb042e258349f2dbbe38a50b7373084bd Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sat, 24 Dec 2022 13:28:22 +0100 Subject: [PATCH 11/43] Add print statement --- metrics/panoptic_quality/panoptic_quality.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index e88772ab4..a4d6bfcc6 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -313,6 +313,8 @@ def pq_compute(gt_json_file, pred_json_file, gt_folder=None, pred_folder=None): if not os.path.isdir(pred_folder): raise Exception("Folder {} with predicted segmentations doesn't exist".format(pred_folder)) + for el in pred_json["annotations"]: + print(el) pred_annotations = {el["image_id"]: el for el in pred_json["annotations"]} matched_annotations_list = [] for gt_ann in gt_json["annotations"]: From abdfb3eff99d498e591a41d2ee0f621c5207ed3b Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sat, 24 Dec 2022 13:35:01 +0100 Subject: [PATCH 12/43] Add print statement --- metrics/panoptic_quality/panoptic_quality.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index a4d6bfcc6..2c770ffd8 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -380,17 +380,19 @@ def _compute( os.mkdir(output_dir) # step 1: dump predicted segmentations to folder - for seg_img, image_id in zip(predictions, image_ids): + for idx, (seg_img, image_id) in enumerate(zip(predictions, image_ids)): seg_img = np.array(seg_img) print("Shape of seg_img:", seg_img.shape) seg_img = Image.fromarray(id2rgb(seg_img)) with io.BytesIO() as out: seg_img.save(out, format="PNG") - file_name = f"{image_id}.png" + file_name = f"{image_id:012d}.png" with open(os.path.join(output_dir, file_name), "wb") as f: f.write(out.getvalue()) + predicted_annotations["image_id"] = image_id + # step 2: create predictions JSON file # predicted_annotations is a list of segments_info json_data = {"annotations": predicted_annotations} From d683c2ff43fc70b9d021e9eab49713ee434cf845 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sat, 24 Dec 2022 13:35:29 +0100 Subject: [PATCH 13/43] Add more fixes --- metrics/panoptic_quality/panoptic_quality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 2c770ffd8..feaa7ee16 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -391,7 +391,7 @@ def _compute( with open(os.path.join(output_dir, file_name), "wb") as f: f.write(out.getvalue()) - predicted_annotations["image_id"] = image_id + predicted_annotations[idx]["image_id"] = image_id # step 2: create predictions JSON file # predicted_annotations is a list of segments_info From c179f18f5d52d8efd73713cd60c1c95de284e382 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sat, 24 Dec 2022 14:22:57 +0100 Subject: [PATCH 14/43] Add image_id to all annotations --- metrics/panoptic_quality/panoptic_quality.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index feaa7ee16..910eddae6 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -382,7 +382,6 @@ def _compute( # step 1: dump predicted segmentations to folder for idx, (seg_img, image_id) in enumerate(zip(predictions, image_ids)): seg_img = np.array(seg_img) - print("Shape of seg_img:", seg_img.shape) seg_img = Image.fromarray(id2rgb(seg_img)) with io.BytesIO() as out: @@ -391,7 +390,9 @@ def _compute( with open(os.path.join(output_dir, file_name), "wb") as f: f.write(out.getvalue()) - predicted_annotations[idx]["image_id"] = image_id + for i in range(len(predicted_annotations[idx])): + predicted_annotations[idx][i]["image_id"] = image_id + predicted_annotations[idx][i]["file_name"] = file_name # step 2: create predictions JSON file # predicted_annotations is a list of segments_info From 16f4aaa061e154473b178d6310aec1b27441b352 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sat, 24 Dec 2022 14:38:47 +0100 Subject: [PATCH 15/43] Add image_id to all annotations --- metrics/panoptic_quality/panoptic_quality.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 910eddae6..e31199ec7 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -366,9 +366,9 @@ def _info(self): def _compute( self, - predictions=None, + predictions=None, # this corresponds to the png_string key of DetrPostProcess references=None, - predicted_annotations=None, + predicted_annotations=None, # list of dicts, each dict containing `segments_info`. Segments info is a list of dicts, each dict containing category_id, id, iscrowd, area, bbox image_ids=None, # references, # reference_annotations, @@ -390,9 +390,9 @@ def _compute( with open(os.path.join(output_dir, file_name), "wb") as f: f.write(out.getvalue()) - for i in range(len(predicted_annotations[idx])): - predicted_annotations[idx][i]["image_id"] = image_id - predicted_annotations[idx][i]["file_name"] = file_name + # add image_id and file_name keys to each of the predicted annotations + predicted_annotations[idx]["image_id"] = image_id + predicted_annotations[idx]["file_name"] = file_name # step 2: create predictions JSON file # predicted_annotations is a list of segments_info From de90b269604fd7119426702bb025f78abc8a626c Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 25 Dec 2022 10:32:20 +0100 Subject: [PATCH 16/43] First draft --- metrics/panoptic_quality/panoptic_quality.py | 130 ++++++++----------- 1 file changed, 53 insertions(+), 77 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index e31199ec7..e0fef4899 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -165,19 +165,20 @@ def pq_average(self, categories, isthing): @get_traceback -def pq_compute_single_core(proc_id, annotation_set, gt_folder, pred_folder, categories): +def pq_compute_single_core(proc_id, annotation_set, predictions, references, categories): pq_stat = PQStat() idx = 0 - for gt_ann, pred_ann in annotation_set: + for pan_pred, pan_gt, (pred_ann, gt_ann) in zip(predictions, references, annotation_set): if idx % 100 == 0: print("Core: {}, {} from {} images processed".format(proc_id, idx, len(annotation_set))) idx += 1 - pan_gt = np.array(Image.open(os.path.join(gt_folder, gt_ann["file_name"])), dtype=np.uint32) - pan_gt = rgb2id(pan_gt) - pan_pred = np.array(Image.open(os.path.join(pred_folder, pred_ann["file_name"])), dtype=np.uint32) - pan_pred = rgb2id(pan_pred) + # TODO perhaps work in the RGB space + # pan_gt = np.array(Image.open(os.path.join(gt_folder, gt_ann["file_name"])), dtype=np.uint32) + # pan_gt = rgb2id(pan_gt) + # pan_pred = np.array(Image.open(os.path.join(pred_folder, pred_ann["file_name"])), dtype=np.uint32) + # pan_pred = rgb2id(pan_pred) gt_segms = {el["id"]: el for el in gt_ann["segments_info"]} pred_segms = {el["id"]: el for el in pred_ann["segments_info"]} @@ -273,14 +274,14 @@ def pq_compute_single_core(proc_id, annotation_set, gt_folder, pred_folder, cate return pq_stat -def pq_compute_multi_core(matched_annotations_list, gt_folder, pred_folder, categories): +def pq_compute_multi_core(matched_annotations_list, predictions, references, categories): cpu_num = multiprocessing.cpu_count() annotations_split = np.array_split(matched_annotations_list, cpu_num) print("Number of cores: {}, images per core: {}".format(cpu_num, len(annotations_split[0]))) workers = multiprocessing.Pool(processes=cpu_num) processes = [] for proc_id, annotation_set in enumerate(annotations_split): - p = workers.apply_async(pq_compute_single_core, (proc_id, annotation_set, gt_folder, pred_folder, categories)) + p = workers.apply_async(pq_compute_single_core, (proc_id, annotation_set, predictions, references, categories)) processes.append(p) pq_stat = PQStat() for p in processes: @@ -288,42 +289,40 @@ def pq_compute_multi_core(matched_annotations_list, gt_folder, pred_folder, cate return pq_stat -def pq_compute(gt_json_file, pred_json_file, gt_folder=None, pred_folder=None): - with open(gt_json_file, "r") as f: - gt_json = json.load(f) - with open(pred_json_file, "r") as f: - pred_json = json.load(f) - - if gt_folder is None: - gt_folder = gt_json_file.replace(".json", "") - if pred_folder is None: - pred_folder = pred_json_file.replace(".json", "") - categories = {el["id"]: el for el in gt_json["categories"]} - - print("Evaluation panoptic segmentation metrics:") - print("Ground truth:") - print("\tSegmentation folder: {}".format(gt_folder)) - print("\tJSON file: {}".format(gt_json_file)) - print("Prediction:") - print("\tSegmentation folder: {}".format(pred_folder)) - print("\tJSON file: {}".format(pred_json_file)) - - if not os.path.isdir(gt_folder): - raise Exception("Folder {} with ground truth segmentations doesn't exist".format(gt_folder)) - if not os.path.isdir(pred_folder): - raise Exception("Folder {} with predicted segmentations doesn't exist".format(pred_folder)) - - for el in pred_json["annotations"]: - print(el) - pred_annotations = {el["image_id"]: el for el in pred_json["annotations"]} +def pq_compute(predictions, references, predicted_annotations, reference_annotations, categories): + # categories = {el["id"]: el for el in gt_json["categories"]} + # with open(gt_json_file, "r") as f: + # gt_json = json.load(f) + # with open(pred_json_file, "r") as f: + # pred_json = json.load(f) + + # if gt_folder is None: + # gt_folder = gt_json_file.replace(".json", "") + # if pred_folder is None: + # pred_folder = pred_json_file.replace(".json", "") + # categories = {el["id"]: el for el in gt_json["categories"]} + + # print("Evaluation panoptic segmentation metrics:") + # print("Ground truth:") + # print("\tSegmentation folder: {}".format(gt_folder)) + # print("\tJSON file: {}".format(gt_json_file)) + # print("Prediction:") + # print("\tSegmentation folder: {}".format(pred_folder)) + # print("\tJSON file: {}".format(pred_json_file)) + + # if not os.path.isdir(gt_folder): + # raise Exception("Folder {} with ground truth segmentations doesn't exist".format(gt_folder)) + # if not os.path.isdir(pred_folder): + # raise Exception("Folder {} with predicted segmentations doesn't exist".format(pred_folder)) + + # for el in pred_json["annotations"]: + # print(el) + # pred_annotations = {el["image_id"]: el for el in pred_json["annotations"]} matched_annotations_list = [] - for gt_ann in gt_json["annotations"]: - image_id = gt_ann["image_id"] - if image_id not in pred_annotations: - raise Exception("no prediction for the image with id: {}".format(image_id)) - matched_annotations_list.append((gt_ann, pred_annotations[image_id])) + for pred_ann, gt_ann in zip(predicted_annotations, reference_annotations): + matched_annotations_list.append((pred_ann, gt_ann)) - pq_stat = pq_compute_multi_core(matched_annotations_list, gt_folder, pred_folder, categories) + pq_stat = pq_compute_multi_core(matched_annotations_list, predictions, references, categories) metrics = [("All", None), ("Things", True), ("Stuff", False)] results = {} @@ -357,10 +356,13 @@ def _info(self): # 1st Seq - height dim, 2nd - width dim { "predictions": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))), + "references": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))), + "predicted_annotations": datasets.Sequence(dict), + "reference_annotations": datasets.Sequence(dict), } ), reference_urls=[ - "https://github.com/open-mmlab/mmsegmentation/blob/71c201b1813267d78764f306a297ca717827c4bf/mmseg/core/evaluation/metrics.py" + "https://github.com/cocodataset/panopticapi/blob/master/panopticapi/evaluation.py" ], ) @@ -369,40 +371,14 @@ def _compute( predictions=None, # this corresponds to the png_string key of DetrPostProcess references=None, predicted_annotations=None, # list of dicts, each dict containing `segments_info`. Segments info is a list of dicts, each dict containing category_id, id, iscrowd, area, bbox - image_ids=None, - # references, - # reference_annotations, - output_dir=None, - gt_folder=None, - gt_json=None, - ): - if not os.path.exists(output_dir): - os.mkdir(output_dir) - - # step 1: dump predicted segmentations to folder - for idx, (seg_img, image_id) in enumerate(zip(predictions, image_ids)): - seg_img = np.array(seg_img) - seg_img = Image.fromarray(id2rgb(seg_img)) - - with io.BytesIO() as out: - seg_img.save(out, format="PNG") - file_name = f"{image_id:012d}.png" - with open(os.path.join(output_dir, file_name), "wb") as f: - f.write(out.getvalue()) - - # add image_id and file_name keys to each of the predicted annotations - predicted_annotations[idx]["image_id"] = image_id - predicted_annotations[idx]["file_name"] = file_name - - # step 2: create predictions JSON file - # predicted_annotations is a list of segments_info - json_data = {"annotations": predicted_annotations} - predictions_json = os.path.join(output_dir, "predictions.json") - with open(predictions_json, "w") as f: - f.write(json.dumps(json_data)) - - # step 5: compute PQ - result = pq_compute(gt_json, predictions_json, gt_folder=gt_folder, pred_folder=output_dir) + reference_annotations=None, + categories=None, + # image_ids=None, + # output_dir=None, + # gt_folder=None, + # gt_json=None, + ): + result = pq_compute(predictions, references, predicted_annotations, reference_annotations, categories) return result From fac3bb5e9de4846ef08536a2a096127cb7582e90 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 25 Dec 2022 10:52:40 +0100 Subject: [PATCH 17/43] Update features --- metrics/panoptic_quality/panoptic_quality.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index e0fef4899..b2c0039ed 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -355,8 +355,8 @@ def _info(self): features=datasets.Features( # 1st Seq - height dim, 2nd - width dim { - "predictions": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))), - "references": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))), + "predictions": datasets.Image(), + "references": datasets.Image(), "predicted_annotations": datasets.Sequence(dict), "reference_annotations": datasets.Sequence(dict), } From c5173f0cf3d4d9f94c2add74a4f824af476d46a9 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 25 Dec 2022 10:58:58 +0100 Subject: [PATCH 18/43] Update features --- metrics/panoptic_quality/panoptic_quality.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index b2c0039ed..fdc81b4f1 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -355,8 +355,8 @@ def _info(self): features=datasets.Features( # 1st Seq - height dim, 2nd - width dim { - "predictions": datasets.Image(), - "references": datasets.Image(), + "predictions": datasets.Image(decode=False), + "references": datasets.Image(decode=False), "predicted_annotations": datasets.Sequence(dict), "reference_annotations": datasets.Sequence(dict), } From 059b6526d0b9acaed8f648335633d0983853482f Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 25 Dec 2022 11:02:00 +0100 Subject: [PATCH 19/43] Update features --- metrics/panoptic_quality/panoptic_quality.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index fdc81b4f1..e16bd6bb7 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -355,8 +355,8 @@ def _info(self): features=datasets.Features( # 1st Seq - height dim, 2nd - width dim { - "predictions": datasets.Image(decode=False), - "references": datasets.Image(decode=False), + "predictions": datasets.Sequence(datasets.Image()), + "references": datasets.Sequence(datasets.Image()), "predicted_annotations": datasets.Sequence(dict), "reference_annotations": datasets.Sequence(dict), } From c77e32685a43e2c3603ffc0a9e7ae19181d0dac7 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 25 Dec 2022 11:08:12 +0100 Subject: [PATCH 20/43] Debug --- metrics/panoptic_quality/panoptic_quality.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index e16bd6bb7..abcf4b065 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -356,9 +356,9 @@ def _info(self): # 1st Seq - height dim, 2nd - width dim { "predictions": datasets.Sequence(datasets.Image()), - "references": datasets.Sequence(datasets.Image()), - "predicted_annotations": datasets.Sequence(dict), - "reference_annotations": datasets.Sequence(dict), + # "references": datasets.Sequence(datasets.Image()), + # "predicted_annotations": datasets.Sequence(dict), + # "reference_annotations": datasets.Sequence(dict), } ), reference_urls=[ From e53249fbda8057fc4d480febd6c8f964650a5e89 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 25 Dec 2022 21:00:45 +0100 Subject: [PATCH 21/43] Add feature --- metrics/panoptic_quality/panoptic_quality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index abcf4b065..3db61c9b6 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -356,7 +356,7 @@ def _info(self): # 1st Seq - height dim, 2nd - width dim { "predictions": datasets.Sequence(datasets.Image()), - # "references": datasets.Sequence(datasets.Image()), + "references": datasets.Sequence(datasets.Image()), # "predicted_annotations": datasets.Sequence(dict), # "reference_annotations": datasets.Sequence(dict), } From ac5827ff05545889f74155493484f1bf7abf7e9c Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 15 Jan 2023 12:58:11 +0100 Subject: [PATCH 22/43] Update features --- metrics/panoptic_quality/panoptic_quality.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 3db61c9b6..21274e9a1 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -355,8 +355,8 @@ def _info(self): features=datasets.Features( # 1st Seq - height dim, 2nd - width dim { - "predictions": datasets.Sequence(datasets.Image()), - "references": datasets.Sequence(datasets.Image()), + "predictions": datasets.Image(), + "references": datasets.Image(), # "predicted_annotations": datasets.Sequence(dict), # "reference_annotations": datasets.Sequence(dict), } @@ -368,10 +368,10 @@ def _info(self): def _compute( self, - predictions=None, # this corresponds to the png_string key of DetrPostProcess - references=None, - predicted_annotations=None, # list of dicts, each dict containing `segments_info`. Segments info is a list of dicts, each dict containing category_id, id, iscrowd, area, bbox - reference_annotations=None, + predictions, # this corresponds to the png_string key of DetrPostProcess + references, + predicted_annotations, # list of dicts, each dict containing `segments_info`. Segments info is a list of dicts, each dict containing category_id, id, iscrowd, area, bbox + reference_annotations, categories=None, # image_ids=None, # output_dir=None, From d15b9aed9c1277633d5c63c939efd94169d1bd26 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 15 Jan 2023 14:34:00 +0100 Subject: [PATCH 23/43] Improve implementation --- metrics/panoptic_quality/panoptic_quality.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 21274e9a1..343f62d76 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -174,11 +174,11 @@ def pq_compute_single_core(proc_id, annotation_set, predictions, references, cat print("Core: {}, {} from {} images processed".format(proc_id, idx, len(annotation_set))) idx += 1 - # TODO perhaps work in the RGB space + # we go from RGB space to id space here # pan_gt = np.array(Image.open(os.path.join(gt_folder, gt_ann["file_name"])), dtype=np.uint32) - # pan_gt = rgb2id(pan_gt) + pan_gt = rgb2id(np.array(pan_gt)) # pan_pred = np.array(Image.open(os.path.join(pred_folder, pred_ann["file_name"])), dtype=np.uint32) - # pan_pred = rgb2id(pan_pred) + pan_pred = rgb2id(np.array(pan_pred)) gt_segms = {el["id"]: el for el in gt_ann["segments_info"]} pred_segms = {el["id"]: el for el in pred_ann["segments_info"]} @@ -357,8 +357,8 @@ def _info(self): { "predictions": datasets.Image(), "references": datasets.Image(), - # "predicted_annotations": datasets.Sequence(dict), - # "reference_annotations": datasets.Sequence(dict), + "predicted_annotations": dict("segments_info"), + "reference_annotations": dict("segments_info"), } ), reference_urls=[ From 30443653d8a0d7955bf7d7fc59fff0cc6f478abf Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 15 Jan 2023 14:48:25 +0100 Subject: [PATCH 24/43] Disable features --- metrics/panoptic_quality/panoptic_quality.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 343f62d76..c0b93366a 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -357,8 +357,8 @@ def _info(self): { "predictions": datasets.Image(), "references": datasets.Image(), - "predicted_annotations": dict("segments_info"), - "reference_annotations": dict("segments_info"), + # "predicted_annotations": dict("segments_info"), + # "reference_annotations": dict("segments_info"), } ), reference_urls=[ From 49d82052c11c46a8f3a4dbaadb8e742ca4e7480a Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 15 Jan 2023 16:28:55 +0100 Subject: [PATCH 25/43] Add features --- metrics/panoptic_quality/panoptic_quality.py | 47 ++++++++++++-------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index c0b93366a..a112a7e53 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -353,33 +353,44 @@ def _info(self): citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features( - # 1st Seq - height dim, 2nd - width dim { "predictions": datasets.Image(), "references": datasets.Image(), - # "predicted_annotations": dict("segments_info"), - # "reference_annotations": dict("segments_info"), + "predicted_annotations": { + "segments_info": datasets.Sequence( + { + "id": datasets.Value("uint16"), + "category_id": datasets.Value("uint16"), + } + ) + }, + "reference_annotations": { + "segments_info": datasets.Sequence( + { + "id": datasets.Value("uint16"), + "category_id": datasets.Value("uint16"), + } + ) + }, } ), - reference_urls=[ - "https://github.com/cocodataset/panopticapi/blob/master/panopticapi/evaluation.py" - ], + reference_urls=["https://github.com/cocodataset/panopticapi/blob/master/panopticapi/evaluation.py"], ) def _compute( self, - predictions, # this corresponds to the png_string key of DetrPostProcess + predictions, # this corresponds to the png_string key of DetrPostProcess references, - predicted_annotations, # list of dicts, each dict containing `segments_info`. Segments info is a list of dicts, each dict containing category_id, id, iscrowd, area, bbox + predicted_annotations, # list of dicts, each dict containing `segments_info`. Segments info is a list of dicts, each dict containing category_id, id, iscrowd, area, bbox reference_annotations, categories=None, # image_ids=None, # output_dir=None, # gt_folder=None, # gt_json=None, - ): + ): result = pq_compute(predictions, references, predicted_annotations, reference_annotations, categories) - + return result @@ -390,11 +401,11 @@ def _compute( # f.write(json.dumps(gt_json_data)) # # step 4 - # gt_folder = os.path.join(output_dir, "gt") - # if not os.path.exists(gt_folder): - # os.mkdir(gt_folder) - # for ref in reference_annotations: - # image_id = ref["image_id"] - # file_name = f"{image_id:012d}.png" - # with open(os.path.join(gt_folder, file_name), "wb") as f: - # f.write(ref["segmentation"]) \ No newline at end of file +# gt_folder = os.path.join(output_dir, "gt") +# if not os.path.exists(gt_folder): +# os.mkdir(gt_folder) +# for ref in reference_annotations: +# image_id = ref["image_id"] +# file_name = f"{image_id:012d}.png" +# with open(os.path.join(gt_folder, file_name), "wb") as f: +# f.write(ref["segmentation"]) From 0546a841a672d8753c081fd8cd1ceb9a5702b3fd Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 15 Jan 2023 16:36:01 +0100 Subject: [PATCH 26/43] Improve features --- metrics/panoptic_quality/panoptic_quality.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index a112a7e53..c412556d5 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -361,6 +361,9 @@ def _info(self): { "id": datasets.Value("uint16"), "category_id": datasets.Value("uint16"), + "iscrowd": datasets.Value("uint16"), + "area": datasets.Value("uint16"), + "bbox": datasets.Sequence(datasets.Value("uint16")), } ) }, @@ -369,6 +372,8 @@ def _info(self): { "id": datasets.Value("uint16"), "category_id": datasets.Value("uint16"), + "was_fused": datasets.Value("bool"), + "score": datasets.Value("float32"), } ) }, From b7a7d2b9d077166ac731eedb951799061aebefee Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 15 Jan 2023 16:40:44 +0100 Subject: [PATCH 27/43] Improve features --- metrics/panoptic_quality/panoptic_quality.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index c412556d5..33582762d 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -361,9 +361,8 @@ def _info(self): { "id": datasets.Value("uint16"), "category_id": datasets.Value("uint16"), - "iscrowd": datasets.Value("uint16"), - "area": datasets.Value("uint16"), - "bbox": datasets.Sequence(datasets.Value("uint16")), + "was_fused": datasets.Value("bool"), + "score": datasets.Value("float32"), } ) }, @@ -372,8 +371,9 @@ def _info(self): { "id": datasets.Value("uint16"), "category_id": datasets.Value("uint16"), - "was_fused": datasets.Value("bool"), - "score": datasets.Value("float32"), + "iscrowd": datasets.Value("uint16"), + "area": datasets.Value("uint16"), + "bbox": datasets.Sequence(datasets.Value("uint16")), } ) }, From 1511fb2339ba86734a47f4c5d13b0063ca3f7a32 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 15 Jan 2023 16:50:48 +0100 Subject: [PATCH 28/43] Debug features --- metrics/panoptic_quality/panoptic_quality.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 33582762d..c313ea8fd 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -356,16 +356,16 @@ def _info(self): { "predictions": datasets.Image(), "references": datasets.Image(), - "predicted_annotations": { - "segments_info": datasets.Sequence( - { - "id": datasets.Value("uint16"), - "category_id": datasets.Value("uint16"), - "was_fused": datasets.Value("bool"), - "score": datasets.Value("float32"), - } - ) - }, + # "predicted_annotations": { + # "segments_info": datasets.Sequence( + # { + # "id": datasets.Value("uint16"), + # "category_id": datasets.Value("uint16"), + # "was_fused": datasets.Value("bool"), + # "score": datasets.Value("float32"), + # } + # ) + # }, "reference_annotations": { "segments_info": datasets.Sequence( { From aeccea0afd8b6dd4c9972090a86e78fc2a83311f Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 15 Jan 2023 16:55:50 +0100 Subject: [PATCH 29/43] Debug features --- metrics/panoptic_quality/panoptic_quality.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index c313ea8fd..f26262c6e 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -359,8 +359,8 @@ def _info(self): # "predicted_annotations": { # "segments_info": datasets.Sequence( # { - # "id": datasets.Value("uint16"), - # "category_id": datasets.Value("uint16"), + # "id": datasets.Value("int32"), + # "category_id": datasets.Value("int32"), # "was_fused": datasets.Value("bool"), # "score": datasets.Value("float32"), # } @@ -369,11 +369,11 @@ def _info(self): "reference_annotations": { "segments_info": datasets.Sequence( { - "id": datasets.Value("uint16"), - "category_id": datasets.Value("uint16"), - "iscrowd": datasets.Value("uint16"), - "area": datasets.Value("uint16"), - "bbox": datasets.Sequence(datasets.Value("uint16")), + "id": datasets.Value("int32"), + "category_id": datasets.Value("int32"), + "iscrowd": datasets.Value("int32"), + "area": datasets.Value("int32"), + "bbox": datasets.Sequence(datasets.Value("int32")), } ) }, From 0ccc70484828904979b476a6f7553fc87673cfcf Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 15 Jan 2023 16:58:32 +0100 Subject: [PATCH 30/43] Debug features --- metrics/panoptic_quality/panoptic_quality.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index f26262c6e..95e978744 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -356,16 +356,16 @@ def _info(self): { "predictions": datasets.Image(), "references": datasets.Image(), - # "predicted_annotations": { - # "segments_info": datasets.Sequence( - # { - # "id": datasets.Value("int32"), - # "category_id": datasets.Value("int32"), - # "was_fused": datasets.Value("bool"), - # "score": datasets.Value("float32"), - # } - # ) - # }, + "predicted_annotations": { + "segments_info": datasets.Sequence( + { + "id": datasets.Value("int32"), + "category_id": datasets.Value("int32"), + "was_fused": datasets.Value("bool"), + "score": datasets.Value("float32"), + } + ) + }, "reference_annotations": { "segments_info": datasets.Sequence( { From d591d98f4a51d28da8208968da7e7ca9119f37de Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 15 Jan 2023 17:03:57 +0100 Subject: [PATCH 31/43] Debug features --- metrics/panoptic_quality/panoptic_quality.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 95e978744..2a41cbf27 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -180,6 +180,9 @@ def pq_compute_single_core(proc_id, annotation_set, predictions, references, cat # pan_pred = np.array(Image.open(os.path.join(pred_folder, pred_ann["file_name"])), dtype=np.uint32) pan_pred = rgb2id(np.array(pan_pred)) + print("Ground truth annotation: ", gt_ann) + print("Predicted annotation: ", pred_ann) + gt_segms = {el["id"]: el for el in gt_ann["segments_info"]} pred_segms = {el["id"]: el for el in pred_ann["segments_info"]} From 9fd134eb45bb2f8df66b4f38298efd013e7fece6 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 15 Jan 2023 17:47:41 +0100 Subject: [PATCH 32/43] Remove segments_info --- metrics/panoptic_quality/panoptic_quality.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 2a41cbf27..a3639abb4 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -183,11 +183,11 @@ def pq_compute_single_core(proc_id, annotation_set, predictions, references, cat print("Ground truth annotation: ", gt_ann) print("Predicted annotation: ", pred_ann) - gt_segms = {el["id"]: el for el in gt_ann["segments_info"]} - pred_segms = {el["id"]: el for el in pred_ann["segments_info"]} + gt_segms = {el["id"]: el for el in gt_ann} + pred_segms = {el["id"]: el for el in pred_ann} # predicted segments area calculation + prediction sanity checks - pred_labels_set = set(el["id"] for el in pred_ann["segments_info"]) + pred_labels_set = set(el["id"] for el in pred_ann) labels, labels_cnt = np.unique(pan_pred, return_counts=True) for label, label_cnt in zip(labels, labels_cnt): if label not in pred_segms: @@ -359,18 +359,15 @@ def _info(self): { "predictions": datasets.Image(), "references": datasets.Image(), - "predicted_annotations": { - "segments_info": datasets.Sequence( + "predicted_annotations": datasets.Sequence( { "id": datasets.Value("int32"), "category_id": datasets.Value("int32"), "was_fused": datasets.Value("bool"), "score": datasets.Value("float32"), } - ) - }, - "reference_annotations": { - "segments_info": datasets.Sequence( + ), + "reference_annotations": datasets.Sequence( { "id": datasets.Value("int32"), "category_id": datasets.Value("int32"), @@ -378,8 +375,7 @@ def _info(self): "area": datasets.Value("int32"), "bbox": datasets.Sequence(datasets.Value("int32")), } - ) - }, + ) } ), reference_urls=["https://github.com/cocodataset/panopticapi/blob/master/panopticapi/evaluation.py"], From 0d0b47c35b4a7539f0cb40abba39da4ca017c1bf Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 15 Jan 2023 18:28:50 +0100 Subject: [PATCH 33/43] Debug --- metrics/panoptic_quality/panoptic_quality.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index a3639abb4..12814a71d 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -166,6 +166,8 @@ def pq_average(self, categories, isthing): @get_traceback def pq_compute_single_core(proc_id, annotation_set, predictions, references, categories): + print("Annotation set:", annotation_set) + pq_stat = PQStat() idx = 0 From e9ca3607b3b20260fd2916c7dc027f274f8b20b7 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 15 Jan 2023 19:58:49 +0100 Subject: [PATCH 34/43] Debug --- metrics/panoptic_quality/panoptic_quality.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 12814a71d..fd3f6fd7c 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -185,8 +185,11 @@ def pq_compute_single_core(proc_id, annotation_set, predictions, references, cat print("Ground truth annotation: ", gt_ann) print("Predicted annotation: ", pred_ann) - gt_segms = {el["id"]: el for el in gt_ann} - pred_segms = {el["id"]: el for el in pred_ann} + # gt_segms = {el["id"]: el for el in gt_ann} + # pred_segms = {el["id"]: el for el in pred_ann} + + gt_segms = {id: {k:v[idx] for k,v in gt_ann.items()} for idx, id in enumerate(gt_ann['id'])} + pred_segms = {id: {k:v[idx] for k,v in pred_ann.items()} for idx, id in enumerate(pred_ann['id'])} # predicted segments area calculation + prediction sanity checks pred_labels_set = set(el["id"] for el in pred_ann) From 678c8eacd58022fa72533feb51d10b98ff43962c Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sun, 15 Jan 2023 20:04:20 +0100 Subject: [PATCH 35/43] Debug --- metrics/panoptic_quality/panoptic_quality.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index fd3f6fd7c..193cf220a 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -192,7 +192,8 @@ def pq_compute_single_core(proc_id, annotation_set, predictions, references, cat pred_segms = {id: {k:v[idx] for k,v in pred_ann.items()} for idx, id in enumerate(pred_ann['id'])} # predicted segments area calculation + prediction sanity checks - pred_labels_set = set(el["id"] for el in pred_ann) + # pred_labels_set = set(el["id"] for el in pred_ann) + pred_labels_set = set(pred_ann["id"]) labels, labels_cnt = np.unique(pan_pred, return_counts=True) for label, label_cnt in zip(labels, labels_cnt): if label not in pred_segms: From 2f0ef026c0dedeedb802ffe8fa1e0a0ffac12192 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Mon, 23 Jan 2023 12:11:58 +0100 Subject: [PATCH 36/43] Debug --- metrics/panoptic_quality/panoptic_quality.py | 33 ++++++++++++++------ 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 193cf220a..c9af52cfe 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -199,23 +199,38 @@ def pq_compute_single_core(proc_id, annotation_set, predictions, references, cat if label not in pred_segms: if label == VOID: continue + # raise KeyError( + # "In the image with ID {} segment with ID {} is presented in PNG and not presented in JSON.".format( + # gt_ann["image_id"], label + # ) + # ) raise KeyError( - "In the image with ID {} segment with ID {} is presented in PNG and not presented in JSON.".format( - gt_ann["image_id"], label + "The segment with ID {} is presented in PNG and not presented in JSON.".format( + label ) ) pred_segms[label]["area"] = label_cnt pred_labels_set.remove(label) if pred_segms[label]["category_id"] not in categories: + # raise KeyError( + # "In the image with ID {} segment with ID {} has unknown category_id {}.".format( + # gt_ann["image_id"], label, pred_segms[label]["category_id"] + # ) + # ) raise KeyError( - "In the image with ID {} segment with ID {} has unknown category_id {}.".format( - gt_ann["image_id"], label, pred_segms[label]["category_id"] + "The segment with ID {} has unknown category_id {}.".format( + label, pred_segms[label]["category_id"] ) ) if len(pred_labels_set) != 0: + # raise KeyError( + # "In the image with ID {} the following segment IDs {} are presented in JSON and not presented in PNG.".format( + # gt_ann["image_id"], list(pred_labels_set) + # ) + # ) raise KeyError( - "In the image with ID {} the following segment IDs {} are presented in JSON and not presented in PNG.".format( - gt_ann["image_id"], list(pred_labels_set) + "The following segment IDs {} are presented in JSON and not presented in PNG.".format( + list(pred_labels_set) ) ) @@ -369,8 +384,8 @@ def _info(self): { "id": datasets.Value("int32"), "category_id": datasets.Value("int32"), - "was_fused": datasets.Value("bool"), - "score": datasets.Value("float32"), + # "was_fused": datasets.Value("bool"), + # "score": datasets.Value("float32"), } ), "reference_annotations": datasets.Sequence( @@ -379,7 +394,7 @@ def _info(self): "category_id": datasets.Value("int32"), "iscrowd": datasets.Value("int32"), "area": datasets.Value("int32"), - "bbox": datasets.Sequence(datasets.Value("int32")), + # "bbox": datasets.Sequence(datasets.Value("int32")), } ) } From f485dc7accbf2b0e2c8bb6e34a39342fa3227b0c Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Mon, 23 Jan 2023 12:20:13 +0100 Subject: [PATCH 37/43] Add features back --- metrics/panoptic_quality/panoptic_quality.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index c9af52cfe..370d259a1 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -384,8 +384,8 @@ def _info(self): { "id": datasets.Value("int32"), "category_id": datasets.Value("int32"), - # "was_fused": datasets.Value("bool"), - # "score": datasets.Value("float32"), + "was_fused": datasets.Value("bool"), + "score": datasets.Value("float32"), } ), "reference_annotations": datasets.Sequence( @@ -394,7 +394,7 @@ def _info(self): "category_id": datasets.Value("int32"), "iscrowd": datasets.Value("int32"), "area": datasets.Value("int32"), - # "bbox": datasets.Sequence(datasets.Value("int32")), + "bbox": datasets.Sequence(datasets.Value("int32")), } ) } From 095ebfcbcd544303178eeefa7149734b75580640 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Mon, 23 Jan 2023 13:49:07 +0100 Subject: [PATCH 38/43] Add print statement --- metrics/panoptic_quality/panoptic_quality.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 370d259a1..4fb6d3a42 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -191,6 +191,9 @@ def pq_compute_single_core(proc_id, annotation_set, predictions, references, cat gt_segms = {id: {k:v[idx] for k,v in gt_ann.items()} for idx, id in enumerate(gt_ann['id'])} pred_segms = {id: {k:v[idx] for k,v in pred_ann.items()} for idx, id in enumerate(pred_ann['id'])} + print("Ground truth segments:", gt_segms) + print("Predicted segments:", pred_segms) + # predicted segments area calculation + prediction sanity checks # pred_labels_set = set(el["id"] for el in pred_ann) pred_labels_set = set(pred_ann["id"]) From 297ebbd9f7b3615a20c784874729428c087158f4 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Mon, 23 Jan 2023 14:41:53 +0100 Subject: [PATCH 39/43] Debug --- metrics/panoptic_quality/panoptic_quality.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 4fb6d3a42..84208fe6d 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -198,7 +198,13 @@ def pq_compute_single_core(proc_id, annotation_set, predictions, references, cat # pred_labels_set = set(el["id"] for el in pred_ann) pred_labels_set = set(pred_ann["id"]) labels, labels_cnt = np.unique(pan_pred, return_counts=True) + + print("Predicted labels set:", pred_labels_set) + print("Labels:", labels) + print("Labels count:", labels_cnt) + for label, label_cnt in zip(labels, labels_cnt): + print("Label:", label) if label not in pred_segms: if label == VOID: continue From ff776c2c70b591b039defcf164e32981acfe2912 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Mon, 23 Jan 2023 15:01:40 +0100 Subject: [PATCH 40/43] Debug --- metrics/panoptic_quality/panoptic_quality.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 84208fe6d..80d7c162b 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -203,9 +203,12 @@ def pq_compute_single_core(proc_id, annotation_set, predictions, references, cat print("Labels:", labels) print("Labels count:", labels_cnt) + print("Predicted segments:", pred_segms.keys()) + for label, label_cnt in zip(labels, labels_cnt): print("Label:", label) if label not in pred_segms: + print(f"Label {label} not in predicted segments {pred_segms.keys()}") if label == VOID: continue # raise KeyError( From a76ca5d15de375c107059e49dba9c9e39f4bfb65 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Mon, 23 Jan 2023 15:14:51 +0100 Subject: [PATCH 41/43] Debug --- metrics/panoptic_quality/panoptic_quality.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 80d7c162b..7c991ca58 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -312,6 +312,8 @@ def pq_compute_single_core(proc_id, annotation_set, predictions, references, cat def pq_compute_multi_core(matched_annotations_list, predictions, references, categories): cpu_num = multiprocessing.cpu_count() + # fix cpu numbers for now (DEBUGGING) + cpu_num = 1 annotations_split = np.array_split(matched_annotations_list, cpu_num) print("Number of cores: {}, images per core: {}".format(cpu_num, len(annotations_split[0]))) workers = multiprocessing.Pool(processes=cpu_num) From cb1464e39bbbfe65065922e9e1a23ce0b576e804 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Mon, 30 Jan 2023 15:30:05 +0100 Subject: [PATCH 42/43] Clean up code --- metrics/panoptic_quality/panoptic_quality.py | 54 ++------------------ 1 file changed, 3 insertions(+), 51 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 7c991ca58..8403117f1 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -312,6 +312,7 @@ def pq_compute_single_core(proc_id, annotation_set, predictions, references, cat def pq_compute_multi_core(matched_annotations_list, predictions, references, categories): cpu_num = multiprocessing.cpu_count() + # TODO support multiprocessing # fix cpu numbers for now (DEBUGGING) cpu_num = 1 annotations_split = np.array_split(matched_annotations_list, cpu_num) @@ -328,34 +329,6 @@ def pq_compute_multi_core(matched_annotations_list, predictions, references, cat def pq_compute(predictions, references, predicted_annotations, reference_annotations, categories): - # categories = {el["id"]: el for el in gt_json["categories"]} - # with open(gt_json_file, "r") as f: - # gt_json = json.load(f) - # with open(pred_json_file, "r") as f: - # pred_json = json.load(f) - - # if gt_folder is None: - # gt_folder = gt_json_file.replace(".json", "") - # if pred_folder is None: - # pred_folder = pred_json_file.replace(".json", "") - # categories = {el["id"]: el for el in gt_json["categories"]} - - # print("Evaluation panoptic segmentation metrics:") - # print("Ground truth:") - # print("\tSegmentation folder: {}".format(gt_folder)) - # print("\tJSON file: {}".format(gt_json_file)) - # print("Prediction:") - # print("\tSegmentation folder: {}".format(pred_folder)) - # print("\tJSON file: {}".format(pred_json_file)) - - # if not os.path.isdir(gt_folder): - # raise Exception("Folder {} with ground truth segmentations doesn't exist".format(gt_folder)) - # if not os.path.isdir(pred_folder): - # raise Exception("Folder {} with predicted segmentations doesn't exist".format(pred_folder)) - - # for el in pred_json["annotations"]: - # print(el) - # pred_annotations = {el["image_id"]: el for el in pred_json["annotations"]} matched_annotations_list = [] for pred_ann, gt_ann in zip(predicted_annotations, reference_annotations): matched_annotations_list.append((pred_ann, gt_ann)) @@ -418,33 +391,12 @@ def _info(self): def _compute( self, - predictions, # this corresponds to the png_string key of DetrPostProcess + predictions, references, - predicted_annotations, # list of dicts, each dict containing `segments_info`. Segments info is a list of dicts, each dict containing category_id, id, iscrowd, area, bbox + predicted_annotations, reference_annotations, categories=None, - # image_ids=None, - # output_dir=None, - # gt_folder=None, - # gt_json=None, ): result = pq_compute(predictions, references, predicted_annotations, reference_annotations, categories) return result - - -# # step 1: create ground truth JSON file -# gt_json_data = {"annotations": reference_annotations, "categories": categories} -# gt_json = os.path.join(output_dir, "gt.json") -# with open(gt_json, "w") as f: -# f.write(json.dumps(gt_json_data)) - -# # step 4 -# gt_folder = os.path.join(output_dir, "gt") -# if not os.path.exists(gt_folder): -# os.mkdir(gt_folder) -# for ref in reference_annotations: -# image_id = ref["image_id"] -# file_name = f"{image_id:012d}.png" -# with open(os.path.join(gt_folder, file_name), "wb") as f: -# f.write(ref["segmentation"]) From 8c6e29be06d729a873bb11e0af992507f8bdf325 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Mon, 30 Jan 2023 15:50:11 +0100 Subject: [PATCH 43/43] Fix style --- metrics/panoptic_quality/panoptic_quality.py | 40 +++++++++----------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/metrics/panoptic_quality/panoptic_quality.py b/metrics/panoptic_quality/panoptic_quality.py index 8403117f1..2455a8418 100644 --- a/metrics/panoptic_quality/panoptic_quality.py +++ b/metrics/panoptic_quality/panoptic_quality.py @@ -167,7 +167,7 @@ def pq_average(self, categories, isthing): @get_traceback def pq_compute_single_core(proc_id, annotation_set, predictions, references, categories): print("Annotation set:", annotation_set) - + pq_stat = PQStat() idx = 0 @@ -188,8 +188,8 @@ def pq_compute_single_core(proc_id, annotation_set, predictions, references, cat # gt_segms = {el["id"]: el for el in gt_ann} # pred_segms = {el["id"]: el for el in pred_ann} - gt_segms = {id: {k:v[idx] for k,v in gt_ann.items()} for idx, id in enumerate(gt_ann['id'])} - pred_segms = {id: {k:v[idx] for k,v in pred_ann.items()} for idx, id in enumerate(pred_ann['id'])} + gt_segms = {id: {k: v[idx] for k, v in gt_ann.items()} for idx, id in enumerate(gt_ann["id"])} + pred_segms = {id: {k: v[idx] for k, v in pred_ann.items()} for idx, id in enumerate(pred_ann["id"])} print("Ground truth segments:", gt_segms) print("Predicted segments:", pred_segms) @@ -216,11 +216,7 @@ def pq_compute_single_core(proc_id, annotation_set, predictions, references, cat # gt_ann["image_id"], label # ) # ) - raise KeyError( - "The segment with ID {} is presented in PNG and not presented in JSON.".format( - label - ) - ) + raise KeyError("The segment with ID {} is presented in PNG and not presented in JSON.".format(label)) pred_segms[label]["area"] = label_cnt pred_labels_set.remove(label) if pred_segms[label]["category_id"] not in categories: @@ -368,22 +364,22 @@ def _info(self): "predictions": datasets.Image(), "references": datasets.Image(), "predicted_annotations": datasets.Sequence( - { - "id": datasets.Value("int32"), - "category_id": datasets.Value("int32"), - "was_fused": datasets.Value("bool"), - "score": datasets.Value("float32"), - } + { + "id": datasets.Value("int32"), + "category_id": datasets.Value("int32"), + "was_fused": datasets.Value("bool"), + "score": datasets.Value("float32"), + } ), "reference_annotations": datasets.Sequence( - { - "id": datasets.Value("int32"), - "category_id": datasets.Value("int32"), - "iscrowd": datasets.Value("int32"), - "area": datasets.Value("int32"), - "bbox": datasets.Sequence(datasets.Value("int32")), - } - ) + { + "id": datasets.Value("int32"), + "category_id": datasets.Value("int32"), + "iscrowd": datasets.Value("int32"), + "area": datasets.Value("int32"), + "bbox": datasets.Sequence(datasets.Value("int32")), + } + ), } ), reference_urls=["https://github.com/cocodataset/panopticapi/blob/master/panopticapi/evaluation.py"],