TFDSv4 carla_video_tracking upgrade.

armory/datasets/adversarial/carla_video_tracking_dev/__init__.py

@@ -0,0 +1,3 @@
+"""carla_video_tracking_dev dataset."""
+
+from .carla_video_tracking_dev import CarlaVideoTrackingDev

armory/data/adversarial/carla_video_tracking_dev.py → armory/datasets/adversarial/carla_video_tracking_dev/carla_video_tracking_dev.py

@@ -4,7 +4,7 @@
 import glob
 import numpy as np
 from PIL import Image
-import tensorflow.compat.v1 as tf
+import tensorflow as tf
 import tensorflow_datasets as tfds
 
 _DESCRIPTION = """
@@ -21,7 +21,7 @@
 }
 """
 
-_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_video_tracking_dev_2.0.0.tar.gz"
+_URL = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_video_tracking_dev_2.0.0.tar.gz"
 
 
 class CarlaVideoTrackingDev(tfds.core.GeneratorBasedBuilder):
@@ -70,14 +70,9 @@ def _info(self) -> tfds.core.DatasetInfo:
 
     def _split_generators(self, dl_manager: tfds.download.DownloadManager):
         """Returns SplitGenerators."""
-        path = dl_manager.download_and_extract(_URLS)
-
-        return [
-            tfds.core.SplitGenerator(
-                name="dev",
-                gen_kwargs={"path": os.path.join(path, "dev")},
-            )
-        ]
+        path = dl_manager.download_and_extract(_URL)
+
+        return {"dev": self._generate_examples(path / "dev")}
 
     def _generate_examples(self, path):
         """Yields examples."""

armory/datasets/adversarial/carla_video_tracking_dev/checksums.tsv

@@ -0,0 +1 @@
+https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_video_tracking_dev_2.0.0.tar.gz	1278862237	8b23ca76bd9602a8e3ff4058335b7fb8ca665660a8a958852715e9a26ffbef20	carla_video_tracking_dev_2.0.0.tar.gz

armory/datasets/adversarial/carla_video_tracking_test/__init__.py

@@ -0,0 +1,3 @@
+"""carla_video_tracking_test dataset."""
+
+from .carla_video_tracking_test import CarlaVideoTrackingTest

armory/data/adversarial/carla_video_tracking_test.py → armory/datasets/adversarial/carla_video_tracking_test/carla_video_tracking_test.py

@@ -4,7 +4,7 @@
 import glob
 import numpy as np
 from PIL import Image
-import tensorflow.compat.v1 as tf
+import tensorflow as tf
 import tensorflow_datasets as tfds
 
 
@@ -22,7 +22,7 @@
 }
 """
 
-_URLS = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_video_tracking_test_2.0.0.tar.gz"
+_URL = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_video_tracking_test_2.0.0.tar.gz"
 
 
 class CarlaVideoTrackingTest(tfds.core.GeneratorBasedBuilder):
@@ -71,14 +71,8 @@ def _info(self) -> tfds.core.DatasetInfo:
 
     def _split_generators(self, dl_manager: tfds.download.DownloadManager):
         """Returns SplitGenerators."""
-        path = dl_manager.download_and_extract(_URLS)
-
-        return [
-            tfds.core.SplitGenerator(
-                name="test",
-                gen_kwargs={"path": os.path.join(path, "test")},
-            )
-        ]
+        path = dl_manager.download_and_extract(_URL)
+        return {"test": self._generate_examples(path / "test")}
 
     def _generate_examples(self, path):
         """Yields examples."""

armory/datasets/adversarial/carla_video_tracking_test/checksums.tsv

@@ -0,0 +1 @@
+https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_video_tracking_test_2.0.0.tar.gz	387465525	6bd09f5cf50c0e16f34b5054e9d77f95cb4491a373ecb842431cc58ae50b882e	carla_video_tracking_test_2.0.0.tar.gz

armory/datasets/art_wrapper.py

@@ -9,6 +9,7 @@ class WrappedDataGenerator(DataGenerator):
     def __init__(self, gen):
         super().__init__(gen.size, gen.batch_size)
         self._iterator = gen
+        self.context = gen.context
 
     def __iter__(self):
         return iter(self._iterator)

armory/datasets/cached_datasets.json

@@ -6,6 +6,20 @@
         "url": null,
         "version": "1.0.0"
     },
+    "carla_video_tracking_dev": {
+        "sha256": "958d470dcd394928050f4123a7af05b0e389ceeec6fa0a3261df55a65e553b69",
+        "size": 1281628036,
+        "subdir": "carla_video_tracking_dev/2.0.0",
+        "url": null,
+        "version": "2.0.0"
+    },
+    "carla_video_tracking_test": {
+        "sha256": "8c52281611807243cba425ad3a588f4abca40dfb2b3ab828b9ad8a5191a7df10",
+        "size": 388218968,
+        "subdir": "carla_video_tracking_test/2.0.0",
+        "url": null,
+        "version": "2.0.0"
+    },
     "digit": {
         "sha256": "805fb5e33caf2029e13f4146c9d06fdb437ac5b0f0aa9668e3201922b617c559",
         "size": 8349857,

armory/datasets/config_load.py

@@ -3,6 +3,7 @@
 """
 
 from armory.datasets import load, preprocessing, generator, filtering
+from armory.datasets.context import contexts
 
 
 def load_dataset(
@@ -20,6 +21,7 @@ def load_dataset(
     index=None,
     class_ids=None,
     drop_remainder=False,
+    context=None,
 ):
     # All are keyword elements by design
     if name is None:
@@ -64,6 +66,9 @@ def load_dataset(
 
     shuffle_elements = shuffle_files
 
+    if context is None and name in contexts:
+        context = contexts[name]
+
     return generator.ArmoryDataGenerator(
         info,
         ds_dict,
@@ -78,4 +83,5 @@ def load_dataset(
         element_map=preprocessing_fn,
         shuffle_elements=shuffle_elements,
         key_map=None,
+        context=context,
     )

armory/datasets/context.py

@@ -0,0 +1,16 @@
+import tensorflow as tf
+
+
+class VideoContext:
+    def __init__(self, x_shape, frame_rate):
+        self.x_shape = x_shape
+        self.input_type = tf.uint8
+        self.frame_rate = frame_rate
+
+
+carla_video_tracking_context = VideoContext(x_shape=(None, 960, 1280, 3), frame_rate=10)
+
+contexts = {
+    "carla_video_tracking_dev": carla_video_tracking_context,
+    "carla_video_tracking_test": carla_video_tracking_context,
+}

armory/datasets/generator.py

@@ -39,6 +39,7 @@ class ArmoryDataGenerator:
     element_filter - predicate of which elements to keep; occurs prior to mapping
         Note: size computations will be wrong when filtering is applied
     element_map - function that takes a dataset element (dict) and maps to new element
+    context - context object
     """
 
     FRAMEWORKS = ("tf", "numpy", "torch")
@@ -58,6 +59,7 @@ def __init__(
         element_filter: callable = None,
         element_map: callable = None,
         key_map=None,
+        context=None,
     ):
         if split not in info.splits:
             raise ValueError(f"split {split} not in info.splits {list(info.splits)}")
@@ -141,6 +143,7 @@ def __init__(
             shuffle_elements=shuffle_elements,
             element_filter=element_filter,
             element_map=element_map,
+            context=context,
         )
 
     def _set_params(self, **kwargs):

armory/datasets/preprocessing.py

@@ -57,6 +57,40 @@ def carla_over_obj_det_dev(element, modality="rgb"):
     )
 
 
+def carla_video_tracking_preprocess(x, y, y_patch_metadata, max_frames=None):
+    # Clip
+    if max_frames:
+        max_frames = int(max_frames)
+        if max_frames <= 0:
+            raise ValueError(f"max_frames {max_frames} must be > 0")
+        x = x[:max_frames, :]
+        y = y[:max_frames, :]
+        y_patch_metadata = {k: v[:max_frames, :] for (k, v) in y_patch_metadata.items()}
+    # Update labels
+    box_array = tf.squeeze(y, axis=0) if y.shape[0] == 1 else y
+    y = {"boxes": box_array}
+    y_patch_metadata = {
+        k: (tf.squeeze(v, axis=0) if v.shape[0] == 1 else v)
+        for k, v in y_patch_metadata.items()
+    }
+    return x, (y, y_patch_metadata)
+
+
+@register
+def carla_video_tracking_dev(element, max_frames=None):
+    return carla_video_tracking_preprocess(
+        element["video"],
+        element["bboxes"],
+        element["patch_metadata"],
+        max_frames=max_frames,
+    )
+
+
+@register
+def carla_video_tracking_test(element, max_frames=None):
+    return carla_video_tracking_dev(element, max_frames)
+
+
 @register
 def xview(element):
     return image_to_canon(element["image"]), convert_tf_obj_det_label_to_pytorch(

armory/metrics/task.py

@@ -522,7 +522,7 @@ def _check_video_tracking_input(y, y_pred):
     y_pred (List[Dict, ...]): same as above
     """
     for input in [y, y_pred]:
-        assert isinstance(input, list)
+        assert isinstance(input, list), f"Expected List[Dict, ...] got {type(input)}"
         for input_dict_i in input:
             assert isinstance(input_dict_i, dict)
             assert "boxes" in input_dict_i

armory/scenarios/carla_video_tracking.py

@@ -27,6 +27,9 @@ def load_test_dataset(self):
     def next(self):
         super().next()
         self.y, self.y_patch_metadata = self.y
+        # Fix for tfdsv4 upgrade - separate batches into list
+        if isinstance(self.y, dict) and "boxes" in self.y:
+            self.y = [{"boxes": batch} for batch in self.y["boxes"]]
         self.probe.update(y=self.y, y_patch_metadata=self.y_patch_metadata)
 
     def run_benign(self):

armory/utils/config_loading.py

@@ -26,7 +26,7 @@
 
 from armory.art_experimental.attacks import patch
 from armory.art_experimental.attacks.sweep import SweepAttack
-from armory.data.datasets import ArmoryDataGenerator, EvalGenerator
+from armory.datasets.generator import ArmoryDataGenerator
 from armory.data.utils import maybe_download_weights_from_s3
 from armory.utils import labels
 import copy

scenario_configs/eval5/carla_video_tracking/carla_video_tracking_goturn_advtextures_defended.json

@@ -14,11 +14,11 @@
         "use_label": true
     },
     "dataset": {
-        "batch_size": 1,
-        "eval_split": "dev",
-        "framework": "numpy",
-        "module": "armory.data.adversarial_datasets",
-        "name": "carla_video_tracking_dev"
+        "test": {
+            "batch_size": 1,
+            "split": "dev",
+            "name": "carla_video_tracking_dev"
+        }
     },
     "defense": {
         "kwargs": {

scenario_configs/eval5/carla_video_tracking/carla_video_tracking_goturn_advtextures_undefended.json

@@ -14,11 +14,11 @@
         "use_label": true
     },
     "dataset": {
-        "batch_size": 1,
-        "eval_split": "dev",
-        "framework": "numpy",
-        "module": "armory.data.adversarial_datasets",
-        "name": "carla_video_tracking_dev"
+        "test": {
+            "batch_size": 1,
+            "split": "dev",
+            "name": "carla_video_tracking_dev"
+        }
     },
     "defense": null,
     "metric": {