refactor(centerpoint, pointpainting): rearrange parameters for ML models and packages

launch/tier4_perception_launch/launch/object_recognition/detection/detector/camera_lidar_detector.launch.xml

@@ -82,8 +82,9 @@
       <arg name="output/objects" value="objects"/>
       <arg name="model_name" value="$(var lidar_detection_model)"/>
       <arg name="model_path" value="$(var pointpainting_model_path)"/>
-      <arg name="model_param_path" value="$(var lidar_model_param_path)/$(var lidar_detection_model).param.yaml"/>
-      <arg name="class_remapper_param_path" value="$(var lidar_model_param_path)/detection_class_remapper.param.yaml"/>
+      <arg name="model_param_path" value="$(var lidar_model_param_path)/pointpainting.param.yaml"/>
+      <arg name="ml_package_param_path" value="$(var model_path)/pointpainting_ml_package.param.yaml"/>
+      <arg name="class_remapper_param_path" value="$(var model_path)/detection_class_remapper.param.yaml"/>
 
       <arg name="use_pointcloud_container" value="true"/>
       <arg name="pointcloud_container_name" value="$(var pointcloud_container_name)"/>

launch/tier4_perception_launch/launch/object_recognition/detection/detector/lidar_dnn_detector.launch.xml

@@ -18,8 +18,9 @@
         <arg name="output/objects" value="objects"/>
         <arg name="model_name" value="$(var centerpoint_model_name)"/>
         <arg name="model_path" value="$(var centerpoint_model_path)"/>
-        <arg name="model_param_path" value="$(var lidar_model_param_path)/$(var centerpoint_model_name).param.yaml"/>
-        <arg name="class_remapper_param_path" value="$(var lidar_model_param_path)/detection_class_remapper.param.yaml"/>
+        <arg name="model_param_path" value="$(var lidar_model_param_path)/centerpoint.param.yaml"/>
+        <arg name="ml_package_param_path" value="$(var centerpoint_model_path)/$(var centerpoint_model_name)_ml_package.param.yaml"/>
+        <arg name="class_remapper_param_path" value="$(var centerpoint_model_path)/detection_class_remapper.param.yaml"/>
 
         <arg name="use_pointcloud_container" value="true"/>
         <arg name="pointcloud_container_name" value="$(var pointcloud_container_name)"/>

perception/image_projection_based_fusion/config/pointpainting.param.yaml

@@ -1,33 +1,21 @@
 /**:
   ros__parameters:
-    trt_precision: fp16
     encoder_onnx_path: "$(var model_path)/pts_voxel_encoder_$(var model_name).onnx"
     encoder_engine_path: "$(var model_path)/pts_voxel_encoder_$(var model_name).engine"
     head_onnx_path: "$(var model_path)/pts_backbone_neck_head_$(var model_name).onnx"
     head_engine_path: "$(var model_path)/pts_backbone_neck_head_$(var model_name).engine"
-
-    model_params:
-      class_names: ["CAR", "TRUCK", "BUS", "BICYCLE", "PEDESTRIAN"]
-      paint_class_names: ["CAR", "BICYCLE", "PEDESTRIAN"]
-      point_feature_size: 7 # x, y, z, time-lag and car, pedestrian, bicycle
-      max_voxel_size: 40000
-      point_cloud_range: [-121.6, -76.8, -3.0, 121.6, 76.8, 5.0]
-      voxel_size: [0.32, 0.32, 8.0]
-      downsample_factor: 1
-      encoder_in_feature_size: 12
-      yaw_norm_thresholds: [0.3, 0.3, 0.3, 0.3, 0.0]
-      has_variance: false
-      has_twist: false
-    densification_params:
-      world_frame_id: "map"
-      num_past_frames: 0
+    trt_precision: fp16
     post_process_params:
       # post-process params
       circle_nms_dist_threshold: 0.3
       iou_nms_target_class_names: ["CAR"]
       iou_nms_search_distance_2d: 10.0
       iou_nms_threshold: 0.1
       score_threshold: 0.35
+      yaw_norm_thresholds: [0.3, 0.3, 0.3, 0.3, 0.0]
+    densification_params:
+      world_frame_id: "map"
+      num_past_frames: 1
     omp_params:
       # omp params
       num_threads: 1

perception/image_projection_based_fusion/config/pointpainting_ml_package.param.yaml

@@ -0,0 +1,13 @@
+/**:
+  ros__parameters:
+    model_params:
+      class_names: ["CAR", "TRUCK", "BUS", "BICYCLE", "PEDESTRIAN"]
+      paint_class_names: ["CAR", "BICYCLE", "PEDESTRIAN"]
+      point_feature_size: 7 # x, y, z, time-lag and car, pedestrian, bicycle
+      max_voxel_size: 40000
+      point_cloud_range: [-121.6, -76.8, -3.0, 121.6, 76.8, 5.0]
+      voxel_size: [0.32, 0.32, 8.0]
+      downsample_factor: 1
+      encoder_in_feature_size: 12
+      has_twist: false
+      has_variance: false

perception/image_projection_based_fusion/docs/pointpainting-fusion.md

@@ -33,17 +33,22 @@ The lidar points are projected onto the output of an image-only 2d object detect
 
 ### Core Parameters
 
-| Name                            | Type   | Default Value | Description                                                 |
-| ------------------------------- | ------ | ------------- | ----------------------------------------------------------- |
-| `score_threshold`               | float  | `0.4`         | detected objects with score less than threshold are ignored |
-| `densification_world_frame_id`  | string | `map`         | the world frame id to fuse multi-frame pointcloud           |
-| `densification_num_past_frames` | int    | `0`           | the number of past frames to fuse with the current frame    |
-| `trt_precision`                 | string | `fp16`        | TensorRT inference precision: `fp32` or `fp16`              |
-| `encoder_onnx_path`             | string | `""`          | path to VoxelFeatureEncoder ONNX file                       |
-| `encoder_engine_path`           | string | `""`          | path to VoxelFeatureEncoder TensorRT Engine file            |
-| `head_onnx_path`                | string | `""`          | path to DetectionHead ONNX file                             |
-| `head_engine_path`              | string | `""`          | path to DetectionHead TensorRT Engine file                  |
-| `build_only`                    | bool   | `false`       | shutdown the node after TensorRT engine file is built       |
+| Name                                             | Type         | Default Value             | Description                                                 |
+| ------------------------------------------------ | ------------ | ------------------------- | ----------------------------------------------------------- |
+| `encoder_onnx_path`                              | string       | `""`                      | path to VoxelFeatureEncoder ONNX file                       |
+| `encoder_engine_path`                            | string       | `""`                      | path to VoxelFeatureEncoder TensorRT Engine file            |
+| `head_onnx_path`                                 | string       | `""`                      | path to DetectionHead ONNX file                             |
+| `head_engine_path`                               | string       | `""`                      | path to DetectionHead TensorRT Engine file                  |
+| `build_only`                                     | bool         | `false`                   | shutdown the node after TensorRT engine file is built       |
+| `trt_precision`                                  | string       | `fp16`                    | TensorRT inference precision: `fp32` or `fp16`              |
+| `post_process_params.score_threshold`            | double       | `0.4`                     | detected objects with score less than threshold are ignored |
+| `post_process_params.yaw_norm_thresholds`        | list[double] | [0.3, 0.3, 0.3, 0.3, 0.0] | An array of distance threshold values of norm of yaw [rad]. |
+| `post_process_params.iou_nms_target_class_names` | list[string] | ["CAR"]                   | An array of class names to be target in NMS.                |
+| `post_process_params.iou_nms_search_distance_2d` | double       | 10.0                      | A maximum distance value to search the nearest objects.     |
+| `post_process_params.iou_nms_threshold`          | double       | 0.1                       | A threshold value of NMS using IoU score.                   |
+| `post_process_params.has_twist`                  | boolean      | false                     | Indicates whether the model outputs twist value.            |
+| `densification_params.world_frame_id`            | string       | `map`                     | the world frame id to fuse multi-frame pointcloud           |
+| `densification_params.num_past_frames`           | int          | `1`                       | the number of past frames to fuse with the current frame    |
 
 ## Assumptions / Known limits
 

perception/image_projection_based_fusion/launch/pointpainting_fusion.launch.xml

@@ -21,8 +21,9 @@
   <arg name="data_path" default="$(env HOME)/autoware_data" description="packages data and artifacts directory path"/>
   <arg name="model_name" default="pointpainting" description="options: `pointpainting`"/>
   <arg name="model_path" default="$(var data_path)/image_projection_based_fusion"/>
-  <arg name="model_param_path" default="$(find-pkg-share image_projection_based_fusion)/config/$(var model_name).param.yaml"/>
-  <arg name="class_remapper_param_path" default="$(find-pkg-share lidar_centerpoint)/config/detection_class_remapper.param.yaml"/>
+  <arg name="model_param_path" default="$(find-pkg-share image_projection_based_fusion)/config/pointpainting.param.yaml"/>
+  <arg name="ml_package_param_path" default="$(var model_path)/$(var model_name)_ml_package.param.yaml"/>
+  <arg name="class_remapper_param_path" default="$(var model_path)/detection_class_remapper.param.yaml"/>
   <arg name="sync_param_path" default="$(find-pkg-share image_projection_based_fusion)/config/roi_sync.param.yaml"/>
   <arg name="build_only" default="false" description="shutdown node after TensorRT engine file is built"/>
 
@@ -48,6 +49,7 @@
         <remap from="~/input/pointcloud" to="$(var input/pointcloud)"/>
         <remap from="~/output/objects" to="$(var output/objects)"/>
         <param from="$(var model_param_path)" allow_substs="true"/>
+        <param from="$(var ml_package_param_path)" allow_substs="true"/>
         <param from="$(var sync_param_path)"/>
         <param from="$(var class_remapper_param_path)"/>
         <param name="rois_number" value="$(var input/rois_number)"/>
@@ -89,6 +91,7 @@
       <remap from="~/input/pointcloud" to="$(var input/pointcloud)"/>
       <remap from="~/output/objects" to="$(var output/objects)"/>
       <param from="$(var model_param_path)" allow_substs="true"/>
+      <param from="$(var ml_package_param_path)" allow_substs="true"/>
       <param from="$(var sync_param_path)"/>
       <param from="$(var class_remapper_param_path)"/>
       <param name="rois_number" value="$(var input/rois_number)"/>

perception/image_projection_based_fusion/schema/pointpainting.schema.json

@@ -6,71 +6,11 @@
     "pointpainting": {
       "type": "object",
       "properties": {
-        "model_params": {
-          "type": "object",
-          "description": "Parameters for model configuration.",
-          "properties": {
-            "class_names": {
-              "type": "array",
-              "description": "An array of class names will be predicted.",
-              "default": ["CAR", "TRUCK", "BUS", "BICYCLE", "PEDESTRIAN"],
-              "uniqueItems": true
-            },
-            "paint_class_names": {
-              "type": "array",
-              "description": "An array of class names will be painted by PointPainting",
-              "default": ["CAR", "BICYCLE", "PEDESTRIAN"],
-              "uniqueItems": true
-            },
-            "point_feature_size": {
-              "type": "integer",
-              "description": "A number of channels of point feature layer.",
-              "default": 7
-            },
-            "max_voxel_size": {
-              "type": "integer",
-              "description": "A maximum size of voxel grid.",
-              "default": 40000
-            },
-            "point_cloud_range": {
-              "type": "array",
-              "description": "An array of distance ranges of each class, this must have same length with `class_names`.",
-              "default": [-121.6, -76.8, -3.0, 121.6, 76.8, 5.0]
-            },
-            "voxel_size": {
-              "type": "array",
-              "description": "An array of voxel grid sizes for PointPainting, this must have same length with `paint_class_names`.",
-              "default": [0.32, 0.32, 8.0]
-            },
-            "down_sample_factor": {
-              "type": "integer",
-              "description": "A scale factor of downsampling points",
-              "default": 1,
-              "minimum": 1
-            },
-            "encoder_in_feature_size": {
-              "type": "integer",
-              "description": "A size of encoder input feature channels.",
-              "default": 12
-            },
-            "yaw_norm_thresholds": {
-              "type": "array",
-              "description": "An array of distance threshold values of norm of yaw [rad].",
-              "default": [0.3, 0.3, 0.3, 0.3, 0.0],
-              "minimum": 0.0,
-              "maximum": 1.0
-            },
-            "has_variance": {
-              "type": "boolean",
-              "description": "Indicates whether the model outputs variance value.",
-              "default": false
-            },
-            "has_twist": {
-              "type": "boolean",
-              "description": "Indicates whether the model outputs twist value.",
-              "default": false
-            }
-          }
+        "trt_precision": {
+          "type": "string",
+          "description": "TensorRT inference precision.",
+          "default": "fp16",
+          "enum": ["fp32", "fp16"]
         },
         "densification_params": {
           "type": "object",
@@ -112,7 +52,7 @@
               "default": ["CAR"],
               "uniqueItems": true
             },
-            "iou_search_distance_2d": {
+            "iou_nms_search_distance_2d": {
               "type": "number",
               "description": "A maximum distance value to search the nearest objects.",
               "default": 10.0,
@@ -124,6 +64,23 @@
               "default": 0.1,
               "minimum": 0.0,
               "maximum": 1.0
+            },
+            "yaw_norm_thresholds": {
+              "type": "array",
+              "description": "An array of distance threshold values of norm of yaw [rad].",
+              "default": [0.3, 0.3, 0.3, 0.3, 0.0],
+              "minimum": 0.0,
+              "maximum": 1.0
+            },
+            "has_variance": {
+              "type": "boolean",
+              "description": "Indicates whether the model outputs variance value.",
+              "default": false
+            },
+            "has_twist": {
+              "type": "boolean",
+              "description": "Indicates whether the model outputs twist value.",
+              "default": false
             }
           }
         },
@@ -139,7 +96,7 @@
           }
         }
       },
-      "required": ["model_params", "densification_params", "post_process_params", "omp_params"]
+      "required": ["densification_params", "post_process_params", "omp_params"]
     }
   },
   "properties": {

perception/image_projection_based_fusion/schema/pointpainting_ml_package.schema.json

@@ -0,0 +1,85 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Parameters for Point Painting ML model",
+  "type": "object",
+  "definitions": {
+    "pointpainting_ml_package": {
+      "type": "object",
+      "properties": {
+        "model_params": {
+          "type": "object",
+          "description": "Parameters for model configuration.",
+          "properties": {
+            "class_names": {
+              "type": "array",
+              "description": "An array of class names will be predicted.",
+              "default": ["CAR", "TRUCK", "BUS", "BICYCLE", "PEDESTRIAN"],
+              "uniqueItems": true
+            },
+            "paint_class_names": {
+              "type": "array",
+              "description": "An array of class names will be painted by PointPainting",
+              "default": ["CAR", "BICYCLE", "PEDESTRIAN"],
+              "uniqueItems": true
+            },
+            "point_feature_size": {
+              "type": "integer",
+              "description": "A number of channels of point feature layer.",
+              "default": 7
+            },
+            "max_voxel_size": {
+              "type": "integer",
+              "description": "A maximum size of voxel grid.",
+              "default": 40000
+            },
+            "point_cloud_range": {
+              "type": "array",
+              "description": "An array of distance ranges of each class, this must have same length with `class_names`.",
+              "default": [-121.6, -76.8, -3.0, 121.6, 76.8, 5.0]
+            },
+            "voxel_size": {
+              "type": "array",
+              "description": "An array of voxel grid sizes for PointPainting, this must have same length with `paint_class_names`.",
+              "default": [0.32, 0.32, 8.0]
+            },
+            "down_sample_factor": {
+              "type": "integer",
+              "description": "A scale factor of downsampling points",
+              "default": 1,
+              "minimum": 1
+            },
+            "encoder_in_feature_size": {
+              "type": "integer",
+              "description": "A size of encoder input feature channels.",
+              "default": 12
+            }
+          }
+        },
+        "omp_params": {
+          "type": "object",
+          "properties": {
+            "num_threads": {
+              "type": "integer",
+              "description": "The number of threads that is set to the environment variable OMP_NUM_THREADS.",
+              "default": 1,
+              "minimum": 1
+            }
+          }
+        }
+      },
+      "required": ["model_params", "omp_params"]
+    }
+  },
+  "properties": {
+    "/**": {
+      "type": "object",
+      "properties": {
+        "ros__parameters": {
+          "$ref": "#/definitions/pointpainting_ml_package"
+        }
+      },
+      "required": ["ros__parameters"]
+    }
+  },
+  "required": ["/**"]
+}

perception/image_projection_based_fusion/src/pointpainting_fusion/node.cpp

@@ -102,7 +102,7 @@ PointPaintingFusionNode::PointPaintingFusionNode(const rclcpp::NodeOptions & opt
   const float circle_nms_dist_threshold = static_cast<float>(
     this->declare_parameter<double>("post_process_params.circle_nms_dist_threshold"));
   const auto yaw_norm_thresholds =
-    this->declare_parameter<std::vector<double>>("model_params.yaw_norm_thresholds");
+    this->declare_parameter<std::vector<double>>("post_process_params.yaw_norm_thresholds");
   // densification param
   const std::string densification_world_frame_id =
     this->declare_parameter<std::string>("densification_params.world_frame_id");
@@ -140,8 +140,8 @@ PointPaintingFusionNode::PointPaintingFusionNode(const rclcpp::NodeOptions & opt
       isClassTable_.erase(cls);
     }
   }
-  has_variance_ = this->declare_parameter<bool>("has_variance");
   has_twist_ = this->declare_parameter<bool>("model_params.has_twist");
+  has_variance_ = this->declare_parameter<bool>("model_params.has_variance");
   const std::size_t point_feature_size = static_cast<std::size_t>(
     this->declare_parameter<std::int64_t>("model_params.point_feature_size"));
   const std::size_t max_voxel_size =