update pointpainting_fusion using centerpoint as shared library

perception/image_projection_based_fusion/CMakeLists.txt

@@ -78,39 +78,10 @@ if(TRT_AVAIL AND CUDA_AVAIL AND CUDNN_AVAIL)
 
   include_directories(
     include
-    lib/include
     ${OpenCV_INCLUDE_DIRS}
     ${EIGEN3_INCLUDE_DIR}
     ${PCL_INCLUDE_DIRS}
     ${CUDA_INCLUDE_DIRS}
-    ${TORCH_INCLUDE_DIRS}
-  )
-
-  ### pointpainting_fusion ###
-  ament_auto_add_library(pointpainting_fusion SHARED
-    lib/src/centerpoint_trt.cpp
-    lib/src/pointcloud_densification.cpp
-    lib/src/voxel_generator.cpp
-    lib/src/tensorrt_wrapper.cpp
-    lib/src/network_trt.cpp
-    lib/src/utils.cpp
-  )
-
-  cuda_add_library(pointpainting_fusion_cuda_libraries SHARED
-    lib/src/circle_nms_kernel.cu
-    lib/src/postprocess_kernel.cu
-    lib/src/preprocess_kernel.cu
-    lib/src/scatter_kernel.cu
-  )
-
-  target_link_libraries(pointpainting_fusion
-    ${NVINFER}
-    ${NVONNXPARSER}
-    ${CUDA_LIBRARIES}
-    ${CUBLAS_LIBRARIES}
-    ${CUDA_curand_LIBRARY}
-    ${CUDNN_LIBRARY}
-    pointpainting_fusion_cuda_libraries
   )
 
   ### node ###
@@ -122,13 +93,24 @@ if(TRT_AVAIL AND CUDA_AVAIL AND CUDNN_AVAIL)
     src/roi_cluster_fusion/node.cpp
     src/roi_detected_object_fusion/node.cpp
     src/pointpainting_fusion/node.cpp
+    src/pointpainting_fusion/pointpainting_trt.cpp
+    src/pointpainting_fusion/voxel_generator.cpp
   )
 
+  cuda_add_library(pointpainting_cuda_lib SHARED
+    src/pointpainting_fusion/preprocess_kernel.cu
+  )
+
   target_link_libraries(${PROJECT_NAME}
-    pointpainting_fusion
     ${OpenCV_LIBRARIES}
     ${EIGEN3_LIBRARIES}
     ${PCL_LIBRARIES}
+    ${NVINFER}
+    ${CUDA_LIBRARIES}
+    ${CUBLAS_LIBRARIES}
+    ${CUDA_curand_LIBRARY}
+    ${CUDNN_LIBRARY}
+    pointpainting_cuda_lib
   )
 
   rclcpp_components_register_node(${PROJECT_NAME}

perception/image_projection_based_fusion/config/pointpainting.param.yaml

@@ -2,3 +2,9 @@
   ros__parameters:
     class_names: ["CAR", "PEDESTRIAN", "BICYCLE"]
     rename_car_to_truck_and_bus: true
+    point_feature_size: 6  # x, y, z and car, pedestrian, bicycle
+    max_voxel_size: 40000
+    point_cloud_range: [-76.8, -76.8, -3.0, 76.8, 76.8, 5.0]
+    voxel_size: [0.32, 0.32, 8.0]
+    downsample_factor: 2
+    encoder_in_feature_size: 11

perception/image_projection_based_fusion/include/image_projection_based_fusion/pointpainting_fusion/pointpainting_trt.hpp

@@ -0,0 +1,43 @@
+// Copyright 2021 TIER IV, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IMAGE_PROJECTION_BASED_FUSION__POINTPAINTING_TRT_HPP_
+#define IMAGE_PROJECTION_BASED_FUSION__POINTPAINTING_TRT_HPP_
+
+#include <image_projection_based_fusion/pointpainting_fusion/voxel_generator.hpp>
+#include <lidar_centerpoint/centerpoint_trt.hpp>
+
+namespace centerpoint
+{
+class PointPaintingTRT : public CenterPointTRT
+{
+public:
+  using CenterPointTRT::CenterPointTRT;
+
+  explicit PointPaintingTRT(
+    const NetworkParam & encoder_param, const NetworkParam & head_param,
+    const DensificationParam & densification_param, const CenterPointConfig & config);
+
+  ~PointPaintingTRT();
+
+protected:
+  bool preprocess(
+    const sensor_msgs::msg::PointCloud2 & input_pointcloud_msg,
+    const tf2_ros::Buffer & tf_buffer) override;
+
+  std::unique_ptr<image_projection_based_fusion::VoxelGeneratorTemplate> vg_ptr_pp{nullptr};
+};
+}  // namespace centerpoint
+
+#endif  // IMAGE_PROJECTION_BASED_FUSION__POINTPAINTING_TRT_HPP_

perception/image_projection_based_fusion/lib/include/preprocess_kernel.hpp → perception/image_projection_based_fusion/include/image_projection_based_fusion/pointpainting_fusion/preprocess_kernel.hpp

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef PREPROCESS_KERNEL_HPP_
-#define PREPROCESS_KERNEL_HPP_
+#ifndef IMAGE_PROJECTION_BASED_FUSION__PREPROCESS_KERNEL_HPP_
+#define IMAGE_PROJECTION_BASED_FUSION__PREPROCESS_KERNEL_HPP_
 
 #include <cuda.h>
 #include <cuda_runtime_api.h>
@@ -22,8 +22,10 @@ namespace image_projection_based_fusion
 {
 cudaError_t generateFeatures_launch(
   const float * voxel_features, const float * voxel_num_points, const int * coords,
-  const std::size_t num_voxels, float * features, cudaStream_t stream);
+  const std::size_t num_voxels, const std::size_t max_voxel_size, const float voxel_size_x,
+  const float voxel_size_y, const float voxel_size_z, const float range_min_x,
+  const float range_min_y, const float range_min_z, float * features, cudaStream_t stream);
 
 }  // namespace image_projection_based_fusion
 
-#endif  // PREPROCESS_KERNEL_HPP_
+#endif  // IMAGE_PROJECTION_BASED_FUSION__PREPROCESS_KERNEL_HPP_

perception/image_projection_based_fusion/lib/include/utils.hpp → perception/image_projection_based_fusion/include/image_projection_based_fusion/pointpainting_fusion/voxel_generator.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 TIER IV, Inc.
+// Copyright 2021 TIER IV, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,30 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef UTILS_HPP_
-#define UTILS_HPP_
+#ifndef IMAGE_PROJECTION_BASED_FUSION__VOXEL_GENERATOR_HPP_
+#define IMAGE_PROJECTION_BASED_FUSION__VOXEL_GENERATOR_HPP_
 
-#include <cstddef>
+#include <lidar_centerpoint/preprocess/voxel_generator.hpp>
 
 namespace image_projection_based_fusion
 {
-struct Box3D
+class VoxelGenerator : public centerpoint::VoxelGenerator
 {
-  // initializer not allowed for __shared__ variable
-  int label;
-  float score;
-  float x;
-  float y;
-  float z;
-  float length;
-  float width;
-  float height;
-  float yaw;
-  float vel_x;
-  float vel_y;
-};
+public:
+  using centerpoint::VoxelGenerator::VoxelGenerator;
 
-std::size_t divup(const std::size_t a, const std::size_t b);
+  std::size_t pointsToVoxels(
+    std::vector<float> & voxels, std::vector<int> & coordinates,
+    std::vector<float> & num_points_per_voxel) override;
+};
 }  // namespace image_projection_based_fusion
 
-#endif  // UTILS_HPP_
+#endif  // IMAGE_PROJECTION_BASED_FUSION__VOXEL_GENERATOR_HPP_

perception/image_projection_based_fusion/launch/pointpainting_fusion.launch.xml

@@ -1,5 +1,5 @@
 <launch>
-  <arg name="input/rois_number" default="1"/>
+  <arg name="input/rois_number" default="6"/>
   <arg name="input/rois0" default="rois0"/>
   <arg name="input/camera_info0" default="/camera_info0"/>
   <arg name="input/rois1" default="rois1"/>
@@ -18,7 +18,7 @@
   <arg name="input/camera_info7" default="/camera_info7"/>
   <arg name="input/pointcloud" default="/sensing/lidar/top/rectified/pointcloud"/>
   <arg name="output/objects" default="objects"/>
-  <arg name="model_name" default="pointpainting" description="options: `default` or `aip_x2` or `pointpainting`"/>
+  <arg name="model_name" default="pointpainting" description="options: `pointpainting`"/>
   <arg name="model_path" default="$(find-pkg-share image_projection_based_fusion)/data"/>
   <arg name="model_param_path" default="$(find-pkg-share image_projection_based_fusion)/config/$(var model_name).param.yaml"/>
 
@@ -65,15 +65,15 @@
     <node pkg="image_projection_based_fusion" exec="pointpainting_fusion_node" name="pointpainting" output="screen">
       <remap from="~/input/pointcloud" to="$(var input/pointcloud)"/>
       <remap from="~/output/objects" to="$(var output/objects)"/>
-      <param name="score_threshold" value="0.7"/>
+      <param name="score_threshold" value="0.45"/>
       <param name="densification_world_frame_id" value="map"/>
       <param name="densification_num_past_frames" value="0"/>
       <param name="trt_precision" value="fp16"/>
       <param name="encoder_onnx_path" value="$(var model_path)/pts_voxel_encoder_$(var model_name).onnx"/>
       <param name="encoder_engine_path" value="$(var model_path)/pts_voxel_encoder_$(var model_name).engine"/>
       <param name="head_onnx_path" value="$(var model_path)/pts_backbone_neck_head_$(var model_name).onnx"/>
       <param name="head_engine_path" value="$(var model_path)/pts_backbone_neck_head_$(var model_name).engine"/>
-      <param from="$(var model_param_path)"/>
+      <param from="$(var model_param_path)"/> 
       <param name="rois_number" value="$(var input/rois_number)"/>
 
       <!-- debug -->