Merge pull request #30 from NVIDIA-ISAAC-ROS/release-dp3

Isaac ROS 0.30.0 (DP3)

.gitattributes

@@ -1,3 +1,6 @@
+# Ignore Python files in linguist
+*.py linguist-detectable=false
+
 # Images
 *.gif filter=lfs diff=lfs merge=lfs -text
 *.jpg filter=lfs diff=lfs merge=lfs -text
@@ -16,5 +19,10 @@
 *.so filter=lfs diff=lfs merge=lfs -text
 *.so.* filter=lfs diff=lfs merge=lfs -text
 
+# ROS Bags
+**/resources/**/*.zstd filter=lfs diff=lfs merge=lfs -text
+**/resources/**/*.db3 filter=lfs diff=lfs merge=lfs -text
+**/resources/**/*.yaml filter=lfs diff=lfs merge=lfs -text
+
 # Model files
-*.onnx filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text

docs/tensorrt-and-triton-info.md

@@ -28,13 +28,13 @@ Users can either prepare a custom model or download pre-trained models from NGC
 
 In order to be a useful component of a ROS graph, both Isaac ROS Triton and TensorRT inference nodes will require application-specific `pre-processor` (`encoder`) and `post-processor` (`decoder`) nodes to handle type conversion and other necessary steps.
 
-A `pre-processor` node should take in a ROS2 message, perform the pre-processing steps dictated by the model, and then convert the data into an Isaac ROS Tensor List message. For example, a `pre-processor` node could resize an image, normalize it, and then convert it into a Tensor List.
+A `pre-processor` node should take in a ROS 2 message, perform the pre-processing steps dictated by the model, and then convert the data into an Isaac ROS Tensor List message. For example, a `pre-processor` node could resize an image, normalize it, and then convert it into a Tensor List.
 
-A `post-processor` node should be used to convert the Isaac ROS Tensor List output of the model inference into a useful ROS2 message. For example, a `post-processor` node may perform argmax to identify the class label from a classification problem.
+A `post-processor` node should be used to convert the Isaac ROS Tensor List output of the model inference into a useful ROS 2 message. For example, a `post-processor` node may perform argmax to identify the class label from a classification problem.
 
 <div align="center">
 
-![Using TensorRT or Triton](../resources/pipeline.png "Using TensorRT or Triton")
+![Using TensorRT or Triton](../resources/graph.png "Using TensorRT or Triton")
 
 </div>
 

docs/troubleshooting.md

@@ -21,3 +21,29 @@ One cause of this issue is when the GPU being used does not have enough memory t
 ### Solution
 
 Try using the Isaac ROS TensorRT node or the Isaac ROS Triton node with the TensorRT backend instead. Otherwise, a discrete GPU with more VRAM may be required.
+
+## Triton fails to create the TensorRT engine and load a model
+
+### Symptom
+
+```log
+1: [component_container_mt-1] I0331 05:56:07.479791 11359 tensorrt.cc:5591] TRITONBACKEND_ModelInitialize: detectnet (version 1)
+1: [component_container_mt-1] I0331 05:56:07.483989 11359 tensorrt.cc:5640] TRITONBACKEND_ModelInstanceInitialize: detectnet (GPU device 0)
+1: [component_container_mt-1] I0331 05:56:08.169240 11359 logging.cc:49] Loaded engine size: 21 MiB
+1: [component_container_mt-1] E0331 05:56:08.209208 11359 logging.cc:43] 1: [runtime.cpp::parsePlan::314] Error Code 1: Serialization (Serialization assertion plan->header.magicTag == rt::kPLAN_MAGIC_TAG failed.)
+1: [component_container_mt-1] I0331 05:56:08.213483 11359 tensorrt.cc:5678] TRITONBACKEND_ModelInstanceFinalize: delete instance state
+1: [component_container_mt-1] I0331 05:56:08.213525 11359 tensorrt.cc:5617] TRITONBACKEND_ModelFinalize: delete model state
+1: [component_container_mt-1] E0331 05:56:08.214059 11359 model_lifecycle.cc:596] failed to load 'detectnet' version 1: Internal: unable to create TensorRT engine
+1: [component_container_mt-1] ERROR: infer_trtis_server.cpp:1057 Triton: failed to load model detectnet, triton_err_str:Invalid argument, err_msg:load failed for model 'detectnet': version 1 is at UNAVAILABLE state: Internal: unable to create TensorRT engine;
+1: [component_container_mt-1] 
+1: [component_container_mt-1] ERROR: infer_trtis_backend.cpp:54 failed to load model: detectnet, nvinfer error:NVDSINFER_TRITON_ERROR
+1: [component_container_mt-1] ERROR: infer_simple_runtime.cpp:33 failed to initialize backend while ensuring model:detectnet ready, nvinfer error:NVDSINFER_TRITON_ERROR
+1: [component_container_mt-1] ERROR: Error in createNNBackend() <infer_simple_context.cpp:76> [UID = 16]: failed to initialize triton simple runtime for model:detectnet, nvinfer error:NVDSINFER_TRITON_ERROR
+1: [component_container_mt-1] ERROR: Error in initialize() <infer_base_context.cpp:79> [UID = 16]: create nn-backend failed, check config file settings, nvinfer error:NVDSINFER_TRITON_ERROR
+```
+
+### Solution
+
+This error can occur when TensorRT attempts to load an incompatible `model.plan` file. The incompatibility may arise due to a versioning or platform mismatch between the time of plan generation and the time of plan execution.
+
+Delete the `model.plan` file that is being passed in as an argument to the Triton node's `model_repository_paths` parameter, and then use the source package's instructions to regenerate the `model.plan` file from the original weights file (often a `.etlt` or `.onnx` file).

isaac_ros_dnn_encoders/CMakeLists.txt

@@ -15,56 +15,21 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.23.2)
 project(isaac_ros_dnn_encoders LANGUAGES C CXX)
 
-# Default to C++17
-if(NOT CMAKE_CXX_STANDARD)
-  set(CMAKE_CXX_STANDARD 17)
-endif()
-
 if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
   add_compile_options(-Wall -Wextra -Wpedantic)
 endif()
 
-# Default to Release build
-if(NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "")
-  set(CMAKE_BUILD_TYPE "Release" CACHE STRING "" FORCE)
-endif()
-message( STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}" )
-
-execute_process(COMMAND uname -m COMMAND tr -d '\n'
-  OUTPUT_VARIABLE ARCHITECTURE
-)
-message( STATUS "Architecture: ${ARCHITECTURE}" )
-
 find_package(ament_cmake_auto REQUIRED)
 ament_auto_find_build_dependencies()
 
-# Find VPI dependency
-find_package(vpi REQUIRED)
-
-# DNN Image Encoder Node
+# DnnImageEncoderNode
 ament_auto_add_library(dnn_image_encoder_node SHARED src/dnn_image_encoder_node.cpp)
-target_compile_definitions(dnn_image_encoder_node
-  PRIVATE "COMPOSITION_BUILDING_DLL"
-)
-target_link_libraries(dnn_image_encoder_node)
 rclcpp_components_register_nodes(dnn_image_encoder_node "nvidia::isaac_ros::dnn_inference::DnnImageEncoderNode")
 set(node_plugins "${node_plugins}nvidia::isaac_ros::dnn_inference::DnnImageEncoderNode;$<TARGET_FILE:dnn_image_encoder_node>\n")
 
-# Install config directory
-install(
-  DIRECTORY config
-  DESTINATION share/${PROJECT_NAME}
-)
-
-install(TARGETS dnn_image_encoder_node
-  ARCHIVE DESTINATION lib
-  LIBRARY DESTINATION lib
-  RUNTIME DESTINATION bin
-)
-
 if(BUILD_TESTING)
   find_package(ament_lint_auto REQUIRED)
   ament_lint_auto_find_test_dependencies()
@@ -73,7 +38,6 @@ if(BUILD_TESTING)
   add_launch_test(test/isaac_ros_dnn_image_encoder_test.py)
   add_launch_test(test/isaac_ros_dnn_image_encoder_image_norm_test.py)
   add_launch_test(test/isaac_ros_dnn_image_encoder_image_resize_test.py)
-
 endif()
 
-ament_auto_package(INSTALL_TO_SHARE)
+ament_auto_package(INSTALL_TO_SHARE config)

isaac_ros_dnn_encoders/include/isaac_ros_dnn_encoders/dnn_image_encoder_node.hpp

@@ -31,6 +31,13 @@ namespace isaac_ros
 namespace dnn_inference
 {
 
+enum class ResizeMode
+{
+  kDistort = 0,
+  kPad = 1,
+  kCrop = 2
+};
+
 class DnnImageEncoderNode : public nitros::NitrosNode
 {
 public:
@@ -47,6 +54,8 @@ class DnnImageEncoderNode : public nitros::NitrosNode
   const uint16_t network_image_height_;
   const std::vector<double> image_mean_;
   const std::vector<double> image_stddev_;
+  int64_t num_blocks_;
+  const ResizeMode resize_mode_;
 };
 
 }  // namespace dnn_inference

isaac_ros_dnn_encoders/package.xml

@@ -21,21 +21,24 @@ SPDX-License-Identifier: Apache-2.0
 <?xml-model href="http://download.ros.org/schema/package_format3.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
 <package format="3">
   <name>isaac_ros_dnn_encoders</name>
-  <version>0.20.0</version>
+  <version>0.30.0</version>
   <description>Encoders for preprocessing before running deep learning inference</description>
   <maintainer email="hemals@nvidia.com">Hemal Shah</maintainer>
   <license>Apache-2.0</license>
   <url type="website">https://developer.nvidia.com/isaac-ros-gems/</url>
   <author>Ethan Yu</author>
   <author>Kajanan Chinniah</author>
   <author>Swapnesh Wani</author>
-  
+
   <depend>rclcpp</depend>
   <depend>rclcpp_components</depend>
+  <depend>isaac_ros_image_proc</depend>
   <depend>isaac_ros_nitros</depend>
   <depend>isaac_ros_nitros_image_type</depend>
   <depend>isaac_ros_nitros_tensor_list_type</depend>
 
+  <build_depend>isaac_ros_common</build_depend>
+
   <test_depend>ament_lint_auto</test_depend>
   <test_depend>ament_lint_common</test_depend>
   <test_depend>isaac_ros_test</test_depend>

isaac_ros_dnn_encoders/src/dnn_image_encoder_node.cpp

@@ -59,11 +59,11 @@ constexpr char APP_YAML_FILENAME[] = "config/dnn_image_encoder_node.yaml";
 constexpr char PACKAGE_NAME[] = "isaac_ros_dnn_encoders";
 
 const std::vector<std::pair<std::string, std::string>> EXTENSIONS = {
-  {"isaac_ros_nitros", "gxf/std/libgxf_std.so"},
-  {"isaac_ros_nitros", "gxf/cuda/libgxf_cuda.so"},
-  {"isaac_ros_nitros", "gxf/serialization/libgxf_serialization.so"},
-  {"isaac_ros_nitros", "gxf/tensorops/libgxf_tensorops.so"},
-  {"isaac_ros_nitros", "gxf/libgxf_message_compositor.so"}
+  {"isaac_ros_gxf", "gxf/lib/std/libgxf_std.so"},
+  {"isaac_ros_gxf", "gxf/lib/cuda/libgxf_cuda.so"},
+  {"isaac_ros_gxf", "gxf/lib/serialization/libgxf_serialization.so"},
+  {"isaac_ros_image_proc", "gxf/lib/image_proc/libgxf_tensorops.so"},
+  {"isaac_ros_gxf", "gxf/lib/libgxf_message_compositor.so"}
 };
 const std::vector<std::string> PRESET_EXTENSION_SPEC_NAMES = {
   "isaac_ros_dnn_encoders",
@@ -108,7 +108,10 @@ DnnImageEncoderNode::DnnImageEncoderNode(const rclcpp::NodeOptions options)
   network_image_width_(declare_parameter<uint16_t>("network_image_width", 0)),
   network_image_height_(declare_parameter<uint16_t>("network_image_height", 0)),
   image_mean_(declare_parameter<std::vector<double>>("image_mean", {0.5, 0.5, 0.5})),
-  image_stddev_(declare_parameter<std::vector<double>>("image_stddev", {0.5, 0.5, 0.5}))
+  image_stddev_(declare_parameter<std::vector<double>>("image_stddev", {0.5, 0.5, 0.5})),
+  num_blocks_(declare_parameter<int64_t>("num_blocks", 40)),
+  resize_mode_(static_cast<ResizeMode>(
+      declare_parameter<int>("resize_mode", static_cast<int>(ResizeMode::kDistort))))
 {
   if (network_image_width_ == 0) {
     throw std::invalid_argument(
@@ -168,6 +171,10 @@ void DnnImageEncoderNode::postLoadGraphCallback()
   getNitrosContext().setParameterUInt64(
     "resizer", "nvidia::cvcore::tensor_ops::Resize", "output_height", network_image_height_);
 
+  getNitrosContext().setParameterBool(
+    "resizer", "nvidia::cvcore::tensor_ops::Resize", "keep_aspect_ratio",
+    resize_mode_ != ResizeMode::kDistort);
+
   const gxf::optimizer::ComponentInfo output_comp_info = {
     "nvidia::gxf::Vault",  // component_type_name
     "vault",               // component_name
@@ -199,6 +206,19 @@ void DnnImageEncoderNode::postLoadGraphCallback()
     getNitrosContext().setParameterUInt64(
       "reshaper", "nvidia::gxf::BlockMemoryPool", "block_size", block_size * sizeof(float));
 
+    // The minimum number of memory blocks is set based on the receiver queue capacity
+    uint64_t num_blocks = std::max(static_cast<int>(num_blocks_), 40);
+    getNitrosContext().setParameterUInt64(
+      "resizer", "nvidia::gxf::BlockMemoryPool", "num_blocks", num_blocks);
+    getNitrosContext().setParameterUInt64(
+      "color_space_converter", "nvidia::gxf::BlockMemoryPool", "num_blocks", num_blocks);
+    getNitrosContext().setParameterUInt64(
+      "normalizer", "nvidia::gxf::BlockMemoryPool", "num_blocks", num_blocks);
+    getNitrosContext().setParameterUInt64(
+      "interleaved_to_planar", "nvidia::gxf::BlockMemoryPool", "num_blocks", num_blocks);
+    getNitrosContext().setParameterUInt64(
+      "reshaper", "nvidia::gxf::BlockMemoryPool", "num_blocks", num_blocks);
+
     std::vector<int32_t> final_tensor_shape{1,
       static_cast<int32_t>(image_type_to_channel_size.at(image_type->second)),
       static_cast<int32_t>(network_image_height_),

isaac_ros_dnn_encoders/test/isaac_ros_dnn_image_encoder_image_resize_test.py

@@ -54,7 +54,8 @@ def generate_test_description():
             'network_image_width': NETWORK_IMAGE_WIDTH,
             'network_image_height': NETWORK_IMAGE_HEIGHT,
             'image_mean': list(IMAGE_MEAN),
-            'image_stddev': list(IMAGE_STDDEV)
+            'image_stddev': list(IMAGE_STDDEV),
+            'resize_mode': 1  # Pad mode
         }],
         remappings=[('encoded_tensor', 'tensors')])
 

isaac_ros_dnn_inference_test/CMakeLists.txt

@@ -15,58 +15,28 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.23.2)
 project(isaac_ros_dnn_inference_test LANGUAGES C CXX)
 
-# Default to C++17
-if(NOT CMAKE_CXX_STANDARD)
-  set(CMAKE_CXX_STANDARD 17)
-endif()
-
 if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
   add_compile_options(-Wall -Wextra -Wpedantic)
 endif()
 
-# Default to Release build
-if(NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "")
-  set(CMAKE_BUILD_TYPE "Release" CACHE STRING "" FORCE)
-endif()
-message( STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}" )
-
-execute_process(COMMAND uname -m COMMAND tr -d '\n'
-  OUTPUT_VARIABLE ARCHITECTURE
-)
-message( STATUS "Architecture: ${ARCHITECTURE}" )
-
 find_package(ament_cmake_auto REQUIRED)
 ament_auto_find_build_dependencies()
 
 # test_tensor_publisher_node
 ament_auto_add_library(test_tensor_publisher_node SHARED src/test_tensor_publisher_node.cpp)
-target_compile_definitions(test_tensor_publisher_node
-  PRIVATE "COMPOSITION_BUILDING_DLL"
-)
-target_link_libraries(test_tensor_publisher_node)
 rclcpp_components_register_nodes(test_tensor_publisher_node "nvidia::isaac_ros::dnn_inference::TestTensorPublisherNode")
 set(node_plugins "${node_plugins}nvidia::isaac_ros::dnn_inference::TestTensorPublisherNode;$<TARGET_FILE:test_tensor_publisher_node>\n")
 
 # run test tensor publisher executable
-ament_auto_add_executable("run_test_publisher"
-  src/test_tensor_publisher_main.cpp
-)
-
-target_link_libraries("run_test_publisher" test_tensor_publisher_node)
-
-install(TARGETS "run_test_publisher"
-  ARCHIVE DESTINATION lib
-  LIBRARY DESTINATION lib
-  RUNTIME DESTINATION bin
-)
+ament_auto_add_executable(run_test_publisher src/test_tensor_publisher_main.cpp)
+target_link_libraries(run_test_publisher test_tensor_publisher_node)
 
 if(BUILD_TESTING)
   find_package(ament_lint_auto REQUIRED)
   ament_lint_auto_find_test_dependencies()
-
 endif()
 
 ament_auto_package()

isaac_ros_dnn_inference_test/package.xml

@@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0
 <?xml-model href="http://download.ros.org/schema/package_format3.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
 <package format="3">
   <name>isaac_ros_dnn_inference_test</name>
-  <version>0.20.0</version>
+  <version>0.30.0</version>
   <description>DNN Inference support for Isaac ROS</description>
 
   <maintainer email="hemals@nvidia.com">Hemal Shah</maintainer>

isaac_ros_tensor_rt/CMakeLists.txt

@@ -15,54 +15,35 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.23.2)
 project(isaac_ros_tensor_rt LANGUAGES C CXX)
 
-# Default to C++17
-if(NOT CMAKE_CXX_STANDARD)
-  set(CMAKE_CXX_STANDARD 17)
-endif()
-
 if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
   add_compile_options(-Wall -Wextra -Wpedantic)
 endif()
 
-# Default to Release build
-if(NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "")
-  set(CMAKE_BUILD_TYPE "Release" CACHE STRING "" FORCE)
-endif()
-message( STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}" )
-
-execute_process(COMMAND uname -m COMMAND tr -d '\n'
-  OUTPUT_VARIABLE ARCHITECTURE
-)
-message( STATUS "Architecture: ${ARCHITECTURE}" )
-
 find_package(ament_cmake_auto REQUIRED)
 ament_auto_find_build_dependencies()
 
-# tensor_rt_node
+# TensorRTNode
 ament_auto_add_library(tensor_rt_node SHARED src/tensor_rt_node.cpp)
-target_compile_definitions(tensor_rt_node
-  PRIVATE "COMPOSITION_BUILDING_DLL"
-)
-target_link_libraries(tensor_rt_node)
 rclcpp_components_register_nodes(tensor_rt_node "nvidia::isaac_ros::dnn_inference::TensorRTNode")
 set(node_plugins "${node_plugins}nvidia::isaac_ros::dnn_inference::TensorRTNode;$<TARGET_FILE:tensor_rt_node>\n")
 
-# Install config directory
-install(
-  DIRECTORY config
-  DESTINATION share/${PROJECT_NAME}
-)
+### Install extensions built from source
+
+# TensorRT
+add_subdirectory(gxf/tensor_rt)
+install(TARGETS gxf_tensor_rt DESTINATION share/${PROJECT_NAME}/gxf/tensor_rt)
+
+### End extensions
 
 if(BUILD_TESTING)
   find_package(ament_lint_auto REQUIRED)
   ament_lint_auto_find_test_dependencies()
 
   find_package(launch_testing_ament_cmake REQUIRED)
   add_launch_test(test/isaac_ros_tensor_rt_test.py TIMEOUT "300")
-
 endif()
 
-ament_auto_package(INSTALL_TO_SHARE launch)
+ament_auto_package(INSTALL_TO_SHARE config launch)