Align fastdeploy prediction precision with yolov5

fastdeploy/vision/common/result.cc

@@ -59,31 +59,6 @@ void DetectionResult::Resize(int size) {
   label_ids.resize(size);
 }
 
-void DetectionResult::Sort() {
-  for (size_t i = 0; i < scores.size(); ++i) {
-    float max_score = scores[i];
-    float index = i;
-    for (size_t j = i + 1; j < scores.size(); ++j) {
-      if (max_score < scores[j]) {
-        max_score = scores[j];
-        index = j;
-      }
-    }
-    if (i == index) {
-      continue;
-    }
-    float tmp_score = scores[i];
-    scores[i] = scores[index];
-    scores[index] = tmp_score;
-    int32_t tmp_label_id = label_ids[i];
-    label_ids[i] = label_ids[index];
-    label_ids[index] = tmp_label_id;
-    std::array<float, 4> tmp_box = boxes[i];
-    boxes[i] = boxes[index];
-    boxes[index] = tmp_box;
-  }
-}
-
 std::string DetectionResult::Str() {
   std::string out;
   out = "DetectionResult: [xmin, ymin, xmax, ymax, score, label_id]\n";
@@ -97,5 +72,5 @@ std::string DetectionResult::Str() {
   return out;
 }
 
-} // namespace vision
-} // namespace fastdeploy
+}  // namespace vision
+}  // namespace fastdeploy

fastdeploy/vision/common/result.h

@@ -53,8 +53,6 @@ struct FASTDEPLOY_DECL DetectionResult : public BaseResult {
 
   void Resize(int size);
 
-  void Sort();
-
   std::string Str();
 };
 

fastdeploy/vision/ultralytics/__init__.py

@@ -51,6 +51,10 @@ def padding_value(self):
     def is_no_pad(self):
         return self.model.is_no_pad
 
+    @property
+    def is_mini_pad(self):
+        return self.model.is_mini_pad
+
     @property
     def is_scale_up(self):
         return self.model.is_scale_up
@@ -59,14 +63,16 @@ def is_scale_up(self):
     def stride(self):
         return self.model.stride
 
+    @property
+    def max_wh(self):
+        return self.model.max_wh
+
     @size.setter
     def size(self, wh):
-        assert isinstance(wh, [
-            list, tuple
-        ]), "The value to set `size` must be type of tuple or list."
-        assert len(
-            wh
-        ) == 2, "The value to set `size` must contatins 2 elements means [width, height], but now it contains {} elements.".format(
+        assert isinstance(wh, [list, tuple]),\
+            "The value to set `size` must be type of tuple or list."
+        assert len(wh) == 2,\
+            "The value to set `size` must contatins 2 elements means [width, height], but now it contains {} elements.".format(
             len(wh))
         self.model.size = wh
 
@@ -83,6 +89,13 @@ def is_no_pad(self, value):
             value, bool), "The value to set `is_no_pad` must be type of bool."
         self.model.is_no_pad = value
 
+    @is_mini_pad.setter
+    def is_mini_pad(self, value):
+        assert isinstance(
+            value,
+            bool), "The value to set `is_mini_pad` must be type of bool."
+        self.model.is_mini_pad = value
+
     @is_scale_up.setter
     def is_scale_up(self, value):
         assert isinstance(
@@ -95,3 +108,9 @@ def stride(self, value):
         assert isinstance(
             value, int), "The value to set `stride` must be type of int."
         self.model.stride = value
+
+    @max_wh.setter
+    def max_wh(self, value):
+        assert isinstance(
+            value, float), "The value to set `max_wh` must be type of float."
+        self.model.max_wh = value

fastdeploy/vision/ultralytics/ultralytics_pybind.cc

@@ -34,6 +34,8 @@ void BindUltralytics(pybind11::module& m) {
                      &vision::ultralytics::YOLOv5::padding_value)
       .def_readwrite("is_mini_pad", &vision::ultralytics::YOLOv5::is_mini_pad)
       .def_readwrite("is_no_pad", &vision::ultralytics::YOLOv5::is_no_pad)
-      .def_readwrite("is_scale_up", &vision::ultralytics::YOLOv5::stride);
+      .def_readwrite("is_scale_up", &vision::ultralytics::YOLOv5::is_scale_up)
+      .def_readwrite("stride", &vision::ultralytics::YOLOv5::stride)
+      .def_readwrite("max_wh", &vision::ultralytics::YOLOv5::max_wh);
 }
 }  // namespace fastdeploy

fastdeploy/vision/ultralytics/yolov5.cc

@@ -64,8 +64,9 @@ bool YOLOv5::Initialize() {
   padding_value = {114.0, 114.0, 114.0};
   is_mini_pad = false;
   is_no_pad = false;
-  is_scale_up = true;
+  is_scale_up = false;
   stride = 32;
+  max_wh = 7680.0;
 
   if (!InitRuntime()) {
     FDERROR << "Failed to initialize fastdeploy backend." << std::endl;
@@ -76,6 +77,18 @@ bool YOLOv5::Initialize() {
 
 bool YOLOv5::Preprocess(Mat* mat, FDTensor* output,
                         std::map<std::string, std::array<float, 2>>* im_info) {
+  // process after image load
+  double ratio = (size[0] * 1.0) / std::max(static_cast<float>(mat->Height()),
+                                            static_cast<float>(mat->Width()));
+  if (ratio != 1.0) {
+    int interp = cv::INTER_AREA;
+    if (ratio > 1.0) {
+      interp = cv::INTER_LINEAR;
+    }
+    int resize_h = int(mat->Height() * ratio);
+    int resize_w = int(mat->Width() * ratio);
+    Resize::Run(mat, resize_w, resize_h, -1, -1, interp);
+  }
   // yolov5's preprocess steps
   // 1. letterbox
   // 2. BGR->RGB
@@ -119,11 +132,14 @@ bool YOLOv5::Postprocess(
     if (confidence <= conf_threshold) {
       continue;
     }
+    int32_t label_id = std::distance(data + s + 5, max_class_score);
     // convert from [x, y, w, h] to [x1, y1, x2, y2]
     result->boxes.emplace_back(std::array<float, 4>{
-        data[s] - data[s + 2] / 2, data[s + 1] - data[s + 3] / 2,
-        data[s + 0] + data[s + 2] / 2, data[s + 1] + data[s + 3] / 2});
-    result->label_ids.push_back(std::distance(data + s + 5, max_class_score));
+        data[s] - data[s + 2] / 2.0f + label_id * max_wh,
+        data[s + 1] - data[s + 3] / 2.0f + label_id * max_wh,
+        data[s + 0] + data[s + 2] / 2.0f + label_id * max_wh,
+        data[s + 1] + data[s + 3] / 2.0f + label_id * max_wh});
+    result->label_ids.push_back(label_id);
     result->scores.push_back(confidence);
   }
   utils::NMS(result, nms_iou_threshold);
@@ -141,8 +157,12 @@ bool YOLOv5::Postprocess(
   for (size_t i = 0; i < result->boxes.size(); ++i) {
     float pad_h = (out_h - ipt_h * scale) / 2;
     float pad_w = (out_w - ipt_w * scale) / 2;
-
+    int32_t label_id = (result->label_ids)[i];
     // clip box
+    result->boxes[i][0] = result->boxes[i][0] - max_wh * label_id;
+    result->boxes[i][1] = result->boxes[i][1] - max_wh * label_id;
+    result->boxes[i][2] = result->boxes[i][2] - max_wh * label_id;
+    result->boxes[i][3] = result->boxes[i][3] - max_wh * label_id;
     result->boxes[i][0] = std::max((result->boxes[i][0] - pad_w) / scale, 0.0f);
     result->boxes[i][1] = std::max((result->boxes[i][1] - pad_h) / scale, 0.0f);
     result->boxes[i][2] = std::max((result->boxes[i][2] - pad_w) / scale, 0.0f);
@@ -183,13 +203,11 @@ bool YOLOv5::Predict(cv::Mat* im, DetectionResult* result, float conf_threshold,
 #endif
 
   input_tensors[0].name = InputInfoOfRuntime(0).name;
-
   std::vector<FDTensor> output_tensors;
   if (!Infer(input_tensors, &output_tensors)) {
     FDERROR << "Failed to inference." << std::endl;
     return false;
   }
-
 #ifdef FASTDEPLOY_DEBUG
   TIMERECORD_END(1, "Inference")
   TIMERECORD_START(2)

fastdeploy/vision/ultralytics/yolov5.h

@@ -79,6 +79,8 @@ class FASTDEPLOY_DECL YOLOv5 : public FastDeployModel {
   bool is_scale_up;
   // padding stride, for is_mini_pad
   int stride;
+  // for offseting the boxes by classes when using NMS
+  float max_wh;
 };
 }  // namespace ultralytics
 }  // namespace vision

fastdeploy/vision/utils/nms.cc

@@ -22,13 +22,13 @@ namespace utils {
 // The implementation refers to
 // https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.4/deploy/cpp/src/utils.cc
 void NMS(DetectionResult* result, float iou_threshold) {
-  result->Sort();
+  utils::SortDetectionResult(result);
 
   std::vector<float> area_of_boxes(result->boxes.size());
   std::vector<int> suppressed(result->boxes.size(), 0);
   for (size_t i = 0; i < result->boxes.size(); ++i) {
-    area_of_boxes[i] = (result->boxes[i][2] - result->boxes[i][0] + 1) *
-                       (result->boxes[i][3] - result->boxes[i][1] + 1);
+    area_of_boxes[i] = (result->boxes[i][2] - result->boxes[i][0]) *
+                       (result->boxes[i][3] - result->boxes[i][1]);
   }
 
   for (size_t i = 0; i < result->boxes.size(); ++i) {
@@ -43,12 +43,11 @@ void NMS(DetectionResult* result, float iou_threshold) {
       float ymin = std::max(result->boxes[i][1], result->boxes[j][1]);
       float xmax = std::min(result->boxes[i][2], result->boxes[j][2]);
       float ymax = std::min(result->boxes[i][3], result->boxes[j][3]);
-      float overlap_w = std::max(0.0f, xmax - xmin + 1);
-      float overlap_h = std::max(0.0f, ymax - ymin + 1);
+      float overlap_w = std::max(0.0f, xmax - xmin);
+      float overlap_h = std::max(0.0f, ymax - ymin);
       float overlap_area = overlap_w * overlap_h;
       float overlap_ratio =
-          overlap_area /
-          (area_of_boxes[i] + area_of_boxes[j] - overlap_area + 1e-06);
+          overlap_area / (area_of_boxes[i] + area_of_boxes[j] - overlap_area);
       if (overlap_ratio > iou_threshold) {
         suppressed[j] = 1;
       }
@@ -67,6 +66,6 @@ void NMS(DetectionResult* result, float iou_threshold) {
   }
 }
 
-} // namespace utils
-} // namespace vision
-} // namespace fastdeploy
+}  // namespace utils
+}  // namespace vision
+}  // namespace fastdeploy

fastdeploy/vision/utils/sort_det_res.cc

@@ -0,0 +1,77 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "fastdeploy/vision/utils/utils.h"
+
+namespace fastdeploy {
+namespace vision {
+namespace utils {
+
+void Merge(DetectionResult* result, size_t low, size_t mid, size_t high) {
+  std::vector<std::array<float, 4>>& boxes = result->boxes;
+  std::vector<float>& scores = result->scores;
+  std::vector<int32_t>& label_ids = result->label_ids;
+  std::vector<std::array<float, 4>> temp_boxes(boxes);
+  std::vector<float> temp_scores(scores);
+  std::vector<int32_t> temp_label_ids(label_ids);
+  size_t i = low;
+  size_t j = mid + 1;
+  size_t k = i;
+  for (; i <= mid && j <= high; k++) {
+    if (temp_scores[i] >= temp_scores[j]) {
+      scores[k] = temp_scores[i];
+      label_ids[k] = temp_label_ids[i];
+      boxes[k] = temp_boxes[i];
+      i++;
+    } else {
+      scores[k] = temp_scores[j];
+      label_ids[k] = temp_label_ids[j];
+      boxes[k] = temp_boxes[j];
+      j++;
+    }
+  }
+  while (i <= mid) {
+    scores[k] = temp_scores[i];
+    label_ids[k] = temp_label_ids[i];
+    boxes[k] = temp_boxes[i];
+    k++;
+    i++;
+  }
+  while (j <= high) {
+    scores[k] = temp_scores[j];
+    label_ids[k] = temp_label_ids[j];
+    boxes[k] = temp_boxes[j];
+    k++;
+    j++;
+  }
+}
+
+void MergeSort(DetectionResult* result, size_t low, size_t high) {
+  if (low < high) {
+    size_t mid = (high - low) / 2 + low;
+    MergeSort(result, low, mid);
+    MergeSort(result, mid + 1, high);
+    Merge(result, low, mid, high);
+  }
+}
+
+void SortDetectionResult(DetectionResult* result) {
+  size_t low = 0;
+  size_t high = result->scores.size() - 1;
+  MergeSort(result, low, high);
+}
+
+}  // namespace utils
+}  // namespace vision
+}  // namespace fastdeploy

fastdeploy/vision/utils/utils.h

@@ -14,11 +14,11 @@
 
 #pragma once
 
+#include <set>
+#include <vector>
 #include "fastdeploy/core/fd_tensor.h"
 #include "fastdeploy/utils/utils.h"
 #include "fastdeploy/vision/common/result.h"
-#include <set>
-#include <vector>
 
 namespace fastdeploy {
 namespace vision {
@@ -53,6 +53,9 @@ std::vector<int32_t> TopKIndices(const T* array, int array_size, int topk) {
 
 void NMS(DetectionResult* output, float iou_threshold = 0.5);
 
-} // namespace utils
-} // namespace vision
-} // namespace fastdeploy
+// MergeSort
+void SortDetectionResult(DetectionResult* output);
+
+}  // namespace utils
+}  // namespace vision
+}  // namespace fastdeploy