【Hackathon 6th No.58】support model convert from fp32 to fp16

paddle2onnx/mapper/nn/pool2d.cc

@@ -116,12 +116,20 @@ void Pool2dMapper::AdaptivePool(const std::vector<TensorInfo>& input_info,
     onnx_pool_type = iter->second[0];
   }
 
-  std::shared_ptr<ONNX_NAMESPACE::NodeProto>* node_ptr;
-  auto input = helper_->AutoCast(input_info[0].name, input_info[0].dtype,
-                                 P2ODataType::FP32);
-  auto node = helper_->MakeNode(onnx_pool_type, {input});
-  helper_->AutoCast(node->output(0), output_info[0].name, P2ODataType::FP32,
-                    output_info[0].dtype);
+  std::shared_ptr<ONNX_NAMESPACE::NodeProto> node(nullptr);
+  if (kNoNeedCastTypesOpSet7.find(input_info[0].dtype) != kNoNeedCastTypesOpSet7.end())
+  {
+    node = helper_->MakeNode(onnx_pool_type, {input_info[0].name}, {output_info[0].name});
+  }
+  else
+  {
+    auto input = helper_->AutoCast(input_info[0].name, input_info[0].dtype,
+                                   P2ODataType::FP32);
+    node = helper_->MakeNode(onnx_pool_type, {input});
+    helper_->AutoCast(node->output(0), output_info[0].name, P2ODataType::FP32,
+                      output_info[0].dtype);
+  }
+
   std::vector<int64_t> kernel_size = {kernel_h, kernel_w};
   AddAttribute(node, "kernel_shape", kernel_size);
   std::vector<int64_t> strides = {stride_h, stride_w};
@@ -165,8 +173,12 @@ void Pool2dMapper::NoAdaptivePool(const std::vector<TensorInfo>& input_info,
 
   int64_t max_ksize = *std::max_element(std::begin(k_size_), std::end(k_size_));
   int64_t max_pads = *std::max_element(std::begin(pads_), std::end(pads_));
-  auto input_x = helper_->AutoCast(input_info[0].name, input_info[0].dtype,
-                                   P2ODataType::FP32);
+  std::string input_x = input_info[0].name;
+  if (kNoNeedCastTypesOpSet7.find(input_info[0].dtype) == kNoNeedCastTypesOpSet7.end())
+  {
+    input_x = helper_->AutoCast(input_info[0].name, input_info[0].dtype,
+                                P2ODataType::FP32);
+  }
   if (max_ksize <= max_pads) {
     std::vector<int64_t> onnx_paddings = {0, 0, pads_[0], pads_[1],
                                           0, 0, pads_[2], pads_[3]};
@@ -199,9 +211,17 @@ void Pool2dMapper::NoAdaptivePool(const std::vector<TensorInfo>& input_info,
     auto iter = op_mapper_.find(pooling_type_);
     onnx_pool_type = iter->second[0];
   }
-  auto node = helper_->MakeNode(onnx_pool_type, {input_x});
-  helper_->AutoCast(node->output(0), output_info[0].name, P2ODataType::FP32,
-                    output_info[0].dtype);
+  std::shared_ptr<ONNX_NAMESPACE::NodeProto> node(nullptr);
+  if (kNoNeedCastTypesOpSet7.find(input_info[0].dtype) != kNoNeedCastTypesOpSet7.end())
+  {
+    node = helper_->MakeNode(onnx_pool_type, {input_x}, {output_info[0].name});
+  }
+  else
+  {
+    node = helper_->MakeNode(onnx_pool_type, {input_x});
+    helper_->AutoCast(node->output(0), output_info[0].name, P2ODataType::FP32,
+                      output_info[0].dtype);
+  }
 
   AddAttribute(node, "kernel_shape", k_size_);
   AddAttribute(node, "strides", strides_);
@@ -317,11 +337,18 @@ void Pool2dMapper::Opset7() {
       auto iter = op_mapper_.find(pooling_type_);
       onnx_pool_type = iter->second[1];
     }
-    auto input = helper_->AutoCast(input_info[0].name, input_info[0].dtype,
-                                   P2ODataType::FP32);
-    auto output = helper_->MakeNode(onnx_pool_type, {input})->output(0);
-    helper_->AutoCast(output, output_info[0].name, P2ODataType::FP32,
-                      output_info[0].dtype);
+    if (kNoNeedCastTypesOpSet7.find(input_info[0].dtype) != kNoNeedCastTypesOpSet7.end())
+    {
+      auto output = helper_->MakeNode(onnx_pool_type, {input_info[0].name}, {output_info[0].name});
+    }
+    else
+    {
+      auto input = helper_->AutoCast(input_info[0].name, input_info[0].dtype,
+                                     P2ODataType::FP32);
+      auto output = helper_->MakeNode(onnx_pool_type, {input})->output(0);
+      helper_->AutoCast(output, output_info[0].name, P2ODataType::FP32,
+                        output_info[0].dtype);
+    }
   } else if (adaptive_) {
     AdaptivePool(input_info, output_info);
   } else {

paddle2onnx/mapper/nn/pool2d.h

@@ -63,6 +63,7 @@ class Pool2dMapper : public Mapper {
                     const std::vector<TensorInfo>& output_info);
   void NoAdaptivePool(const std::vector<TensorInfo>& input_info,
                       const std::vector<TensorInfo>& output_info);
+  const std::unordered_set<int32_t> kNoNeedCastTypesOpSet7{P2ODataType::FP16, P2ODataType::FP32};
   bool ceil_mode_;
   bool global_pooling_;
   bool adaptive_;

paddle2onnx/mapper/nn/pool3d.cc

@@ -56,12 +56,21 @@ void Pool3dMapper::AdaptivePool(const std::vector<TensorInfo>& input_info,
     auto iter = op_mapper_.find(pooling_type_);
     onnx_pool_type = iter->second[0];
   }
-  std::shared_ptr<ONNX_NAMESPACE::NodeProto>* node_ptr;
-  auto input = helper_->AutoCast(input_info[0].name, input_info[0].dtype,
-                                 P2ODataType::FP32);
-  auto node = helper_->MakeNode(onnx_pool_type, {input});
-  helper_->AutoCast(node->output(0), output_info[0].name, P2ODataType::FP32,
-                    output_info[0].dtype);
+
+  std::shared_ptr<ONNX_NAMESPACE::NodeProto> node;
+  if (kNoNeedCastTypesOpSet7.find(input_info[0].dtype) != kNoNeedCastTypesOpSet7.end())
+  {
+    node = helper_->MakeNode(onnx_pool_type, {input_info[0].name}, {output_info[0].name});
+  }
+  else
+  {
+    auto input = helper_->AutoCast(input_info[0].name, input_info[0].dtype,
+                                   P2ODataType::FP32);
+    node = helper_->MakeNode(onnx_pool_type, {input});
+    helper_->AutoCast(node->output(0), output_info[0].name, P2ODataType::FP32,
+                      output_info[0].dtype);
+  }
+
   std::vector<int64_t> kernel_size = {kernel_d, kernel_h, kernel_w};
   AddAttribute(node, "kernel_shape", kernel_size);
   std::vector<int64_t> strides = {stride_d, stride_h, stride_w};
@@ -109,8 +118,13 @@ void Pool3dMapper::NoAdaptivePool(const std::vector<TensorInfo>& input_info,
 
   int64_t max_ksize = *std::max_element(std::begin(k_size_), std::end(k_size_));
   int64_t max_pads = *std::max_element(std::begin(pads_), std::end(pads_));
-  auto input_x = helper_->AutoCast(input_info[0].name, input_info[0].dtype,
-                                   P2ODataType::FP32);
+  auto input_x = input_info[0].name;
+  if (kNoNeedCastTypesOpSet7.find(input_info[0].dtype) == kNoNeedCastTypesOpSet7.end())
+  {
+    input_x = helper_->AutoCast(input_info[0].name, input_info[0].dtype,
+                                P2ODataType::FP32);
+  }
+
   if (max_ksize <= max_pads) {
     std::vector<int64_t> onnx_paddings = {0, 0, pads_[0], pads_[1], pads_[2],
                                           0, 0, pads_[3], pads_[4], pads_[5]};
@@ -143,9 +157,17 @@ void Pool3dMapper::NoAdaptivePool(const std::vector<TensorInfo>& input_info,
     auto iter = op_mapper_.find(pooling_type_);
     onnx_pool_type = iter->second[0];
   }
-  auto node = helper_->MakeNode(onnx_pool_type, {input_x});
-  helper_->AutoCast(node->output(0), output_info[0].name, P2ODataType::FP32,
-                    output_info[0].dtype);
+  std::shared_ptr<ONNX_NAMESPACE::NodeProto> node(nullptr);
+  if (kNoNeedCastTypesOpSet7.find(input_info[0].dtype) != kNoNeedCastTypesOpSet7.end())
+  {
+    node = helper_->MakeNode(onnx_pool_type, {input_x}, {output_info[0].name});
+  }
+  else
+  {
+    node = helper_->MakeNode(onnx_pool_type, {input_x});
+    helper_->AutoCast(node->output(0), output_info[0].name, P2ODataType::FP32,
+                      output_info[0].dtype);
+  }
 
   AddAttribute(node, "kernel_shape", k_size_);
   AddAttribute(node, "strides", strides_);
@@ -247,11 +269,19 @@ void Pool3dMapper::Opset7() {
       auto iter = op_mapper_.find(pooling_type_);
       onnx_pool_type = iter->second[1];
     }
-    auto input = helper_->AutoCast(input_info[0].name, input_info[0].dtype,
-                                   P2ODataType::FP32);
-    auto output = helper_->MakeNode(onnx_pool_type, {input})->output(0);
-    helper_->AutoCast(output, output_info[0].name, P2ODataType::FP32,
-                      output_info[0].dtype);
+
+    if (kNoNeedCastTypesOpSet7.find(input_info[0].dtype) != kNoNeedCastTypesOpSet7.end())
+    {
+      auto output = helper_->MakeNode(onnx_pool_type, {input_info[0].name}, {output_info[0].name});
+    }
+    else
+    {
+      auto input = helper_->AutoCast(input_info[0].name, input_info[0].dtype,
+                                     P2ODataType::FP32);
+      auto output = helper_->MakeNode(onnx_pool_type, {input})->output(0);
+      helper_->AutoCast(output, output_info[0].name, P2ODataType::FP32,
+                        output_info[0].dtype);
+    }
   } else if (adaptive_) {
     AdaptivePool(input_info, output_info);
   } else {

paddle2onnx/mapper/nn/pool3d.h

@@ -50,6 +50,7 @@ class Pool3dMapper : public Mapper {
                     const std::vector<TensorInfo>& output_info);
   void NoAdaptivePool(const std::vector<TensorInfo>& input_info,
                       const std::vector<TensorInfo>& output_info);
+  const std::unordered_set<int32_t> kNoNeedCastTypesOpSet7{P2ODataType::FP16, P2ODataType::FP32};
   bool ceil_mode_;
   bool global_pooling_;
   bool adaptive_;

paddle2onnx/mapper/tensor/fill_constant.cc

@@ -25,11 +25,12 @@ int32_t FillConstantMapper::GetMinOpset(bool verbose) {
   auto onnx_dtype = GetOnnxDtype(out_info[0].dtype);
   if (onnx_dtype != ONNX_NAMESPACE::TensorProto::INT32 &&
       onnx_dtype != ONNX_NAMESPACE::TensorProto::INT64 &&
+      onnx_dtype != ONNX_NAMESPACE::TensorProto::FLOAT16 &&
       onnx_dtype != ONNX_NAMESPACE::TensorProto::FLOAT &&
       onnx_dtype != ONNX_NAMESPACE::TensorProto::DOUBLE && 
       onnx_dtype != ONNX_NAMESPACE::TensorProto::BOOL
       ) {
-    Error() << "Only support int32/int64/float32/float64/bool data type in "
+    Error() << "Only support int32/int64/float16/float32/float64/bool data type in "
                "fill_constant operator."
             << std::endl;
     return -1;
@@ -79,9 +80,8 @@ void FillConstantMapper::Opset7() {
   float value = GetFillValue();
   if (HasInput("ValueTensor")) {
     auto value_info = GetInput("ValueTensor");
-    auto value_tensor = helper_->AutoCast(value_info[0].name, value_info[0].dtype, out_info[0].dtype);
     auto out = helper_->Constant(shape, GetOnnxDtype(out_info[0].dtype), float(0.0));
-    helper_->MakeNode("Add", {out, value_tensor}, {out_info[0].name});
+    helper_->MakeNode("Add", {out, value_info[0].name}, {out_info[0].name});
   } else {
     helper_->Constant(out_info[0].name, shape, GetOnnxDtype(out_info[0].dtype), value);
   }
@@ -149,9 +149,7 @@ void FillConstantMapper::Opset9() {
   }
   if (value_is_tensor) {
     auto value_info = GetInput("ValueTensor");
-    std::string cast_value = helper_->AutoCast(
-        value_info[0].name, value_info[0].dtype, out_info[0].dtype);
-    helper_->MakeNode("Add", {out, cast_value}, {out_info[0].name});
+    helper_->MakeNode("Add", {out, value_info[0].name}, {out_info[0].name});
   } else {
     helper_->MakeNode("Identity", {out}, {out_info[0].name});
   }

paddle2onnx/mapper/tensor/matmul.cc

@@ -20,7 +20,7 @@ REGISTER_MAPPER(matmul, MatmulMapper)
 
 std::string MatmulMapper::GetTrans(std::vector<TensorInfo>& input_info) {
   std::string castd_name = input_info[0].name;
-  if (input_info[0].dtype == P2ODataType::FP64) {
+  if (kNoNeedCastTypesOpSet7.find(input_info[0].dtype) == kNoNeedCastTypesOpSet7.end()) {
     castd_name = helper_->AutoCast(input_info[0].name, input_info[0].dtype,
                                    P2ODataType::FP32);
   }
@@ -43,11 +43,30 @@ void MatmulMapper::Opset7() {
   if (transpose_Y_) {
     input_y = GetTrans(input_y_info);
   }
-  if (fabs(alpha_ - 1.0) < 1e-6) {
+
+  if (kNoNeedCastTypesOpSet7.find(input_x_info[0].dtype) != kNoNeedCastTypesOpSet7.end())
+  {
+    if (fabs(alpha_ - 1.0) < 1e-6)
+    {
+      auto node = helper_->MakeNode("MatMul", {input_x, input_y}, {output_info[0].name});
+    }
+    else
+    {
+      auto mutmul_node = helper_->MakeNode("MatMul", {input_x, input_y});
+      std::string scale_node =
+          helper_->Constant({1}, GetOnnxDtype(input_x_info[0].dtype), alpha_);
+      auto mul_node =
+          helper_->MakeNode("Mul", {mutmul_node->output(0), scale_node}, {output_info[0].name});
+    }
+  }
+  else if (fabs(alpha_ - 1.0) < 1e-6)
+  {
     auto node = helper_->MakeNode("MatMul", {input_x, input_y});
     helper_->AutoCast(node->output(0), output_info[0].name, P2ODataType::FP32,
                       input_y_info[0].dtype);
-  } else {
+  }
+  else
+  {
     auto mutmul_node = helper_->MakeNode("MatMul", {input_x, input_y});
     std::string scale_node =
         helper_->Constant({1}, GetOnnxDtype(input_x_info[0].dtype), alpha_);

paddle2onnx/mapper/tensor/matmul.h

@@ -34,6 +34,7 @@ class MatmulMapper : public Mapper {
 
  private:
   std::string GetTrans(std::vector<TensorInfo>& input_info);
+  const std::unordered_set<int32_t> kNoNeedCastTypesOpSet7{P2ODataType::FP16, P2ODataType::FP32, P2ODataType::INT32, P2ODataType::INT64};
   bool transpose_X_ = false;
   bool transpose_Y_ = false;
   float alpha_ = 1.0;

paddle2onnx/mapper/tensor/matmul_v2.cc

@@ -22,8 +22,13 @@ namespace paddle2onnx {
 REGISTER_MAPPER(matmul_v2, MatmulV2Mapper)
 
 std::string MatmulV2Mapper::GetTrans(std::vector<TensorInfo>& input_info) {
-  std::string castd_name = helper_->AutoCast(
-      input_info[0].name, input_info[0].dtype, P2ODataType::FP32);
+  std::string castd_name = input_info[0].name;
+  if (kNoNeedCastTypesOpSet7.find(input_info[0].dtype) == kNoNeedCastTypesOpSet7.end())
+  {
+    castd_name = helper_->AutoCast(
+        input_info[0].name, input_info[0].dtype, P2ODataType::FP32);
+  }
+
   std::vector<int64_t> perm = Arange(0, input_info[0].Rank());
   std::swap(perm[perm.size() - 1], perm[perm.size() - 2]);
   auto transpose_node = helper_->MakeNode("Transpose", {castd_name});
@@ -43,9 +48,16 @@ void MatmulV2Mapper::Opset7() {
   if (trans_y_) {
     input_y = GetTrans(input_y_info);
   }
-  auto node = helper_->MakeNode("MatMul", {input_x, input_y});
-  helper_->AutoCast(node->output(0), output_info[0].name, P2ODataType::FP32,
-                    input_y_info[0].dtype);
+  if (kNoNeedCastTypesOpSet7.find(input_y_info[0].dtype) != kNoNeedCastTypesOpSet7.end())
+  {
+    auto node = helper_->MakeNode("MatMul", {input_x, input_y}, {output_info[0].name});
+  }
+  else
+  {
+    auto node = helper_->MakeNode("MatMul", {input_x, input_y});
+    helper_->AutoCast(node->output(0), output_info[0].name, P2ODataType::FP32,
+                      input_y_info[0].dtype);
+  }
 }
 
 }  // namespace paddle2onnx

paddle2onnx/mapper/tensor/matmul_v2.h

@@ -33,6 +33,7 @@ class MatmulV2Mapper : public Mapper {
 
  private:
   std::string GetTrans(std::vector<TensorInfo>& input_info);
+  const std::unordered_set<int32_t> kNoNeedCastTypesOpSet7{P2ODataType::FP16, P2ODataType::FP32, P2ODataType::INT32, P2ODataType::INT64};
   bool trans_x_ = false;
   bool trans_y_ = false;
 };

paddle2onnx/parser/parser.cc

@@ -815,7 +815,6 @@ void PaddleParser::GetGlobalBlockInputOutputInfo() {
 }
 
 int32_t PaddleDataTypeSize(int32_t paddle_dtype) {
-  Assert(paddle_dtype != FP16, "Float16 is not supported.");
   if (paddle_dtype == P2ODataType::BOOL) {
     return sizeof(bool);
   } else if (paddle_dtype == P2ODataType::INT8) {
@@ -828,6 +827,8 @@ int32_t PaddleDataTypeSize(int32_t paddle_dtype) {
     return sizeof(int64_t);
   } else if (paddle_dtype == P2ODataType::FP32) {
     return sizeof(float);
+  } else if (paddle_dtype == P2ODataType::FP16) {
+    return sizeof(int16_t);
   } else if (paddle_dtype == P2ODataType::FP64) {
     return sizeof(double);
   } else if (paddle_dtype == P2ODataType::UINT8) {

tests/run.sh

@@ -60,7 +60,8 @@ ignore="test_auto_scan_multiclass_nms.py
         test_unsqueeze.py \
         test_quantize_model.py \
         test_quantize_model_minist.py \
-        test_quantize_model_speedup.py"
+        test_quantize_model_speedup.py \
+        test_resnet_fp16.py"
 bug=0
 
 # Install Python Packet
@@ -69,6 +70,7 @@ $PY_CMD -m pip install pytest
 $PY_CMD -m pip install onnx onnxruntime tqdm filelock
 $PY_CMD -m pip install paddlepaddle==2.6.0
 $PY_CMD -m pip install six hypothesis
+$PY_CMD -m pip install numpy==1.26.4
 
 
 export ENABLE_DEV=ON