Skip to content

[Paddle inference] Add conv_fusion_fp16 #44435

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jul 21, 2022
Merged

[Paddle inference] Add conv_fusion_fp16 #44435

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7f2faf1
convfusionfp16
xiaoxiaohehe001 Jul 19, 2022
830c409
convfusionfp16
xiaoxiaohehe001 Jul 19, 2022
f9d5994
convfusionfp16
xiaoxiaohehe001 Jul 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 58 additions & 0 deletions paddle/fluid/framework/ir/conv_bn_fuse_pass.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,27 @@ void ConvBNFusePass::ApplyImpl(ir::Graph* graph) const {
return;
}

// conv_weight fp32 --> fp16
auto* conv_weight_tensor =
scope->FindVar(conv_weight->Name())->GetMutable<LoDTensor>();
auto tensor_type = conv_weight_tensor->dtype();

if (tensor_type == paddle::experimental::DataType::FLOAT16) {
framework::Tensor weight_float_tensor;
weight_float_tensor.set_type(paddle::experimental::DataType::FLOAT32);
weight_float_tensor.Resize(conv_weight_tensor->dims());
auto* weight_float_data =
weight_float_tensor.mutable_data<float>(platform::CPUPlace());
auto* data =
conv_weight_tensor->mutable_data<float16>(platform::CPUPlace());
for (int i = 0; i < conv_weight_tensor->numel(); i++) {
weight_float_data[i] = static_cast<float>(data[i]);
}
conv_weight_tensor->clear();
paddle::framework::TensorCopySync(
weight_float_tensor, platform::CPUPlace(), conv_weight_tensor);
}

// Get batch norm bias
auto* bn_bias_tensor =
scope->FindVar(bn_bias->Name())->GetMutable<LoDTensor>();
Expand Down Expand Up @@ -319,6 +340,43 @@ void ConvBNFusePass::ApplyImpl(ir::Graph* graph) const {
epsilon,
conv_type());

if (tensor_type == paddle::experimental::DataType::FLOAT16) {
{
framework::Tensor weight_float16_tensor;
weight_float16_tensor.set_type(paddle::experimental::DataType::FLOAT16);
weight_float16_tensor.Resize(conv_weight_tensor->dims());
auto* weight_float16_data =
weight_float16_tensor.mutable_data<float16>(platform::CPUPlace());
auto* data =
conv_weight_tensor->mutable_data<float>(platform::CPUPlace());
for (int i = 0; i < conv_weight_tensor->numel(); i++) {
weight_float16_data[i] = static_cast<float16>(data[i]);
}
conv_weight_tensor->clear();
paddle::framework::TensorCopySync(
weight_float16_tensor, platform::CPUPlace(), conv_weight_tensor);
}

{
framework::Tensor eltwise_y_in_float16_tensor;
eltwise_y_in_float16_tensor.set_type(
paddle::experimental::DataType::FLOAT16);
eltwise_y_in_float16_tensor.Resize(eltwise_y_in_tensor->dims());
auto* eltwise_y_in_float16_data =
eltwise_y_in_float16_tensor.mutable_data<float16>(
platform::CPUPlace());
auto* data =
eltwise_y_in_tensor->mutable_data<float>(platform::CPUPlace());
for (int i = 0; i < eltwise_y_in_tensor->numel(); i++) {
eltwise_y_in_float16_data[i] = static_cast<float16>(data[i]);
}
eltwise_y_in_tensor->clear();
paddle::framework::TensorCopySync(eltwise_y_in_float16_tensor,
platform::CPUPlace(),
eltwise_y_in_tensor);
}
}

// with MKL-DNN fuse conv+bn into conv with bias
// without MKL-DNN fuse conv+bn into conv+elementwise_add
if (fuse_option == FUSE_MKLDNN) {
Expand Down
9 changes: 6 additions & 3 deletions paddle/fluid/inference/api/paddle_pass_builder.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,9 +154,12 @@ const std::vector<std::string> kLiteSubgraphPasses({
// support fp16/bf16 precision, temporarily use low precision pass to prevent
// running errors. After fusion operator supports low precision, delete this.
const std::vector<std::string> kGpuLowerPrecisionPasses{
// "conv_bn_fuse_pass",
// "conv_eltwiseadd_bn_fuse_pass",
};
"conv_bn_fuse_pass",
"conv_eltwiseadd_bn_fuse_pass",
"conv_elementwise_add_act_fuse_pass",
"conv_elementwise_add2_act_fuse_pass",
"conv_elementwise_add_fuse_pass"};

const std::vector<std::string> kTrtLowerPrecisionPasses{
// "conv_bn_fuse_pass",
// "conv_eltwiseadd_bn_fuse_pass",
Expand Down