From f0dbd136ae7b1a65364fedfca72844410553ec10 Mon Sep 17 00:00:00 2001 From: ziqi-jin <67993288+ziqi-jin@users.noreply.github.com> Date: Wed, 31 Aug 2022 22:39:15 +0800 Subject: [PATCH] Add PPMatting Support (#163) * first commit for yolov7 * pybind for yolov7 * CPP README.md * CPP README.md * modified yolov7.cc * README.md * python file modify * delete license in fastdeploy/ * repush the conflict part * README.md modified * README.md modified * file path modified * file path modified * file path modified * file path modified * file path modified * README modified * README modified * move some helpers to private * add examples for yolov7 * api.md modified * api.md modified * api.md modified * YOLOv7 * yolov7 release link * yolov7 release link * yolov7 release link * copyright * change some helpers to private * change variables to const and fix documents. * gitignore * Transfer some funtions to private member of class * Transfer some funtions to private member of class * Merge from develop (#9) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> * first commit for yolor * for merge * Develop (#11) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> * Yolor (#16) * Develop (#11) (#12) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> * Develop (#13) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> * documents * documents * documents * documents * documents * documents * documents * documents * documents * documents * documents * documents * Develop (#14) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: Jason <928090362@qq.com> * add is_dynamic for YOLO series (#22) * ppmat * first commit for ppmatting * keep VisMattingAlpha with background funtion * support ppmatting and swapbackground function * modify docs error * add photos for docs * add photos for docs * add photos for docs * modify docs * modify docs * modify docs * change remove_small_connected_area function, modify docs * fix compile error Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: Jason <928090362@qq.com> --- .gitignore | 2 +- csrc/fastdeploy/vision.h | 1 + .../vision/common/processors/limit_short.cc | 70 ++++++ .../vision/common/processors/limit_short.h | 44 ++++ .../common/processors/resize_to_int_mult.cc | 56 +++++ .../common/processors/resize_to_int_mult.h | 42 ++++ .../vision/common/processors/transform.h | 2 + .../vision/matting/matting_pybind.cc | 2 + .../vision/matting/ppmatting/ppmatting.cc | 200 ++++++++++++++++++ .../vision/matting/ppmatting/ppmatting.h | 55 +++++ .../matting/ppmatting/ppmatting_pybind.cc | 29 +++ .../vision/visualize/matting_alpha.cc | 46 +--- .../visualize/remove_small_connnected_area.cc | 72 +++++++ .../vision/visualize/swap_background.cc | 130 ++++++++++++ csrc/fastdeploy/vision/visualize/visualize.h | 10 + .../vision/visualize/visualize_pybind.cc | 29 +++ examples/vision/matting/README.md | 1 + examples/vision/matting/modnet/cpp/README.md | 21 +- examples/vision/matting/modnet/cpp/infer.cc | 61 ++++-- .../vision/matting/modnet/python/README.md | 18 +- .../vision/matting/modnet/python/infer.py | 21 +- examples/vision/matting/ppmatting/README.md | 37 ++++ .../matting/ppmatting/cpp/CMakeLists.txt | 14 ++ .../vision/matting/ppmatting/cpp/README.md | 90 ++++++++ .../vision/matting/ppmatting/cpp/infer.cc | 140 ++++++++++++ .../vision/matting/ppmatting/python/README.md | 77 +++++++ .../vision/matting/ppmatting/python/infer.py | 67 ++++++ .../segmentation/paddleseg/python/infer.py | 4 +- fastdeploy/vision/matting/__init__.py | 1 + .../vision/matting/ppmatting/__init__.py | 38 ++++ fastdeploy/vision/visualize/__init__.py | 24 +++ 31 files changed, 1316 insertions(+), 88 deletions(-) create mode 100644 csrc/fastdeploy/vision/common/processors/limit_short.cc create mode 100644 csrc/fastdeploy/vision/common/processors/limit_short.h create mode 100644 csrc/fastdeploy/vision/common/processors/resize_to_int_mult.cc create mode 100644 csrc/fastdeploy/vision/common/processors/resize_to_int_mult.h create mode 100644 csrc/fastdeploy/vision/matting/ppmatting/ppmatting.cc create mode 100644 csrc/fastdeploy/vision/matting/ppmatting/ppmatting.h create mode 100644 csrc/fastdeploy/vision/matting/ppmatting/ppmatting_pybind.cc create mode 100644 csrc/fastdeploy/vision/visualize/remove_small_connnected_area.cc create mode 100644 csrc/fastdeploy/vision/visualize/swap_background.cc create mode 100644 examples/vision/matting/ppmatting/README.md create mode 100644 examples/vision/matting/ppmatting/cpp/CMakeLists.txt create mode 100644 examples/vision/matting/ppmatting/cpp/README.md create mode 100644 examples/vision/matting/ppmatting/cpp/infer.cc create mode 100644 examples/vision/matting/ppmatting/python/README.md create mode 100644 examples/vision/matting/ppmatting/python/infer.py create mode 100644 fastdeploy/vision/matting/ppmatting/__init__.py diff --git a/.gitignore b/.gitignore index bb754dbd6a..d61f64c9c3 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,4 @@ fastdeploy/ThirdPartyNotices* *.so* fastdeploy/libs/third_libs csrc/fastdeploy/core/config.h -csrc/fastdeploy/pybind/main.cc \ No newline at end of file +csrc/fastdeploy/pybind/main.cc diff --git a/csrc/fastdeploy/vision.h b/csrc/fastdeploy/vision.h index 1b86bd48ea..c2bfedc87a 100644 --- a/csrc/fastdeploy/vision.h +++ b/csrc/fastdeploy/vision.h @@ -37,6 +37,7 @@ #include "fastdeploy/vision/faceid/contrib/partial_fc.h" #include "fastdeploy/vision/faceid/contrib/vpl.h" #include "fastdeploy/vision/matting/contrib/modnet.h" +#include "fastdeploy/vision/matting/ppmatting/ppmatting.h" #include "fastdeploy/vision/ocr/ppocr/classifier.h" #include "fastdeploy/vision/ocr/ppocr/dbdetector.h" #include "fastdeploy/vision/ocr/ppocr/ppocr_system_v2.h" diff --git a/csrc/fastdeploy/vision/common/processors/limit_short.cc b/csrc/fastdeploy/vision/common/processors/limit_short.cc new file mode 100644 index 0000000000..b08bb0ef58 --- /dev/null +++ b/csrc/fastdeploy/vision/common/processors/limit_short.cc @@ -0,0 +1,70 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/common/processors/limit_short.h" + +namespace fastdeploy { +namespace vision { + +bool LimitShort::CpuRun(Mat* mat) { + cv::Mat* im = mat->GetCpuMat(); + int origin_w = im->cols; + int origin_h = im->rows; + int im_size_min = std::min(origin_w, origin_h); + int target = im_size_min; + if (max_short_ > 0 && im_size_min > max_short_) { + target = max_short_; + } else if (min_short_ > 0 && im_size_min < min_short_) { + target = min_short_; + } + if (target != im_size_min) { + double scale = + static_cast(target) / static_cast(im_size_min); + cv::resize(*im, *im, cv::Size(), scale, scale, interp_); + mat->SetWidth(im->cols); + mat->SetHeight(im->rows); + } + return true; +} + +#ifdef ENABLE_OPENCV_CUDA +bool LimitShort::GpuRun(Mat* mat) { + cv::cuda::GpuMat* im = mat->GetGpuMat(); + int origin_w = im->cols; + int origin_h = im->rows; + im->convertTo(*im, CV_32FC(im->channels())); + int im_size_min = std::min(origin_w, origin_h); + int target = im_size_min; + if (max_short_ > 0 && im_size_min > max_short_) { + target = max_short_; + } else if (min_short_ > 0 && im_size_min < min_short_) { + target = min_short_; + } + if (target != im_size_min) { + double scale = + static_cast(target) / static_cast(im_size_min); + cv::cuda::resize(*im, *im, cv::Size(), scale, scale, interp_); + mat->SetWidth(im->cols); + mat->SetHeight(im->rows); + } + return true; +} +#endif + +bool LimitShort::Run(Mat* mat, int max_short, int min_short, ProcLib lib) { + auto l = LimitShort(max_short, min_short); + return l(mat, lib); +} +} // namespace vision +} // namespace fastdeploy diff --git a/csrc/fastdeploy/vision/common/processors/limit_short.h b/csrc/fastdeploy/vision/common/processors/limit_short.h new file mode 100644 index 0000000000..51a26a1ba6 --- /dev/null +++ b/csrc/fastdeploy/vision/common/processors/limit_short.h @@ -0,0 +1,44 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class LimitShort : public Processor { + public: + explicit LimitShort(int max_short = -1, int min_short = -1, int interp = 1) { + max_short_ = max_short; + min_short_ = min_short; + interp_ = interp; + } + bool CpuRun(Mat* mat); +#ifdef ENABLE_OPENCV_CUDA + bool GpuRun(Mat* mat); +#endif + std::string Name() { return "LimitShort"; } + + static bool Run(Mat* mat, int max_short = -1, int min_short = -1, + ProcLib lib = ProcLib::OPENCV_CPU); + + private: + int max_short_; + int min_short_; + int interp_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/csrc/fastdeploy/vision/common/processors/resize_to_int_mult.cc b/csrc/fastdeploy/vision/common/processors/resize_to_int_mult.cc new file mode 100644 index 0000000000..9659b101f0 --- /dev/null +++ b/csrc/fastdeploy/vision/common/processors/resize_to_int_mult.cc @@ -0,0 +1,56 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/common/processors/resize_to_int_mult.h" + +namespace fastdeploy { +namespace vision { + +bool ResizeToIntMult::CpuRun(Mat* mat) { + cv::Mat* im = mat->GetCpuMat(); + int origin_w = im->cols; + int origin_h = im->rows; + int rw = origin_w - origin_w % mult_int_; + int rh = origin_h - origin_h % mult_int_; + if (rw != origin_w || rh != origin_w) { + cv::resize(*im, *im, cv::Size(rw, rh), 0, 0, interp_); + mat->SetWidth(im->cols); + mat->SetHeight(im->rows); + } + return true; +} + +#ifdef ENABLE_OPENCV_CUDA +bool ResizeToIntMult::GpuRun(Mat* mat) { + cv::cuda::GpuMat* im = mat->GetGpuMat(); + int origin_w = im->cols; + int origin_h = im->rows; + im->convertTo(*im, CV_32FC(im->channels())); + int rw = origin_w - origin_w % mult_int_; + int rh = origin_h - origin_h % mult_int_; + if (rw != origin_w || rh != origin_w) { + cv::cuda::resize(*im, *im, cv::Size(rw, rh), 0, 0, interp_); + mat->SetWidth(im->cols); + mat->SetHeight(im->rows); + } + return true; +} +#endif + +bool ResizeToIntMult::Run(Mat* mat, int mult_int, int interp, ProcLib lib) { + auto r = ResizeToIntMult(mult_int, interp); + return r(mat, lib); +} +} // namespace vision +} // namespace fastdeploy diff --git a/csrc/fastdeploy/vision/common/processors/resize_to_int_mult.h b/csrc/fastdeploy/vision/common/processors/resize_to_int_mult.h new file mode 100644 index 0000000000..71a2aaa8d8 --- /dev/null +++ b/csrc/fastdeploy/vision/common/processors/resize_to_int_mult.h @@ -0,0 +1,42 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class ResizeToIntMult : public Processor { + public: + explicit ResizeToIntMult(int mult_int = 32, int interp = 1) { + mult_int_ = mult_int; + interp_ = interp; + } + bool CpuRun(Mat* mat); +#ifdef ENABLE_OPENCV_CUDA + bool GpuRun(Mat* mat); +#endif + std::string Name() { return "ResizeToIntMult"; } + + static bool Run(Mat* mat, int mult_int = 32, int interp = 1, + ProcLib lib = ProcLib::OPENCV_CPU); + + private: + int interp_; + int mult_int_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/csrc/fastdeploy/vision/common/processors/transform.h b/csrc/fastdeploy/vision/common/processors/transform.h index fed3d0c9a2..065006fbfb 100644 --- a/csrc/fastdeploy/vision/common/processors/transform.h +++ b/csrc/fastdeploy/vision/common/processors/transform.h @@ -19,9 +19,11 @@ #include "fastdeploy/vision/common/processors/color_space_convert.h" #include "fastdeploy/vision/common/processors/convert.h" #include "fastdeploy/vision/common/processors/hwc2chw.h" +#include "fastdeploy/vision/common/processors/limit_short.h" #include "fastdeploy/vision/common/processors/normalize.h" #include "fastdeploy/vision/common/processors/pad.h" #include "fastdeploy/vision/common/processors/pad_to_size.h" #include "fastdeploy/vision/common/processors/resize.h" #include "fastdeploy/vision/common/processors/resize_by_short.h" +#include "fastdeploy/vision/common/processors/resize_to_int_mult.h" #include "fastdeploy/vision/common/processors/stride_pad.h" diff --git a/csrc/fastdeploy/vision/matting/matting_pybind.cc b/csrc/fastdeploy/vision/matting/matting_pybind.cc index e5fd78925d..8c514a4286 100644 --- a/csrc/fastdeploy/vision/matting/matting_pybind.cc +++ b/csrc/fastdeploy/vision/matting/matting_pybind.cc @@ -17,10 +17,12 @@ namespace fastdeploy { void BindMODNet(pybind11::module& m); +void BindPPMatting(pybind11::module& m); void BindMatting(pybind11::module& m) { auto matting_module = m.def_submodule("matting", "Image object matting models."); BindMODNet(matting_module); + BindPPMatting(matting_module); } } // namespace fastdeploy diff --git a/csrc/fastdeploy/vision/matting/ppmatting/ppmatting.cc b/csrc/fastdeploy/vision/matting/ppmatting/ppmatting.cc new file mode 100644 index 0000000000..63a9eff584 --- /dev/null +++ b/csrc/fastdeploy/vision/matting/ppmatting/ppmatting.cc @@ -0,0 +1,200 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "fastdeploy/vision/matting/ppmatting/ppmatting.h" +#include "fastdeploy/vision.h" +#include "fastdeploy/vision/utils/utils.h" +#include "yaml-cpp/yaml.h" + +namespace fastdeploy { +namespace vision { +namespace matting { + +PPMatting::PPMatting(const std::string& model_file, + const std::string& params_file, + const std::string& config_file, + const RuntimeOption& custom_option, + const Frontend& model_format) { + config_file_ = config_file; + valid_cpu_backends = {Backend::PDINFER, Backend::ORT}; + valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; + runtime_option = custom_option; + runtime_option.model_format = model_format; + runtime_option.model_file = model_file; + runtime_option.params_file = params_file; + initialized = Initialize(); +} + +bool PPMatting::Initialize() { + if (!BuildPreprocessPipelineFromConfig()) { + FDERROR << "Failed to build preprocess pipeline from configuration file." + << std::endl; + return false; + } + if (!InitRuntime()) { + FDERROR << "Failed to initialize fastdeploy backend." << std::endl; + return false; + } + return true; +} + +bool PPMatting::BuildPreprocessPipelineFromConfig() { + processors_.clear(); + YAML::Node cfg; + processors_.push_back(std::make_shared()); + try { + cfg = YAML::LoadFile(config_file_); + } catch (YAML::BadFile& e) { + FDERROR << "Failed to load yaml file " << config_file_ + << ", maybe you should check this file." << std::endl; + return false; + } + + if (cfg["Deploy"]["transforms"]) { + auto preprocess_cfg = cfg["Deploy"]["transforms"]; + for (const auto& op : preprocess_cfg) { + FDASSERT(op.IsMap(), + "Require the transform information in yaml be Map type."); + if (op["type"].as() == "LimitShort") { + int max_short = -1; + int min_short = -1; + if (op["max_short"]) { + max_short = op["max_short"].as(); + } + if (op["min_short"]) { + min_short = op["min_short"].as(); + } + processors_.push_back( + std::make_shared(max_short, min_short)); + } else if (op["type"].as() == "ResizeToIntMult") { + int mult_int = 32; + if (op["mult_int"]) { + mult_int = op["mult_int"].as(); + } + processors_.push_back(std::make_shared(mult_int)); + } else if (op["type"].as() == "Normalize") { + std::vector mean = {0.5, 0.5, 0.5}; + std::vector std = {0.5, 0.5, 0.5}; + if (op["mean"]) { + mean = op["mean"].as>(); + } + if (op["std"]) { + std = op["std"].as>(); + } + processors_.push_back(std::make_shared(mean, std)); + } + } + processors_.push_back(std::make_shared()); + } + return true; +} + +bool PPMatting::Preprocess(Mat* mat, FDTensor* output, + std::map>* im_info) { + for (size_t i = 0; i < processors_.size(); ++i) { + if (!(*(processors_[i].get()))(mat)) { + FDERROR << "Failed to process image data in " << processors_[i]->Name() + << "." << std::endl; + return false; + } + } + + // Record output shape of preprocessed image + (*im_info)["output_shape"] = {static_cast(mat->Height()), + static_cast(mat->Width())}; + + mat->ShareWithTensor(output); + output->shape.insert(output->shape.begin(), 1); + output->name = InputInfoOfRuntime(0).name; + return true; +} + +bool PPMatting::Postprocess( + std::vector& infer_result, MattingResult* result, + const std::map>& im_info) { + FDASSERT((infer_result.size() == 1), + "The default number of output tensor must be 1 according to " + "modnet."); + FDTensor& alpha_tensor = infer_result.at(0); // (1,h,w,1) + FDASSERT((alpha_tensor.shape[0] == 1), "Only support batch =1 now."); + if (alpha_tensor.dtype != FDDataType::FP32) { + FDERROR << "Only support post process with float32 data." << std::endl; + return false; + } + + // 先获取alpha并resize (使用opencv) + auto iter_ipt = im_info.find("input_shape"); + auto iter_out = im_info.find("output_shape"); + FDASSERT(iter_out != im_info.end() && iter_ipt != im_info.end(), + "Cannot find input_shape or output_shape from im_info."); + int out_h = iter_out->second[0]; + int out_w = iter_out->second[1]; + int ipt_h = iter_ipt->second[0]; + int ipt_w = iter_ipt->second[1]; + + // TODO: 需要修改成FDTensor或Mat的运算 现在依赖cv::Mat + float* alpha_ptr = static_cast(alpha_tensor.Data()); + cv::Mat alpha_zero_copy_ref(out_h, out_w, CV_32FC1, alpha_ptr); + Mat alpha_resized(alpha_zero_copy_ref); // ref-only, zero copy. + if ((out_h != ipt_h) || (out_w != ipt_w)) { + // already allocated a new continuous memory after resize. + // cv::resize(alpha_resized, alpha_resized, cv::Size(ipt_w, ipt_h)); + Resize::Run(&alpha_resized, ipt_w, ipt_h, -1, -1); + } + + result->Clear(); + // note: must be setup shape before Resize + result->contain_foreground = false; + // 和输入原图大小对应的alpha + result->shape = {static_cast(ipt_h), static_cast(ipt_w)}; + int numel = ipt_h * ipt_w; + int nbytes = numel * sizeof(float); + result->Resize(numel); + std::memcpy(result->alpha.data(), alpha_resized.GetCpuMat()->data, nbytes); + return true; +} + +bool PPMatting::Predict(cv::Mat* im, MattingResult* result) { + Mat mat(*im); + std::vector processed_data(1); + + std::map> im_info; + + // Record the shape of image and the shape of preprocessed image + im_info["input_shape"] = {static_cast(mat.Height()), + static_cast(mat.Width())}; + im_info["output_shape"] = {static_cast(mat.Height()), + static_cast(mat.Width())}; + + if (!Preprocess(&mat, &(processed_data[0]), &im_info)) { + FDERROR << "Failed to preprocess input data while using model:" + << ModelName() << "." << std::endl; + return false; + } + std::vector infer_result(1); + if (!Infer(processed_data, &infer_result)) { + FDERROR << "Failed to inference while using model:" << ModelName() << "." + << std::endl; + return false; + } + if (!Postprocess(infer_result, result, im_info)) { + FDERROR << "Failed to postprocess while using model:" << ModelName() << "." + << std::endl; + return false; + } + return true; +} + +} // namespace matting +} // namespace vision +} // namespace fastdeploy diff --git a/csrc/fastdeploy/vision/matting/ppmatting/ppmatting.h b/csrc/fastdeploy/vision/matting/ppmatting/ppmatting.h new file mode 100644 index 0000000000..148fe10e81 --- /dev/null +++ b/csrc/fastdeploy/vision/matting/ppmatting/ppmatting.h @@ -0,0 +1,55 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once +#include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" + +namespace fastdeploy { +namespace vision { +namespace matting { + +class FASTDEPLOY_DECL PPMatting : public FastDeployModel { + public: + PPMatting(const std::string& model_file, const std::string& params_file, + const std::string& config_file, + const RuntimeOption& custom_option = RuntimeOption(), + const Frontend& model_format = Frontend::PADDLE); + + std::string ModelName() const { return "PaddleMat"; } + + virtual bool Predict(cv::Mat* im, MattingResult* result); + + bool with_softmax = false; + + bool is_vertical_screen = false; + + private: + bool Initialize(); + + bool BuildPreprocessPipelineFromConfig(); + + bool Preprocess(Mat* mat, FDTensor* outputs, + std::map>* im_info); + + bool Postprocess(std::vector& infer_result, MattingResult* result, + const std::map>& im_info); + + std::vector> processors_; + std::string config_file_; +}; + +} // namespace matting +} // namespace vision +} // namespace fastdeploy diff --git a/csrc/fastdeploy/vision/matting/ppmatting/ppmatting_pybind.cc b/csrc/fastdeploy/vision/matting/ppmatting/ppmatting_pybind.cc new file mode 100644 index 0000000000..16e524b56e --- /dev/null +++ b/csrc/fastdeploy/vision/matting/ppmatting/ppmatting_pybind.cc @@ -0,0 +1,29 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy { +void BindPPMatting(pybind11::module& m) { + pybind11::class_(m, "PPMatting") + .def(pybind11::init()) + .def("predict", + [](vision::matting::PPMatting& self, pybind11::array& data) { + auto mat = PyArrayToCvMat(data); + vision::MattingResult* res = new vision::MattingResult(); + self.Predict(&mat, res); + return res; + }); +} +} // namespace fastdeploy diff --git a/csrc/fastdeploy/vision/visualize/matting_alpha.cc b/csrc/fastdeploy/vision/visualize/matting_alpha.cc index 1018018c62..03e003224e 100644 --- a/csrc/fastdeploy/vision/visualize/matting_alpha.cc +++ b/csrc/fastdeploy/vision/visualize/matting_alpha.cc @@ -21,50 +21,6 @@ namespace fastdeploy { namespace vision { -static void RemoveSmallConnectedArea(cv::Mat* alpha_pred, - float threshold = 0.05f) { - // 移除小的联通区域和噪点 开闭合形态学处理 - // 假设输入的是透明度alpha, 值域(0.,1.) - cv::Mat gray, binary; - (*alpha_pred).convertTo(gray, CV_8UC1, 255.f); - // 255 * 0.05 ~ 13 - unsigned int binary_threshold = static_cast(255.f * threshold); - cv::threshold(gray, binary, binary_threshold, 255, cv::THRESH_BINARY); - // morphologyEx with OPEN operation to remove noise first. - auto kernel = cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(3, 3), - cv::Point(-1, -1)); - cv::morphologyEx(binary, binary, cv::MORPH_OPEN, kernel); - // Computationally connected domain - cv::Mat labels = cv::Mat::zeros((*alpha_pred).size(), CV_32S); - cv::Mat stats, centroids; - int num_labels = - cv::connectedComponentsWithStats(binary, labels, stats, centroids, 8, 4); - if (num_labels <= 1) { - // no noise, skip. - return; - } - // find max connected area, 0 is background - int max_connected_id = 1; // 1,2,... - int max_connected_area = stats.at(max_connected_id, cv::CC_STAT_AREA); - for (int i = 1; i < num_labels; ++i) { - int tmp_connected_area = stats.at(i, cv::CC_STAT_AREA); - if (tmp_connected_area > max_connected_area) { - max_connected_area = tmp_connected_area; - max_connected_id = i; - } - } - const int h = (*alpha_pred).rows; - const int w = (*alpha_pred).cols; - // remove small connected area. - for (int i = 0; i < h; ++i) { - int* label_row_ptr = labels.ptr(i); - float* alpha_row_ptr = (*alpha_pred).ptr(i); - for (int j = 0; j < w; ++j) { - if (label_row_ptr[j] != max_connected_id) alpha_row_ptr[j] = 0.f; - } - } -} - cv::Mat Visualize::VisMattingAlpha(const cv::Mat& im, const MattingResult& result, bool remove_small_connected_area) { @@ -83,7 +39,7 @@ cv::Mat Visualize::VisMattingAlpha(const cv::Mat& im, float* alpha_ptr = static_cast(alpha_copy.data()); cv::Mat alpha(out_h, out_w, CV_32FC1, alpha_ptr); if (remove_small_connected_area) { - RemoveSmallConnectedArea(&alpha, 0.05f); + alpha = RemoveSmallConnectedArea(alpha, 0.05f); } if ((out_h != height) || (out_w != width)) { cv::resize(alpha, alpha, cv::Size(width, height)); diff --git a/csrc/fastdeploy/vision/visualize/remove_small_connnected_area.cc b/csrc/fastdeploy/vision/visualize/remove_small_connnected_area.cc new file mode 100644 index 0000000000..c523c99870 --- /dev/null +++ b/csrc/fastdeploy/vision/visualize/remove_small_connnected_area.cc @@ -0,0 +1,72 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifdef ENABLE_VISION_VISUALIZE + +#include "fastdeploy/vision/visualize/visualize.h" +#include "opencv2/highgui.hpp" +#include "opencv2/imgproc/imgproc.hpp" + +namespace fastdeploy { +namespace vision { + +cv::Mat Visualize::RemoveSmallConnectedArea(const cv::Mat& alpha_pred, + float threshold) { + // 移除小的联通区域和噪点 开闭合形态学处理 + // 假设输入的是透明度alpha, 值域(0.,1.) + cv::Mat gray, binary; + alpha_pred.convertTo(gray, CV_8UC1, 255.f); + cv::Mat alpha_pred_clone = alpha_pred.clone(); + // 255 * 0.05 ~ 13 + unsigned int binary_threshold = static_cast(255.f * threshold); + cv::threshold(gray, binary, binary_threshold, 255, cv::THRESH_BINARY); + // morphologyEx with OPEN operation to remove noise first. + auto kernel = cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(3, 3), + cv::Point(-1, -1)); + cv::morphologyEx(binary, binary, cv::MORPH_OPEN, kernel); + // Computationally connected domain + cv::Mat labels = cv::Mat::zeros(alpha_pred_clone.size(), CV_32S); + cv::Mat stats, centroids; + int num_labels = + cv::connectedComponentsWithStats(binary, labels, stats, centroids, 8, 4); + if (num_labels <= 1) { + // no noise, skip. + return alpha_pred; + } + // find max connected area, 0 is background + int max_connected_id = 1; // 1,2,... + int max_connected_area = stats.at(max_connected_id, cv::CC_STAT_AREA); + for (int i = 1; i < num_labels; ++i) { + int tmp_connected_area = stats.at(i, cv::CC_STAT_AREA); + if (tmp_connected_area > max_connected_area) { + max_connected_area = tmp_connected_area; + max_connected_id = i; + } + } + const int h = alpha_pred_clone.rows; + const int w = alpha_pred_clone.cols; + // remove small connected area. + for (int i = 0; i < h; ++i) { + int* label_row_ptr = labels.ptr(i); + float* alpha_row_ptr = alpha_pred_clone.ptr(i); + for (int j = 0; j < w; ++j) { + if (label_row_ptr[j] != max_connected_id) alpha_row_ptr[j] = 0.f; + } + } + return alpha_pred_clone; +} + +} // namespace vision +} // namespace fastdeploy +#endif \ No newline at end of file diff --git a/csrc/fastdeploy/vision/visualize/swap_background.cc b/csrc/fastdeploy/vision/visualize/swap_background.cc new file mode 100644 index 0000000000..e8ad04b771 --- /dev/null +++ b/csrc/fastdeploy/vision/visualize/swap_background.cc @@ -0,0 +1,130 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifdef ENABLE_VISION_VISUALIZE + +#include "fastdeploy/vision/visualize/visualize.h" +#include "opencv2/highgui.hpp" +#include "opencv2/imgproc/imgproc.hpp" + +namespace fastdeploy { +namespace vision { + +cv::Mat Visualize::SwapBackgroundMatting(const cv::Mat& im, + const cv::Mat& background, + const MattingResult& result, + bool remove_small_connected_area) { + // 只可视化alpha，fgr(前景)本身就是一张图 不需要可视化 + FDASSERT((!im.empty()), "Image can't be empty!"); + FDASSERT((im.channels() == 3), "Only support 3 channels image mat!"); + FDASSERT((!background.empty()), "Background image can't be empty!"); + FDASSERT((background.channels() == 3), + "Only support 3 channels background image mat!"); + auto vis_img = im.clone(); + auto background_copy = background.clone(); + int out_h = static_cast(result.shape[0]); + int out_w = static_cast(result.shape[1]); + int height = im.rows; + int width = im.cols; + int bg_height = background.rows; + int bg_width = background.cols; + // alpha to cv::Mat && 避免resize等操作修改外部数据 + std::vector alpha_copy; + alpha_copy.assign(result.alpha.begin(), result.alpha.end()); + float* alpha_ptr = static_cast(alpha_copy.data()); + cv::Mat alpha(out_h, out_w, CV_32FC1, alpha_ptr); + if (remove_small_connected_area) { + alpha = Visualize::RemoveSmallConnectedArea(alpha, 0.05f); + } + if ((vis_img).type() != CV_8UC3) { + (vis_img).convertTo((vis_img), CV_8UC3); + } + if ((background_copy).type() != CV_8UC3) { + (background_copy).convertTo((background_copy), CV_8UC3); + } + if ((bg_height != height) || (bg_width != width)) { + cv::resize(background, background_copy, cv::Size(width, height)); + } + if ((out_h != height) || (out_w != width)) { + cv::resize(alpha, alpha, cv::Size(width, height)); + } + uchar* vis_data = static_cast(vis_img.data); + uchar* background_data = static_cast(background_copy.data); + uchar* im_data = static_cast(im.data); + float* alpha_data = reinterpret_cast(alpha.data); + for (size_t i = 0; i < height; ++i) { + for (size_t j = 0; j < width; ++j) { + float alpha_val = alpha_data[i * width + j]; + for (size_t c = 0; c < 3; ++c) { + vis_data[i * width * 3 + j * 3 + c] = cv::saturate_cast( + static_cast(im_data[i * width * 3 + j * 3 + c]) * alpha_val + + (1.f - alpha_val) * background_data[i * width * 3 + j * 3 + c]); + } + } + } + + return vis_img; +} +// 对SegmentationResult做背景替换，由于分割模型可以预测多个类别，其中 +// background_label 表示预测为背景类的标签 +// 由于不同模型和数据集训练的背景类别标签可能不同，用户可以自己输入背景类对应的标签。 +cv::Mat Visualize::SwapBackgroundSegmentation( + const cv::Mat& im, const cv::Mat& background, int background_label, + const SegmentationResult& result) { + FDASSERT((!im.empty()), "Image can't be empty!"); + FDASSERT((im.channels() == 3), "Only support 3 channels image mat!"); + FDASSERT((!background.empty()), "Background image can't be empty!"); + FDASSERT((background.channels() == 3), + "Only support 3 channels background image mat!"); + auto vis_img = im.clone(); + auto background_copy = background.clone(); + int height = im.rows; + int width = im.cols; + int bg_height = background.rows; + int bg_width = background.cols; + if ((vis_img).type() != CV_8UC3) { + (vis_img).convertTo((vis_img), CV_8UC3); + } + if ((background_copy).type() != CV_8UC3) { + (background_copy).convertTo((background_copy), CV_8UC3); + } + if ((bg_height != height) || (bg_width != width)) { + cv::resize(background, background_copy, cv::Size(width, height)); + } + uchar* vis_data = static_cast(vis_img.data); + uchar* background_data = static_cast(background_copy.data); + uchar* im_data = static_cast(im.data); + float keep_value = 0.f; + for (size_t i = 0; i < height; ++i) { + for (size_t j = 0; j < width; ++j) { + int category_id = result.label_map[i * width + j]; + if (background_label != category_id) { + keep_value = 1.0f; + } else { + keep_value = 0.f; + } + for (size_t c = 0; c < 3; ++c) { + vis_data[i * width * 3 + j * 3 + c] = cv::saturate_cast( + static_cast(im_data[i * width * 3 + j * 3 + c]) * + keep_value + + (1.f - keep_value) * background_data[i * width * 3 + j * 3 + c]); + } + } + } + return vis_img; +} + +} // namespace vision +} // namespace fastdeploy +#endif \ No newline at end of file diff --git a/csrc/fastdeploy/vision/visualize/visualize.h b/csrc/fastdeploy/vision/visualize/visualize.h index 89b3d0ea2f..25de7af011 100644 --- a/csrc/fastdeploy/vision/visualize/visualize.h +++ b/csrc/fastdeploy/vision/visualize/visualize.h @@ -35,7 +35,17 @@ class FASTDEPLOY_DECL Visualize { const SegmentationResult& result); static cv::Mat VisMattingAlpha(const cv::Mat& im, const MattingResult& result, bool remove_small_connected_area = false); + static cv::Mat RemoveSmallConnectedArea(const cv::Mat& alpha_pred, + float threshold); + static cv::Mat SwapBackgroundMatting( + const cv::Mat& im, const cv::Mat& background, const MattingResult& result, + bool remove_small_connected_area = false); + static cv::Mat SwapBackgroundSegmentation(const cv::Mat& im, + const cv::Mat& background, + int background_label, + const SegmentationResult& result); static cv::Mat VisOcr(const cv::Mat& srcimg, const OCRResult& ocr_result); + }; } // namespace vision diff --git a/csrc/fastdeploy/vision/visualize/visualize_pybind.cc b/csrc/fastdeploy/vision/visualize/visualize_pybind.cc index e8523f561a..bf6b124c8f 100644 --- a/csrc/fastdeploy/vision/visualize/visualize_pybind.cc +++ b/csrc/fastdeploy/vision/visualize/visualize_pybind.cc @@ -48,6 +48,35 @@ void BindVisualize(pybind11::module& m) { vision::Mat(vis_im).ShareWithTensor(&out); return TensorToPyArray(out); }) + .def_static( + "swap_background_matting", + [](pybind11::array& im_data, pybind11::array& background_data, + vision::MattingResult& result, bool remove_small_connected_area) { + cv::Mat im = PyArrayToCvMat(im_data); + cv::Mat background = PyArrayToCvMat(background_data); + auto vis_im = vision::Visualize::SwapBackgroundMatting( + im, background, result, remove_small_connected_area); + FDTensor out; + vision::Mat(vis_im).ShareWithTensor(&out); + return TensorToPyArray(out); + }) + .def_static("swap_background_segmentation", + [](pybind11::array& im_data, pybind11::array& background_data, + int background_label, vision::SegmentationResult& result) { + cv::Mat im = PyArrayToCvMat(im_data); + cv::Mat background = PyArrayToCvMat(background_data); + auto vis_im = vision::Visualize::SwapBackgroundSegmentation( + im, background, background_label, result); + FDTensor out; + vision::Mat(vis_im).ShareWithTensor(&out); + return TensorToPyArray(out); + }) + .def_static("remove_small_connected_area", + [](pybind11::array& alpha_pred_data, float threshold) { + cv::Mat alpha_pred = PyArrayToCvMat(alpha_pred_data); + auto vis_im = vision::Visualize::RemoveSmallConnectedArea( + alpha_pred, threshold); + }) .def_static("vis_ppocr", [](pybind11::array& im_data, vision::OCRResult& result) { auto im = PyArrayToCvMat(im_data); diff --git a/examples/vision/matting/README.md b/examples/vision/matting/README.md index 1076d14b45..fafc3b6e1b 100644 --- a/examples/vision/matting/README.md +++ b/examples/vision/matting/README.md @@ -5,3 +5,4 @@ FastDeploy目前支持如下抠图模型部署 | 模型 | 说明 | 模型格式 | 版本 | | :--- | :--- | :------- | :--- | | [ZHKKKe/MODNet](./modnet) | MODNet 系列模型 | ONNX | [CommitID:28165a4](https://github.com/ZHKKKe/MODNet/commit/28165a4) | +| [PaddleSeg/PPMatting](./ppmatting) | PPMatting 系列模型 | Paddle | [Release/2.6](https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.6/Matting) | diff --git a/examples/vision/matting/modnet/cpp/README.md b/examples/vision/matting/modnet/cpp/README.md index 838af56de3..5b94b13e64 100644 --- a/examples/vision/matting/modnet/cpp/README.md +++ b/examples/vision/matting/modnet/cpp/README.md @@ -10,31 +10,36 @@ 以Linux上CPU推理为例，在本目录执行如下命令即可完成编译测试 ```bash -mkdir build -cd build wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.2.0.tgz tar xvf fastdeploy-linux-x64-0.2.0.tgz + +mkdir build && cd build cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.2.0 make -j #下载官方转换好的MODNet模型文件和测试图片 wget https://bj.bcebos.com/paddlehub/fastdeploy/modnet_photographic_portrait_matting.onnx -wget https://raw.githubusercontent.com/DefTruth/lite.ai.toolkit/main/examples/lite/resources/test_lite_matting_input.jpg +wget https://bj.bcebos.com/paddlehub/fastdeploy/matting_input.jpg +wget https://bj.bcebos.com/paddlehub/fastdeploy/matting_bgr.jpg # CPU推理 -./infer_demo modnet_photographic_portrait_matting.onnx test_lite_matting_input.jpg 0 +./infer_demo modnet_photographic_portrait_matting.onnx matting_input.jpg matting_bgr.jpg 0 # GPU推理 -./infer_demo modnet_photographic_portrait_matting.onnx test_lite_matting_input.jpg 1 +./infer_demo modnet_photographic_portrait_matting.onnx matting_input.jpg matting_bgr.jpg 1 # GPU上TensorRT推理 -./infer_demo modnet_photographic_portrait_matting.onnx test_lite_matting_input.jpg 2 +./infer_demo modnet_photographic_portrait_matting.onnx matting_input.jpg matting_bgr.jpg 2 ``` 运行完成可视化结果如下图所示 - - +
+ + + + +
以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: - [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/compile/how_to_use_sdk_on_windows.md) diff --git a/examples/vision/matting/modnet/cpp/infer.cc b/examples/vision/matting/modnet/cpp/infer.cc index dbc0e38fa6..1bf53cdc53 100644 --- a/examples/vision/matting/modnet/cpp/infer.cc +++ b/examples/vision/matting/modnet/cpp/infer.cc @@ -14,7 +14,8 @@ #include "fastdeploy/vision.h" -void CpuInfer(const std::string& model_file, const std::string& image_file) { +void CpuInfer(const std::string& model_file, const std::string& image_file, + const std::string& background_file) { auto model = fastdeploy::vision::matting::MODNet(model_file); if (!model.Initialized()) { std::cerr << "Failed to initialize." << std::endl; @@ -24,6 +25,7 @@ void CpuInfer(const std::string& model_file, const std::string& image_file) { model.size = {256, 256}; auto im = cv::imread(image_file); auto im_bak = im.clone(); + cv::Mat bg = cv::imread(background_file); fastdeploy::vision::MattingResult res; if (!model.Predict(&im, &res)) { @@ -32,11 +34,17 @@ void CpuInfer(const std::string& model_file, const std::string& image_file) { } auto vis_im = fastdeploy::vision::Visualize::VisMattingAlpha(im_bak, res); - cv::imwrite("vis_result.jpg", vis_im); - std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; + auto vis_im_with_bg = + fastdeploy::vision::Visualize::SwapBackgroundMatting(im_bak, bg, res); + cv::imwrite("visualized_result.jpg", vis_im_with_bg); + cv::imwrite("visualized_result_fg.jpg", vis_im); + std::cout << "Visualized result save in ./visualized_result_replaced_bg.jpg " + "and ./visualized_result_fg.jpg" + << std::endl; } -void GpuInfer(const std::string& model_file, const std::string& image_file) { +void GpuInfer(const std::string& model_file, const std::string& image_file, + const std::string& background_file) { auto option = fastdeploy::RuntimeOption(); option.UseGpu(); auto model = fastdeploy::vision::matting::MODNet(model_file, "", option); @@ -49,6 +57,7 @@ void GpuInfer(const std::string& model_file, const std::string& image_file) { auto im = cv::imread(image_file); auto im_bak = im.clone(); + cv::Mat bg = cv::imread(background_file); fastdeploy::vision::MattingResult res; if (!model.Predict(&im, &res)) { @@ -57,11 +66,17 @@ void GpuInfer(const std::string& model_file, const std::string& image_file) { } auto vis_im = fastdeploy::vision::Visualize::VisMattingAlpha(im_bak, res); - cv::imwrite("vis_result.jpg", vis_im); - std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; + auto vis_im_with_bg = + fastdeploy::vision::Visualize::SwapBackgroundMatting(im_bak, bg, res); + cv::imwrite("visualized_result.jpg", vis_im_with_bg); + cv::imwrite("visualized_result_fg.jpg", vis_im); + std::cout << "Visualized result save in ./visualized_result_replaced_bg.jpg " + "and ./visualized_result_fg.jpg" + << std::endl; } -void TrtInfer(const std::string& model_file, const std::string& image_file) { +void TrtInfer(const std::string& model_file, const std::string& image_file, + const std::string& background_file) { auto option = fastdeploy::RuntimeOption(); option.UseGpu(); option.UseTrtBackend(); @@ -75,6 +90,7 @@ void TrtInfer(const std::string& model_file, const std::string& image_file) { model.size = {256, 256}; auto im = cv::imread(image_file); auto im_bak = im.clone(); + cv::Mat bg = cv::imread(background_file); fastdeploy::vision::MattingResult res; if (!model.Predict(&im, &res)) { @@ -83,27 +99,32 @@ void TrtInfer(const std::string& model_file, const std::string& image_file) { } auto vis_im = fastdeploy::vision::Visualize::VisMattingAlpha(im_bak, res); - cv::imwrite("vis_result.jpg", vis_im); - std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; + auto vis_im_with_bg = + fastdeploy::vision::Visualize::SwapBackgroundMatting(im_bak, bg, res); + cv::imwrite("visualized_result.jpg", vis_im_with_bg); + cv::imwrite("visualized_result_fg.jpg", vis_im); + std::cout << "Visualized result save in ./visualized_result_replaced_bg.jpg " + "and ./visualized_result_fg.jpg" + << std::endl; } int main(int argc, char* argv[]) { - if (argc < 4) { - std::cout << "Usage: infer_demo path/to/model path/to/image run_option, " - "e.g ./modnet_photographic_portrait_matting.onnx ./test.jpeg 0" - << std::endl; + if (argc < 5) { + std::cout + << "Usage: infer_demo path/to/model_dir path/to/image run_option, " + "e.g ./infer_model ./PP-Matting-512 ./test.jpg ./test_bg.jpg 0" + << std::endl; std::cout << "The data type of run_option is int, 0: run with cpu; 1: run " "with gpu; 2: run with gpu and use tensorrt backend." << std::endl; return -1; } - - if (std::atoi(argv[3]) == 0) { - CpuInfer(argv[1], argv[2]); - } else if (std::atoi(argv[3]) == 1) { - GpuInfer(argv[1], argv[2]); - } else if (std::atoi(argv[3]) == 2) { - TrtInfer(argv[1], argv[2]); + if (std::atoi(argv[4]) == 0) { + CpuInfer(argv[1], argv[2], argv[3]); + } else if (std::atoi(argv[4]) == 1) { + GpuInfer(argv[1], argv[2], argv[3]); + } else if (std::atoi(argv[4]) == 2) { + TrtInfer(argv[1], argv[2], argv[3]); } return 0; } diff --git a/examples/vision/matting/modnet/python/README.md b/examples/vision/matting/modnet/python/README.md index 0d441c400a..99abdc6828 100644 --- a/examples/vision/matting/modnet/python/README.md +++ b/examples/vision/matting/modnet/python/README.md @@ -14,21 +14,25 @@ cd examples/vision/matting/modnet/python/ #下载modnet模型文件和测试图片 wget https://bj.bcebos.com/paddlehub/fastdeploy/modnet_photographic_portrait_matting.onnx -wget https://raw.githubusercontent.com/DefTruth/lite.ai.toolkit/main/examples/lite/resources/test_lite_matting_input.jpg +wget https://bj.bcebos.com/paddlehub/fastdeploy/matting_input.jpg +wget https://bj.bcebos.com/paddlehub/fastdeploy/matting_bgr.jpg # CPU推理 -python infer.py --model modnet_photographic_portrait_matting.onnx --image test_lite_matting_input.jpg --device cpu +python infer.py --model modnet_photographic_portrait_matting.onnx --image matting_input.jpg --bg matting_bgr.jpg --device cpu # GPU推理 -python infer.py --model modnet_photographic_portrait_matting.onnx --image test_lite_matting_input.jpg --device gpu +python infer.py --model modnet_photographic_portrait_matting.onnx --image matting_input.jpg --bg matting_bgr.jpg --device gpu # GPU上使用TensorRT推理 -python infer.py --model modnet_photographic_portrait_matting.onnx --image test_lite_matting_input.jpg --device gpu --use_trt True +python infer.py --model modnet_photographic_portrait_matting.onnx --image matting_input.jpg --bg matting_bgr.jpg --device gpu --use_trt True ``` 运行完成可视化结果如下图所示 - - - +
+ + + + +
## MODNet Python接口 ```python diff --git a/examples/vision/matting/modnet/python/infer.py b/examples/vision/matting/modnet/python/infer.py index 5980e1a1fc..408ba23408 100644 --- a/examples/vision/matting/modnet/python/infer.py +++ b/examples/vision/matting/modnet/python/infer.py @@ -15,6 +15,12 @@ def parse_arguments(): type=str, default='cpu', help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--bg", + type=str, + required=True, + default=None, + help="Path of test background image file.") parser.add_argument( "--use_trt", type=ast.literal_eval, @@ -43,11 +49,16 @@ def build_option(args): #设置推理size, 必须和模型文件一致 model.size = (256, 256) -# 预测图片检测结果 +# 预测图片抠图结果 im = cv2.imread(args.image) +bg = cv2.imread(args.bg) result = model.predict(im.copy()) - -# 预测结果可视化 +print(result) +# 可视化结果 vis_im = fd.vision.vis_matting_alpha(im, result) -cv2.imwrite("visualized_result.jpg", vis_im) -print("Visualized result save in ./visualized_result.jpg") +vis_im_with_bg = fd.vision.swap_background_matting(im, bg, result) +cv2.imwrite("visualized_result_fg.jpg", vis_im) +cv2.imwrite("visualized_result_replaced_bg.jpg", vis_im_with_bg) +print( + "Visualized result save in ./visualized_result_replaced_bg.jpg and ./visualized_result_fg.jpg" +) diff --git a/examples/vision/matting/ppmatting/README.md b/examples/vision/matting/ppmatting/README.md new file mode 100644 index 0000000000..0e12051150 --- /dev/null +++ b/examples/vision/matting/ppmatting/README.md @@ -0,0 +1,37 @@ +# PPMatting模型部署 + +## 模型版本说明 + +- [PPMatting Release/2.6](https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.6/Matting) + +## 支持模型列表 + +目前FastDeploy支持如下模型的部署 + +- [PPMatting系列模型](https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.6/Matting) + + +## 导出部署模型 + +在部署前，需要先将PPMatting导出成部署模型，导出步骤参考文档[导出模型](https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.6/Matting) + +注意：在导出模型时不要进行NMS的去除操作，正常导出即可。 + +## 下载预训练模型 + +为了方便开发者的测试，下面提供了PPMatting导出的各系列模型，开发者可直接下载使用。 + +其中精度指标来源于PPMatting中对各模型的介绍，详情各参考PPMatting中的说明。 + + +| 模型 | 参数大小 | 精度 | 备注 | +|:---------------------------------------------------------------- |:----- |:----- | :------ | +| [PPMatting](https://bj.bcebos.com/paddlehub/fastdeploy/PP-Matting-512.tgz) | 87MB | - | +| [PPMatting](https://bj.bcebos.com/paddlehub/fastdeploy/PP-Matting-1024.tgz) | 87MB | - | + + + +## 详细部署文档 + +- [Python部署](python) +- [C++部署](cpp) diff --git a/examples/vision/matting/ppmatting/cpp/CMakeLists.txt b/examples/vision/matting/ppmatting/cpp/CMakeLists.txt new file mode 100644 index 0000000000..fea1a2888b --- /dev/null +++ b/examples/vision/matting/ppmatting/cpp/CMakeLists.txt @@ -0,0 +1,14 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.12) + +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/matting/ppmatting/cpp/README.md b/examples/vision/matting/ppmatting/cpp/README.md new file mode 100644 index 0000000000..20ef29f7b3 --- /dev/null +++ b/examples/vision/matting/ppmatting/cpp/README.md @@ -0,0 +1,90 @@ +# PPMatting C++部署示例 + +本目录下提供`infer.cc`快速完成PPMatting在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/the%20software%20and%20hardware%20requirements.md) +- 2. 根据开发环境，下载预编译部署库和samples代码，参考[FastDeploy预编译库](../../../../../docs/quick_start) + +以Linux上 PPMatting 推理为例，在本目录执行如下命令即可完成编译测试（如若只需在CPU上部署，可在[Fastdeploy C++预编译库](../../../../../docs/quick_start/CPP_prebuilt_libraries.md)下载CPU推理库） + +```bash +#下载SDK，编译模型examples代码（SDK中包含了examples代码） +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.2.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.2.0.tgz +cd fastdeploy-linux-x64-gpu-0.2.0/examples/vision/matting/ppmatting/cpp/ +mkdir build && cd build +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.2.0 +make -j + +# 下载PPMatting模型文件和测试图片 +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP-Matting-512.tgz +tar -xvf PP-Matting-512.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/matting_input.jpg +wget https://bj.bcebos.com/paddlehub/fastdeploy/matting_bgr.jpg + + +# CPU推理 +./infer_demo PP-Matting-512 matting_input.jpg matting_bgr.jpg 0 +# GPU推理 (TODO: ORT-GPU 推理会报错) +./infer_demo PP-Matting-512 matting_input.jpg matting_bgr.jpg 1 +# GPU上TensorRT推理 +./infer_demo PP-Matting-512 matting_input.jpg matting_bgr.jpg 2 +``` + +运行完成可视化结果如下图所示 +
+ + + + +
+ +以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: +- [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/compile/how_to_use_sdk_on_windows.md) + +## PPMatting C++接口 + +### PPMatting类 + +```c++ +fastdeploy::vision::matting::PPMatting( + const string& model_file, + const string& params_file = "", + const string& config_file, + const RuntimeOption& runtime_option = RuntimeOption(), + const Frontend& model_format = Frontend::PADDLE) +``` + +PPMatting模型加载和初始化，其中model_file为导出的Paddle模型格式。 + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **config_file**(str): 推理部署配置文件 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(Frontend): 模型格式，默认为Paddle格式 + +#### Predict函数 + +> ```c++ +> PPMatting::Predict(cv::Mat* im, MattingResult* result) +> ``` +> +> 模型预测接口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **im**: 输入图像，注意需为HWC，BGR格式 +> > * **result**: 分割结果，包括分割预测的标签以及标签对应的概率值, MattingResult说明参考[视觉模型预测结果](../../../../../docs/api/vision_results/) + +### 类成员属性 +#### 预处理参数 +用户可按照自己的实际需求，修改下列预处理参数，从而影响最终的推理和部署效果 + + +- [模型介绍](../../) +- [Python部署](../python) +- [视觉模型预测结果](../../../../../docs/api/vision_results/) diff --git a/examples/vision/matting/ppmatting/cpp/infer.cc b/examples/vision/matting/ppmatting/cpp/infer.cc new file mode 100644 index 0000000000..bb8c6eac28 --- /dev/null +++ b/examples/vision/matting/ppmatting/cpp/infer.cc @@ -0,0 +1,140 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision.h" + +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void CpuInfer(const std::string& model_dir, const std::string& image_file, + const std::string& background_file) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto config_file = model_dir + sep + "deploy.yaml"; + auto model = fastdeploy::vision::matting::PPMatting(model_file, params_file, + config_file); + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + + auto im = cv::imread(image_file); + auto im_bak = im.clone(); + cv::Mat bg = cv::imread(background_file); + fastdeploy::vision::MattingResult res; + if (!model.Predict(&im, &res)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + auto vis_im = fastdeploy::vision::Visualize::VisMattingAlpha(im_bak, res); + auto vis_im_with_bg = + fastdeploy::vision::Visualize::SwapBackgroundMatting(im_bak, bg, res); + cv::imwrite("visualized_result.jpg", vis_im_with_bg); + cv::imwrite("visualized_result_fg.jpg", vis_im); + std::cout << "Visualized result save in ./visualized_result_replaced_bg.jpg " + "and ./visualized_result_fg.jpg" + << std::endl; +} + +void GpuInfer(const std::string& model_dir, const std::string& image_file, + const std::string& background_file) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto config_file = model_dir + sep + "deploy.yaml"; + + auto option = fastdeploy::RuntimeOption(); + option.UseGpu(); + auto model = fastdeploy::vision::matting::PPMatting(model_file, params_file, + config_file, option); + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + + auto im = cv::imread(image_file); + auto im_bak = im.clone(); + cv::Mat bg = cv::imread(background_file); + fastdeploy::vision::MattingResult res; + if (!model.Predict(&im, &res)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + auto vis_im = fastdeploy::vision::Visualize::VisMattingAlpha(im_bak, res); + auto vis_im_with_bg = + fastdeploy::vision::Visualize::SwapBackgroundMatting(im_bak, bg, res); + cv::imwrite("visualized_result.jpg", vis_im_with_bg); + cv::imwrite("visualized_result_fg.jpg", vis_im); + std::cout << "Visualized result save in ./visualized_result_replaced_bg.jpg " + "and ./visualized_result_fg.jpgg" + << std::endl; +} + +void TrtInfer(const std::string& model_dir, const std::string& image_file, + const std::string& background_file) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto config_file = model_dir + sep + "deploy.yaml"; + + auto option = fastdeploy::RuntimeOption(); + option.UseGpu(); + option.UseTrtBackend(); + option.SetTrtInputShape("img", {1, 3, 512, 512}); + auto model = fastdeploy::vision::matting::PPMatting(model_file, params_file, + config_file, option); + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + + auto im = cv::imread(image_file); + auto im_bak = im.clone(); + cv::Mat bg = cv::imread(background_file); + fastdeploy::vision::MattingResult res; + if (!model.Predict(&im, &res)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + auto vis_im = fastdeploy::vision::Visualize::VisMattingAlpha(im_bak, res); + auto vis_im_with_bg = + fastdeploy::vision::Visualize::SwapBackgroundMatting(im_bak, bg, res); + cv::imwrite("visualized_result.jpg", vis_im_with_bg); + cv::imwrite("visualized_result_fg.jpg", vis_im); + std::cout << "Visualized result save in ./visualized_result_replaced_bg.jpg " + "and ./visualized_result_fg.jpg" + << std::endl; +} + +int main(int argc, char* argv[]) { + if (argc < 5) { + std::cout + << "Usage: infer_demo path/to/model_dir path/to/image run_option, " + "e.g ./infer_model ./PP-Matting-512 ./test.jpg ./test_bg.jpg 0" + << std::endl; + std::cout << "The data type of run_option is int, 0: run with cpu; 1: run " + "with gpu; 2: run with gpu and use tensorrt backend." + << std::endl; + return -1; + } + if (std::atoi(argv[4]) == 0) { + CpuInfer(argv[1], argv[2], argv[3]); + } else if (std::atoi(argv[4]) == 1) { + GpuInfer(argv[1], argv[2], argv[3]); + } else if (std::atoi(argv[4]) == 2) { + TrtInfer(argv[1], argv[2], argv[3]); + } + return 0; +} diff --git a/examples/vision/matting/ppmatting/python/README.md b/examples/vision/matting/ppmatting/python/README.md new file mode 100644 index 0000000000..cd4ed7d2df --- /dev/null +++ b/examples/vision/matting/ppmatting/python/README.md @@ -0,0 +1,77 @@ +# PPMatting Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/the%20software%20and%20hardware%20requirements.md) +- 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../../../docs/quick_start) + +本目录下提供`infer.py`快速完成PPMatting在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。执行如下脚本即可完成 + +```bash +#下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/matting/ppmatting/python + +# 下载PPmatting模型文件和测试图片 +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP-Matting-512.tgz +tar -xvf PP-Matting-512.tgz +wget https://bj.bcebos.com/paddlehub/fastdeploy/matting_input.jpg +wget https://bj.bcebos.com/paddlehub/fastdeploy/matting_bgr.jpg +# CPU推理 +python infer.py --model PP-Matting-512 --image matting_input.jpg --bg matting_bgr.jpg --device cpu +# GPU推理 (TODO: ORT-GPU 推理会报错) +python infer.py --model PP-Matting-512 --image matting_input.jpg --bg matting_bgr.jpg --device gpu +# GPU上使用TensorRT推理 （注意：TensorRT推理第一次运行，有序列化模型的操作，有一定耗时，需要耐心等待） +python infer.py --model PP-Matting-512 --image matting_input.jpg --bg matting_bgr.jpg --device gpu --use_trt True +``` + +运行完成可视化结果如下图所示 +
+ + + + +
+## PPMatting Python接口 + +```python +fd.vision.matting.PPMatting(model_file, params_file, config_file, runtime_option=None, model_format=Frontend.PADDLE) +``` + +PPMatting模型加载和初始化，其中model_file, params_file以及config_file为训练模型导出的Paddle inference文件，具体请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.6/Matting) + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **config_file**(str): 推理部署配置文件 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(Frontend): 模型格式，默认为Paddle格式 + +### predict函数 + +> ```python +> PPMatting.predict(input_image) +> ``` +> +> 模型预测结口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **input_image**(np.ndarray): 输入数据，注意需为HWC，BGR格式 + +> **返回** +> +> > 返回`fastdeploy.vision.MattingResult`结构体，结构体说明参考文档[视觉模型预测结果](../../../../../docs/api/vision_results/) + +### 类成员属性 +#### 预处理参数 +用户可按照自己的实际需求，修改下列预处理参数，从而影响最终的推理和部署效果 + + + +## 其它文档 + +- [PPMatting 模型介绍](..) +- [PPMatting C++部署](../cpp) +- [模型预测结果说明](../../../../../docs/api/vision_results/) diff --git a/examples/vision/matting/ppmatting/python/infer.py b/examples/vision/matting/ppmatting/python/infer.py new file mode 100644 index 0000000000..4f5e7ffc4e --- /dev/null +++ b/examples/vision/matting/ppmatting/python/infer.py @@ -0,0 +1,67 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", required=True, help="Path of PaddleSeg model.") + parser.add_argument( + "--image", type=str, required=True, help="Path of test image file.") + parser.add_argument( + "--bg", + type=str, + required=True, + default=None, + help="Path of test background image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + + if args.device.lower() == "gpu": + option.use_gpu() + + if args.use_trt: + option.use_trt_backend() + option.set_trt_input_shape("img", [1, 3, 512, 512]) + return option + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model_file = os.path.join(args.model, "model.pdmodel") +params_file = os.path.join(args.model, "model.pdiparams") +config_file = os.path.join(args.model, "deploy.yaml") +model = fd.vision.matting.PPMatting( + model_file, params_file, config_file, runtime_option=runtime_option) + +# 预测图片抠图结果 +im = cv2.imread(args.image) +bg = cv2.imread(args.bg) +result = model.predict(im.copy()) +print(result) +# 可视化结果 +vis_im = fd.vision.vis_matting_alpha(im, result) +vis_im_with_bg = fd.vision.swap_background_matting(im, bg, result) +cv2.imwrite("visualized_result_fg.jpg", vis_im) +cv2.imwrite("visualized_result_replaced_bg.jpg", vis_im_with_bg) +print( + "Visualized result save in ./visualized_result_replaced_bg.jpg and ./visualized_result_fg.jpg" +) diff --git a/examples/vision/segmentation/paddleseg/python/infer.py b/examples/vision/segmentation/paddleseg/python/infer.py index 3877815910..dd65ebf273 100644 --- a/examples/vision/segmentation/paddleseg/python/infer.py +++ b/examples/vision/segmentation/paddleseg/python/infer.py @@ -47,9 +47,9 @@ def build_option(args): model = fd.vision.segmentation.PaddleSegModel( model_file, params_file, config_file, runtime_option=runtime_option) -# 预测图片分类结果 +# 预测图片分割结果 im = cv2.imread(args.image) -result = model.predict(im) +result = model.predict(im.copy()) print(result) # 可视化结果 diff --git a/fastdeploy/vision/matting/__init__.py b/fastdeploy/vision/matting/__init__.py index 1411461352..9a03d81e08 100644 --- a/fastdeploy/vision/matting/__init__.py +++ b/fastdeploy/vision/matting/__init__.py @@ -14,3 +14,4 @@ from __future__ import absolute_import from .contrib.modnet import MODNet +from .ppmatting import PPMatting diff --git a/fastdeploy/vision/matting/ppmatting/__init__.py b/fastdeploy/vision/matting/ppmatting/__init__.py new file mode 100644 index 0000000000..50e221589c --- /dev/null +++ b/fastdeploy/vision/matting/ppmatting/__init__.py @@ -0,0 +1,38 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +import logging +from .... import FastDeployModel, Frontend +from .... import c_lib_wrap as C + + +class PPMatting(FastDeployModel): + def __init__(self, + model_file, + params_file, + config_file, + runtime_option=None, + model_format=Frontend.PADDLE): + super(PPMatting, self).__init__(runtime_option) + + assert model_format == Frontend.PADDLE, "PPMatting model only support model format of Frontend.Paddle now." + self._model = C.vision.matting.PPMatting( + model_file, params_file, config_file, self._runtime_option, + model_format) + assert self.initialized, "PPMatting model initialize failed." + + def predict(self, input_image): + assert input_image is not None, "The input image data is None." + return self._model.predict(input_image) diff --git a/fastdeploy/vision/visualize/__init__.py b/fastdeploy/vision/visualize/__init__.py index 3fafe9df64..c74c9b6e9f 100644 --- a/fastdeploy/vision/visualize/__init__.py +++ b/fastdeploy/vision/visualize/__init__.py @@ -42,5 +42,29 @@ def vis_matting_alpha(im_data, remove_small_connected_area) +def swap_background_matting(im_data, + background, + result, + remove_small_connected_area=False): + assert isinstance( + result, + C.vision.MattingResult), "The result must be MattingResult type" + return C.vision.Visualize.swap_background_matting( + im_data, background, result, remove_small_connected_area) + + +def swap_background_segmentation(im_data, background, background_label, + result): + assert isinstance( + result, C.vision. + SegmentationResult), "The result must be SegmentaitonResult type" + return C.vision.Visualize.swap_background_segmentation( + im_data, background, background_label, result) + + +def remove_small_connected_area(alpha_pred_data, threshold): + assert len(alpha_pred_data.shape) == 3, "alpha has a (h, w, 1) shape" + return C.vision.Visualize.remove_small_connected_area(alpha_pred_data, + threshold) def vis_ppocr(im_data, det_result): return C.vision.Visualize.vis_ppocr(im_data, det_result)