Skip to content

Commit 2a2c83b

Browse files
Superjomnluotao1
Superjomn
authored and
luotao1
committed
feature/convert tensorrt io (#10440)
* init * init * add ut * split singleton from base class * add singleton * ad singleton
1 parent b708ec0 commit 2a2c83b

File tree

5 files changed

+250
-0
lines changed

5 files changed

+250
-0
lines changed

‎paddle/fluid/inference/tensorrt/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
nv_test(test_tensorrt SRCS test_tensorrt.cc DEPS dynload_cuda device_context dynamic_loader)
22
nv_test(test_tensorrt_engine SRCS test_engine.cc engine.cc DEPS dynload_cuda)
3+
nv_test(test_io_converter SRCS test_io_converter.cc io_converter.cc DEPS dynload_cuda dynamic_loader lod_tensor)
34
set(ENGINE_FILE ${CMAKE_CURRENT_SOURCE_DIR}/engine.cc)
45
add_subdirectory(convert)

‎paddle/fluid/inference/tensorrt/io_converter.cc‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. */
14+
15+
#include "paddle/fluid/inference/tensorrt/io_converter.h"
16+
#include <cuda.h>
17+
#include "paddle/fluid/platform/enforce.h"
18+
19+
namespace paddle {
20+
namespace inference {
21+
namespace tensorrt {
22+
23+
using platform::is_gpu_place;
24+
using platform::is_cpu_place;
25+
26+
class DefaultInputConverter : public EngineInputConverter {
27+
public:
28+
DefaultInputConverter() {}
29+
// NOTE out is GPU memory.
30+
virtual void operator()(const LoDTensor& in, void* out,
31+
size_t max_size) override {
32+
PADDLE_ENFORCE(out != nullptr);
33+
PADDLE_ENFORCE_LE(in.memory_size(), max_size);
34+
const auto& place = in.place();
35+
if (is_cpu_place(place)) {
36+
PADDLE_ENFORCE(stream_ != nullptr);
37+
PADDLE_ENFORCE_EQ(0,
38+
cudaMemcpyAsync(out, in.data<float>(), in.memory_size(),
39+
cudaMemcpyHostToDevice, *stream_));
40+
41+
} else if (is_gpu_place(place)) {
42+
PADDLE_ENFORCE_EQ(0,
43+
cudaMemcpyAsync(out, in.data<float>(), in.memory_size(),
44+
cudaMemcpyHostToHost, *stream_));
45+
46+
} else {
47+
PADDLE_THROW("Unknown device for converter");
48+
}
49+
cudaStreamSynchronize(*stream_);
50+
}
51+
};
52+
53+
REGISTER_TENSORRT_INPUT_CONVERTER(mul, DefaultInputConverter);
54+
55+
} // namespace tensorrt
56+
} // namespace inference
57+
} // namespace paddle

‎paddle/fluid/inference/tensorrt/io_converter.h‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. */
14+
15+
#pragma once
16+
17+
#include <unordered_map>
18+
#include "paddle/fluid/framework/lod_tensor.h"
19+
#include "paddle/fluid/inference/utils/singleton.h"
20+
21+
namespace paddle {
22+
namespace inference {
23+
namespace tensorrt {
24+
25+
using framework::LoDTensor;
26+
27+
/*
28+
* Convert Input from Fluid to an Engine.
29+
* TensorRT's ITensor follows row major, NCHW. Fluid is also row major, so in
30+
* most cases just need to copy the data.
31+
*/
32+
class EngineInputConverter {
33+
public:
34+
EngineInputConverter() {}
35+
36+
virtual void operator()(const LoDTensor& in, void* out, size_t max_size) {}
37+
38+
void SetStream(cudaStream_t* stream) { stream_ = stream; }
39+
40+
static void Run(const std::string& in_op_type, const LoDTensor& in, void* out,
41+
size_t max_size, cudaStream_t* stream) {
42+
PADDLE_ENFORCE(stream != nullptr);
43+
auto* converter = Registry<EngineInputConverter>::Lookup(in_op_type);
44+
PADDLE_ENFORCE_NOT_NULL(converter);
45+
converter->SetStream(stream);
46+
(*converter)(in, out, max_size);
47+
}
48+
49+
virtual ~EngineInputConverter() {}
50+
51+
protected:
52+
cudaStream_t* stream_{nullptr};
53+
};
54+
55+
} // namespace tensorrt
56+
} // namespace inference
57+
} // namespace paddle
58+
59+
#define REGISTER_TENSORRT_INPUT_CONVERTER(in_op_type__, Converter__) \
60+
struct trt_input_##in_op_type__##_converter { \
61+
trt_input_##in_op_type__##_converter() { \
62+
::paddle::inference::Registry<EngineInputConverter>::Register< \
63+
Converter__>(#in_op_type__); \
64+
} \
65+
}; \
66+
trt_input_##in_op_type__##_converter trt_input_##in_op_type__##_converter__;

‎paddle/fluid/inference/tensorrt/test_io_converter.cc‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. */
14+
15+
#include "paddle/fluid/framework/lod_tensor.h"
16+
#include "paddle/fluid/inference/tensorrt/io_converter.h"
17+
18+
#include <gtest/gtest.h>
19+
20+
namespace paddle {
21+
namespace inference {
22+
namespace tensorrt {
23+
24+
class EngineInputConverterTester : public ::testing::Test {
25+
public:
26+
void SetUp() override { tensor.Resize({10, 10}); }
27+
28+
framework::LoDTensor tensor;
29+
};
30+
31+
TEST_F(EngineInputConverterTester, DefaultCPU) {
32+
void* buffer;
33+
tensor.mutable_data<float>(platform::CPUPlace());
34+
ASSERT_EQ(cudaMalloc(&buffer, tensor.memory_size()), 0);
35+
36+
cudaStream_t stream;
37+
EngineInputConverter::Run("mul", tensor, buffer, tensor.memory_size(),
38+
&stream);
39+
}
40+
41+
TEST_F(EngineInputConverterTester, DefaultGPU) {
42+
void* buffer;
43+
tensor.mutable_data<float>(platform::CUDAPlace());
44+
ASSERT_EQ(cudaMalloc(&buffer, tensor.memory_size()), 0);
45+
46+
cudaStream_t stream;
47+
EngineInputConverter::Run("mul", tensor, buffer, tensor.memory_size(),
48+
&stream);
49+
}
50+
51+
} // namespace tensorrt
52+
} // namespace inference
53+
} // namespace paddle

‎paddle/fluid/inference/utils/singleton.h‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. */
14+
15+
#pragma once
16+
17+
#include <unordered_map>
18+
#include "paddle/fluid/platform/enforce.h"
19+
20+
namespace paddle {
21+
namespace inference {
22+
23+
// NOTE not thread-safe.
24+
template <typename T>
25+
struct Singleton {
26+
static T& Global() {
27+
static T* x = new T;
28+
return *x;
29+
}
30+
31+
Singleton() = delete;
32+
Singleton& operator=(const Singleton&) = delete;
33+
};
34+
35+
/*
36+
* An registor for any type.
37+
* NOTE not thread-safe.
38+
*/
39+
template <typename ItemParent>
40+
struct Registry {
41+
static Registry& Global() {
42+
static auto* x = new Registry<ItemParent>;
43+
return *x;
44+
}
45+
46+
template <typename ItemChild>
47+
static void Register(const std::string& name) {
48+
PADDLE_ENFORCE_EQ(items_.count(name), 0);
49+
items_[name] = new ItemChild;
50+
}
51+
52+
static ItemParent* Lookup(const std::string& name) {
53+
auto it = items_.find(name);
54+
if (it == items_.end()) return nullptr;
55+
return it->second;
56+
}
57+
58+
~Registry() {
59+
for (auto& item : items_) {
60+
delete item.second;
61+
}
62+
}
63+
64+
private:
65+
Registry() = default;
66+
static std::unordered_map<std::string, ItemParent*> items_;
67+
};
68+
69+
template <typename ItemParent>
70+
std::unordered_map<std::string, ItemParent*> Registry<ItemParent>::items_;
71+
72+
} // namespace inference
73+
} // namespace paddle

0 commit comments

Comments
 (0)