Skip to content

Commit 6ed365d

Browse files
whybeyoungwhybeyoung
committed
fix: fix buffer length error when
1 parent 0d054c5 commit 6ed365d

File tree

3 files changed

+779
-108
lines changed

3 files changed

+779
-108
lines changed

‎pyWrapper.cpp

Lines changed: 62 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -19,58 +19,23 @@ std::map<std::string, const char *> SID_USRTAG;
1919

2020
PYBIND11_EMBEDDED_MODULE(aiges_embed, module) {
2121
module.def("callback", &callBack, py::return_value_policy::automatic_reference);
22-
py::class_<ResponseData> responseData(module, "ResponseData", py::buffer_protocol());
22+
py::class_<ResponseData> responseData(module, "ResponseData");
2323
responseData.def(py::init<>())
2424
.def(py::init<std::string, unsigned int, int, int>())
25-
.def(py::init([](const py::buffer &b) {
26-
py::buffer_info info = b.request();
27-
if (info.format != py::format_descriptor<unsigned char>::format() || info.ndim != 1) {
28-
throw std::runtime_error("Incompatible buffer format! Please Pass Bytes...");
29-
}
30-
auto *v = new ResponseData();
31-
v->len = info.shape[0];
32-
v->data = info.ptr;
33-
// memcpy( v->data, info.ptr, info.shape[0] );
34-
return (v);
35-
}))
36-
.def("setData", [](ResponseData &r, const py::buffer &b) {
37-
py::buffer_info info = b.request();
38-
if (info.format != py::format_descriptor<unsigned char>::format() || info.ndim != 1) {
39-
throw std::runtime_error("Incompatible buffer format! Please Pass Bytes..");
40-
}
41-
// if (r.data == nullptr) {
42-
// void *p = malloc(info.shape[0]);
43-
// if (p == nullptr) {
44-
// throw std::runtime_error("Can't Allocate memory!");
45-
// }
46-
// r.data = p;
47-
// }
48-
r.len = info.shape[0];
49-
r.data = info.ptr;
50-
// memcpy(r.data, info.ptr, info.shape[0]);
51-
})
52-
/* / Bare bones interface */
5325
.def("setDataType",
5426
[](ResponseData &r, int i) {
5527
r.type = i;
5628
})
57-
/* / Provide buffer access */
58-
.def_buffer([](ResponseData &r) -> py::buffer_info {
59-
return (py::buffer_info(
60-
r.data, /* Pointer to buffer */
61-
sizeof(unsigned char), /* Size of one scalar */
62-
py::format_descriptor<unsigned char>::format(), /* Python struct-style format descriptor */
63-
1, /* Number of dimensions */
64-
{size_t(r.len)}, /* Buffer dimensions */
65-
{ /* Strides (in bytes) for each index */
66-
sizeof(unsigned char)}
67-
));
29+
.def("setData", [](ResponseData &r, const py::bytes &b) {
30+
r.data = b;
31+
Py_ssize_t size = PyBytes_GET_SIZE(b.ptr());
32+
r.len = size;
33+
6834
})
6935
.def_readwrite("key", &ResponseData::key, py::return_value_policy::automatic_reference)
70-
.def_property_readonly("data",
71-
py::cpp_function(&ResponseData::get_data, py::return_value_policy::automatic_reference))
36+
.def_readwrite("data", &ResponseData::data, py::return_value_policy::automatic_reference)
7237
.def_readwrite("status", &ResponseData::status, py::return_value_policy::automatic_reference)
73-
.def_property_readonly("len", py::cpp_function(&ResponseData::get_len, py::return_value_policy::automatic_reference))
38+
.def_readwrite("len", &ResponseData::len, py::return_value_policy::automatic_reference)
7439
.def_readwrite("type", &ResponseData::type, py::return_value_policy::automatic_reference);
7540

7641
py::class_<Response> response(module, "Response");
@@ -136,33 +101,27 @@ DataListNode *DataListCls::get(std::string key) {
136101
PyWrapper::PyWrapper() {
137102
// 仅仅为了 加载下python lib库使 其部分函数可被导出使用
138103
// https://stackoverflow.com/questions/67891197/ctypes-cpython-39-x86-64-linux-gnu-so-undefined-symbol-pyfloat-type-in-embedd
139-
try {
140-
dlopen(PythonSo, RTLD_GLOBAL | RTLD_NOW);
104+
dlopen(PythonSo, RTLD_GLOBAL | RTLD_NOW);
141105

142-
// if (config.count(wrapperFileKey) == 0)
143-
py::gil_scoped_acquire acquire;
144-
_wrapper = py::module::import(WrapperFile);
145-
_obj = _wrapper.attr(WrapperClass)();
146-
_wrapper_abs = _wrapper.attr("__file__").cast<std::string>(); // 获取加载的wrapper.py的绝对地址
106+
// if (config.count(wrapperFileKey) == 0)
107+
py::gil_scoped_acquire acquire;
108+
_wrapper = py::module::import(WrapperFile);
109+
_obj = _wrapper.attr(WrapperClass)();
110+
_wrapper_abs = _wrapper.attr("__file__").cast<std::string>(); // 获取加载的wrapper.py的绝对地址
147111

148-
_wrapperInit = _obj.attr("wrapperInit");
149-
_wrapperFini = _obj.attr("wrapperFini");
150-
_wrapperOnceExec = _obj.attr("wrapperOnceExec");
151-
_wrapperOnceExecAsync = _obj.attr("wrapperOnceExecAsync");
152-
_wrapperError = _obj.attr("wrapperError");
153-
// stream support
154-
_wrapperCreate = _obj.attr("wrapperCreate");
155-
_wrapperWrite = _obj.attr("wrapperWrite");
156-
_wrapperRead = _obj.attr("wrapperRead");
157-
_wrapperTest = _obj.attr("wrapperTestFunc");
112+
_wrapperInit = _obj.attr("wrapperInit");
113+
_wrapperFini = _obj.attr("wrapperFini");
114+
_wrapperOnceExec = _obj.attr("wrapperOnceExec");
115+
_wrapperOnceExecAsync = _obj.attr("wrapperOnceExecAsync");
116+
_wrapperError = _obj.attr("wrapperError");
117+
// stream support
118+
_wrapperCreate = _obj.attr("wrapperCreate");
119+
_wrapperWrite = _obj.attr("wrapperWrite");
120+
_wrapperRead = _obj.attr("wrapperRead");
121+
_wrapperTest = _obj.attr("wrapperTestFunc");
158122

159-
py::gil_scoped_release release;
160-
StartMonitorWrapperClass(_wrapper_abs);
161-
} catch (py::error_already_set &e) {
162-
spdlog::get("stderr_console")->error("_wrapperInit error: {}", e.what());
163-
exit(-1);
164-
return ;
165-
}
123+
py::gil_scoped_release release;
124+
StartMonitorWrapperClass(_wrapper_abs);
166125

167126
}
168127

@@ -283,7 +242,7 @@ int PyWrapper::wrapperOnceExec(const char *usrTag, std::map <std::string, std::s
283242

284243
}
285244
SetSidUsrTag(sid, usrTag);
286-
params["sid"] = sid;
245+
287246
py::gil_scoped_acquire acquire;
288247
// 执行python exec 推理
289248
py::object r = _wrapperOnceExec(params, reqData);
@@ -315,20 +274,21 @@ int PyWrapper::wrapperOnceExec(const char *usrTag, std::map <std::string, std::s
315274
tmpData->type = DataType(itemData.type);
316275
tmpData->desc = nullptr;
317276
// 这里判断数据类型,todo 未来根据数据类型 决定是否拷贝，比如某些数据比较大，可以不拷贝
318-
// void *pr;
319-
// pr = malloc(itemData.len);
320-
// if (pr == nullptr) {
321-
// int ret = -1;
322-
// spdlog::get("stderr_console")->error("can't malloc memory for data, sid:{}", sid);
323-
// return ret;
324-
// }
325-
// ptr = PyBytes_AsString(itemData.data.ptr());
326-
// Py_ssize_t size = PyBytes_GET_SIZE(itemData.data.ptr());
327-
// printf("GetSIze data len: %d", itemData.len);
328-
// memcpy(pr, ptr, itemData.len);
277+
void *pr;
278+
pr = malloc(itemData.len);
279+
if (pr == nullptr) {
280+
int ret = -1;
281+
spdlog::get("stderr_console")->error("can't malloc memory for data, sid:{}", sid);
282+
return ret;
283+
}
284+
ptr = PyBytes_AsString(itemData.data.ptr());
285+
Py_ssize_t size = PyBytes_GET_SIZE(itemData.data.ptr());
286+
// printf("GetSIze, %d", size);
287+
// printf("item data len: %d", itemData.len);
288+
memcpy(pr, ptr, itemData.len);
329289
//char *data_ = new char[itemData.data.length()+1];
330290
// strdup(.c_str());
331-
tmpData->data = itemData.data;
291+
tmpData->data = pr;
332292
tmpData->status = DataStatus(itemData.status);
333293
if (idx == 0) {
334294
headPtr = tmpData;
@@ -366,7 +326,7 @@ int PyWrapper::wrapperOnceExecAsync(const char *usrTag, std::map <std::string, s
366326
}
367327
int ret = 0;
368328
SetSidUsrTag(sid, usrTag);
369-
params["sid"] = sid;
329+
370330
py::gil_scoped_acquire acquire;
371331
// 执行python exec 推理
372332
py::object r = _wrapperOnceExecAsync(params, reqData, sid);
@@ -488,18 +448,18 @@ int PyWrapper::wrapperRead(char *handle, pDataList *respData, std::string sid) {
488448
tmpData->len = itemData.len;
489449
tmpData->type = DataType(itemData.type);
490450
tmpData->desc = nullptr;
491-
// // 这里判断数据类型,todo 未来根据数据类型 决定是否拷贝，比如某些数据比较大，可以不拷贝
492-
// void *pr;
493-
// pr = malloc(itemData.len);
494-
// if (pr == nullptr) {
495-
// int ret = -1;
496-
// spdlog::get("stderr_console")->error("can't malloc memory for data, sid:{}", sid);
497-
// return ret;
498-
// }
499-
// memcpy(pr, (const void *) itemData.data.ptr(), itemData.len);
451+
// 这里判断数据类型,todo 未来根据数据类型 决定是否拷贝，比如某些数据比较大，可以不拷贝
452+
void *pr;
453+
pr = malloc(itemData.len);
454+
if (pr == nullptr) {
455+
int ret = -1;
456+
spdlog::get("stderr_console")->error("can't malloc memory for data, sid:{}", sid);
457+
return ret;
458+
}
459+
memcpy(pr, (const void *) itemData.data.ptr(), itemData.len);
500460
//char *data_ = new char[itemData.data.length()+1];
501461
// strdup(.c_str());
502-
tmpData->data = itemData.data;
462+
tmpData->data = pr;
503463
tmpData->status = DataStatus(itemData.status);
504464
if (idx == 0) {
505465
headPtr = tmpData;
@@ -608,20 +568,20 @@ int callBack(Response *resp, std::string sid) {
608568
tmpData->type = DataType(itemData.type);
609569
tmpData->desc = nullptr;
610570
// 这里判断数据类型,todo 未来根据数据类型 决定是否拷贝，比如某些数据比较大，可以不拷贝
611-
// void *pr;
612-
// pr = malloc(itemData.len);
613-
// if (pr == nullptr) {
614-
// int ret = -1;
615-
// spdlog::get("stderr_console")->error("can't malloc memory for data, sid:{}", sid);
616-
// return ret;
617-
// }
618-
// ptr = PyBytes_AsString(itemData.data.ptr());
619-
// Py_ssize_t size = PyBytes_GET_SIZE(itemData.data.ptr());
620-
// memcpy(pr, ptr, itemData.len);
571+
void *pr;
572+
pr = malloc(itemData.len);
573+
if (pr == nullptr) {
574+
int ret = -1;
575+
spdlog::get("stderr_console")->error("can't malloc memory for data, sid:{}", sid);
576+
return ret;
577+
}
578+
ptr = PyBytes_AsString(itemData.data.ptr());
579+
Py_ssize_t size = PyBytes_GET_SIZE(itemData.data.ptr());
580+
memcpy(pr, ptr, itemData.len);
621581
// 还是有问题：：memcpy(pr, (const void *) itemData.data.ptr(), itemData.len);
622582
//char *data_ = new char[itemData.data.length()+1];
623583
// strdup(.c_str());
624-
tmpData->data = itemData.data;
584+
tmpData->data = pr;
625585
tmpData->status = DataStatus(itemData.status);
626586
if (idx == 0) {
627587
headPtr = tmpData;

‎pyWrapper.h

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,11 @@ class ResponseData {
4747

4848
}
4949

50-
void *get_data() {
50+
py::bytes get_data() {
5151
return data;
5252
}
5353

54-
void set_data(void *data_) {
54+
void set_data(py::bytes data_) {
5555
data = data_;
5656
}
5757

@@ -64,7 +64,7 @@ class ResponseData {
6464
}
6565

6666
std::string key;
67-
void *data;
67+
py::bytes data;
6868
unsigned int len;
6969
int status;
7070
int type;
@@ -124,10 +124,8 @@ class PyWrapper {
124124
int wrapperOnceExec(const char *usrTag, std::map <std::string, std::string> params, DataListCls reqData,
125125
pDataList *respData,
126126
std::string sid, wrapperCallback cb);
127-
128127
int wrapperOnceExecAsync(const char *usrTag, std::map <std::string, std::string> params, DataListCls reqData,
129128
std::string sid, wrapperCallback cb);
130-
131129
int wrapperFini();
132130

133131
std::string
@@ -140,7 +138,7 @@ class PyWrapper {
140138

141139
int wrapperDestroy(std::string sid);
142140

143-
int wrapperExecFree(const char *usrTag);
141+
int wrapperExecFree(const char* usrTag);
144142

145143
int wrapperTest();
146144

0 commit comments

Comments
 (0)