This repository has been archived by the owner on Nov 17, 2023. It is now read-only.
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.
Closed
Description
Description
During backporting change #18602 to branch v1.x, CI detected problem similar to original issue (#18569).
Backward propagation of softmax and log_softmax causes MXNetError: Check failed: delay_alloc:
Error Message
Traceback (most recent call last):
File "r.py", line 10, in <module>
a.grad.wait_to_read()
File "/home/wihajster/Desktop/incubator-mxnet/python/mxnet/ndarray/ndarray.py", line 2371, in wait_to_read
check_call(_LIB.MXNDArrayWaitToRead(self.handle))
File "/home/wihajster/Desktop/incubator-mxnet/python/mxnet/base.py", line 246, in check_call
raise get_last_ffi_error()
mxnet.base.MXNetError: Traceback (most recent call last):
[bt] (9) /home/wihajster/Desktop/incubator-mxnet/python/mxnet/../../lib/libmxnet.so(std::function<void (nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&)>::operator()(nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&) const+0xa6) [0x7f6d58049c9a]
[bt] (8) /home/wihajster/Desktop/incubator-mxnet/python/mxnet/../../lib/libmxnet.so(std::_Function_handler<void (nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&), void (*)(nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&)>::_M_invoke(std::_Any_data const&, nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&)+0x91) [0x7f6d56fc8a7d]
[bt] (7) /home/wihajster/Desktop/incubator-mxnet/python/mxnet/../../lib/libmxnet.so(+0xbe611e0) [0x7f6d609281e0]
[bt] (6) /home/wihajster/Desktop/incubator-mxnet/python/mxnet/../../lib/libmxnet.so(mxnet::MKLDNNRun(std::function<void (nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&)>, nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&)+0xad) [0x7f6d580426f3]
[bt] (5) /home/wihajster/Desktop/incubator-mxnet/python/mxnet/../../lib/libmxnet.so(std::function<void (nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&)>::operator()(nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&) const+0xa6) [0x7f6d58049c9a]
[bt] (4) /home/wihajster/Desktop/incubator-mxnet/python/mxnet/../../lib/libmxnet.so(std::_Function_handler<void (nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&), void (*)(nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&)>::_M_invoke(std::_Any_data const&, nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&)+0x91) [0x7f6d56fc8a7d]
[bt] (3) /home/wihajster/Desktop/incubator-mxnet/python/mxnet/../../lib/libmxnet.so(mxnet::op::MKLDNNLogSoftmaxBackward(nnvm::NodeAttrs const&, mxnet::OpContext const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> > const&)+0x1b6) [0x7f6d580b7d37]
[bt] (2) /home/wihajster/Desktop/incubator-mxnet/python/mxnet/../../lib/libmxnet.so(mxnet::NDArray::GetMKLDNNData() const+0x43b) [0x7f6d61188a0f]
[bt] (1) /home/wihajster/Desktop/incubator-mxnet/python/mxnet/../../lib/libmxnet.so(mxnet::NDArray::Chunk::SetMKLMem(mxnet::TShape const&, int)+0x3c4) [0x7f6d61186dbe]
[bt] (0) /home/wihajster/Desktop/incubator-mxnet/python/mxnet/../../lib/libmxnet.so(dmlc::LogMessageFatal::~LogMessageFatal()+0x4d) [0x7f6d569b2acf]
File "src/ndarray/ndarray.cc", line 580
MXNetError: Check failed: delay_alloc:
To Reproduce
import mxnet as mx
from mxnet import npx
npx.set_np()
a = mx.np.array([]).reshape(2, 1, 0)
a.attach_grad()
with mx.autograd.record():
b = mx.npx.log_softmax(a)
b.backward()
a.grad.wait_to_read()
PR is on its way