xft will be blocked when MPI + QWEN14B + do_sample=true

[a3213105]

using cmd below and run several times generate, then the xft will be blocked in oneccl.

OMP_NUM_THREADS=48 LD_PRELOAD=libiomp5.so mpirun -n 1 numactl --physcpubind 16-63 --localalloc python demo.py -t /mnt/data/LLM_Models/Qwen-14B-Chat/ -m /mnt/data/LLM_Models/Qwen-14B-Chat/cpu/ --output_len 512 --dtype bf16_fp16 --do_sample true : -n 1 numactl --physcpubind 80-127 --localalloc python demo.py -t /mnt/data/LLM_Models/Qwen-14B-Chat/ -m /mnt/data/LLM_Models/Qwen-14B-Chat/cpu/ --output_len 512 --dtype bf16_fp16 --do_sample true

[a3213105]

(gdb) where
#0 0x00007fe90471e292 in void ShmReduction::reduceAdd(float*, float*, unsigned long, int, int) ()
from /home/sgui/xFasterTransformer/src/xfastertransformer/libxfastertransformer_pt.so
#1 0x00007fe9046b9b0d in CommonDecoder<Attention<float16_t, QwenRotaryEmbedding, xft::RmsNorm, float, float, float, true>, LlamaMLP<float16_t, float, float, float>, float, false>::forward(int*, long*, int, bool) () from /home/sgui/xFasterTransformer/src/xfastertransformer/libxfastertransformer_pt.so
#2 0x00007fe9045d0d20 in HybridModel<Qwen, bfloat16_t, float16_t>::forward(int*, long*, int, bool) ()
from /home/sgui/xFasterTransformer/src/xfastertransformer/libxfastertransformer_pt.so
#3 0x00007fe90471c2ef in SampleSearch::getNextToken() () from /home/sgui/xFasterTransformer/src/xfastertransformer/libxfastertransformer_pt.so
#4 0x00007fe9045920e1 in xft::Model::generate() () from /home/sgui/xFasterTransformer/src/xfastertransformer/libxfastertransformer_pt.so
#5 0x00007fe904583269 in TorchAutoModel::generate() () from /home/sgui/xFasterTransformer/src/xfastertransformer/libxfastertransformer_pt.so
#6 0x00007fe904589e87 in std::Function_handler<void (std::vector<c10::IValue, std::allocatorc10::IValue >&), torch::class::defineMethod<torch::detail::WrapMethod<at::Tensor (TorchAutoModel::)()> >(std::string, torch::detail::WrapMethod<at::Tensor (TorchAutoModel::)()>, std::string, std::initializer_listtorch::arg)::{lambda(std::vector<c10::IValue, std::allocatorc10::IValue >&)#1}>::_M_invoke(std::_Any_data const&, std::vector<c10::IValue, std::allocatorc10::IValue >&) () from /home/sgui/xFasterTransformer/src/xfastertransformer/libxfastertransformer_pt.so
#7 0x00007fe984cafaae in torch::jit::runAndInsertCall(torch::jit::Function&, torch::jit::tuple_slice const&, pybind11::kwargs const&, c10::optionalc10::IValue, std::function<torch::jit::Value* (torch::jit::Graph&, torch::jit::MatchedSchema const&)> const&) ()
from /home/sgui/miniconda3/lib/python3.10/site-packages/torch/lib/libtorch_python.so
#8 0x00007fe984d81dd3 in torch::jit::invokeScriptMethodFromPython(torch::jit::Method&, torch::jit::tuple_slice const&, pybind11::kwargs const&) ()
from /home/sgui/miniconda3/lib/python3.10/site-packages/torch/lib/libtorch_python.so
#9 0x00007fe984d43e31 in pybind11::cpp_function::initialize<torch::jit::initJitScriptBindings(_object*)::{lambda(pybind11::args, pybind11::kwargs)#68}, pybind11::object, pybind11::args, pybind11::kwargs, pybind11::name, pybind11::is_method, pybind11::sibling>(torch::jit::initJitScriptBindings(_object*)::{lambda(pybind11::args, pybind11::kwargs)#68}&&, pybind11::object ()(pybind11::args, pybind11::kwargs), pybind11::name const&, pybind11::is_method const&, pybind11::sibling const&)::{lambda(pybind11::detail::function_call&)#3}::_FUN(pybind11::detail::function_call&) ()
from /home/sgui/miniconda3/lib/python3.10/site-packages/torch/lib/libtorch_python.so
#10 0x00007fe9847b7285 in pybind11::cpp_function::dispatcher(_object, _object*, _object*) ()
from /home/sgui/miniconda3/lib/python3.10/site-packages/torch/lib/libtorch_python.so
#11 0x00000000004fee27 in cfunction_call (func=0x7fe944d42ac0, args=, kwargs=)
at /usr/local/src/conda/python-3.10.9/Objects/methodobject.c:543
#12 0x00000000004f886b in _PyObject_MakeTpCall (tstate=0x1c7a170, callable=0x7fe944d42ac0, args=, nargs=, keywords=0x0)
at /usr/local/src/conda/python-3.10.9/Objects/call.c:215
#13 0x000000000050abda in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=1, args=0x7ffd158fd038, callable=0x7fe944d42ac0, tstate=0x1c7a170)
at /usr/local/src/conda/python-3.10.9/Include/cpython/abstract.h:112
#14 _PyObject_VectorcallTstate (kwnames=0x0, nargsf=1, args=0x7ffd158fd038, callable=0x7fe944d42ac0, tstate=0x1c7a170)
at /usr/local/src/conda/python-3.10.9/Include/cpython/abstract.h:99
#15 method_vectorcall (method=, args=0x7fe9bb704088, nargsf=, kwnames=)
at /usr/local/src/conda/python-3.10.9/Objects/classobject.c:61
#16 0x0000000000467e1c in slot_tp_call (self=0x7fe911df8db0, args=0x7fe9bb704070, kwds=0x0) at /usr/local/src/conda/python-3.10.9/Objects/typeobject.c:7497
#17 0x00000000004f886b in _PyObject_MakeTpCall (tstate=0x1c7a170, callable=0x7fe911df8db0, args=, nargs=, keywords=0x0)
at /usr/local/src/conda/python-3.10.9/Objects/call.c:215
#18 0x00000000004f4a36 in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=, args=0x7fe910d85158, callable=0x7fe911df8db0,

[a3213105]

it seems hanging in void ShmReduction::reduceAdd(float*, float*, unsigned long, int, int) ()

[a3213105]

another MPI instance stack

#0 0x00007f64e933fd01 in ccl_yield(ccl_yield_type) ()
from /opt/intel/oneapi/ccl/latest/lib/cpu_gpu_dpcpp/libccl.so.1
#1 0x00007f64e934e620 in ccl_executor::wait(ccl_request const*) ()
from /opt/intel/oneapi/ccl/latest/lib/cpu_gpu_dpcpp/libccl.so.1
#2 0x00007f64e930aea7 in ccl::host_event_impl::wait() ()
from /opt/intel/oneapi/ccl/latest/lib/cpu_gpu_dpcpp/libccl.so.1
#3 0x00007f656142c9b6 in broadcast ()
from /home/sgui/xFasterTransformer/src/xfastertransformer/libxft_comm_helper.so

[Duyi-Wang]

Fixed by #260