Description
It is the bug introduced by #254 , fix later
Traceback (most recent call last):
File "/home/work/user-job-dir/V0003/tools/train.py", line 243, in
main(config)
File "/home/work/user-job-dir/V0003/tools/train.py", line 202, in main
dataset_sink_mode=cfg.train.dataset_sink_mode)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/mindspore/train/model.py", line 1051, in train
initial_epoch=initial_epoch)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/mindspore/train/model.py", line 98, in wrapper
func(self, *args, **kwargs)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/mindspore/train/model.py", line 618, in _train
self._train_process(epoch, train_dataset, list_callback, cb_params, initial_epoch, valid_infos)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/mindspore/train/model.py", line 935, in _train_process
list_callback.on_train_epoch_end(run_context)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/mindspore/train/callback/_callback.py", line 371, in on_train_epoch_end
cb.on_train_epoch_end(run_context)
File "/home/work/user-job-dir/V0003/mindocr/utils/callbacks.py", line 268, in on_train_epoch_end
train_loss = ms.ops.stack(self._losses).mean().asnumpy()
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/mindspore/ops/function/array_func.py", line 1209, in stack
return _stack(input_x)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/mindspore/ops/primitive.py", line 296, in call
return _run_op(self, self.name, args)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/mindspore/common/api.py", line 98, in wrapper
results = fn(*arg, **kwargs)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/mindspore/ops/primitive.py", line 733, in _run_op
output = real_run_op(obj, op_name, args)
RuntimeError: Single op compile failed, op: concat_d_3769627495608740356_2
except_msg: 2023-05-12 03:25:57.396075+00:00: Query except_msg:Traceback (most recent call last):
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/te_fusion/parallel_compilation.py", line 1643, in run
optional_input_mode=self._optional_input_mode)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/te_fusion/fusion_manager.py", line 1337, in build_single_op
compile_info = call_op()
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/te_fusion/fusion_manager.py", line 1315, in call_op
opfunc(*inputs, *outputs, *new_attrs, **kwargs)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/tbe/common/register/operation_func_mgr.py", line 98, in wrapper
return func(*args, **kwargs)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/tbe/common/utils/para_check.py", line 547, in _in_wrapper
return func(*args, **kwargs)
File "/usr/local/Ascend/ascend-toolkit/latest/opp/built-in/op_impl/ai_core/tbe/impl/dynamic/concat_d.py", line 183, in concat_d
concat_d_dsl(input_values, output_data, concat_dim, kernel_name)
File "/usr/local/Ascend/ascend-toolkit/latest/opp/built-in/op_impl/ai_core/tbe/impl/dynamic/concat_d.py", line 144, in concat_d_dsl
ins = classify([input_values], "concat", extra_params)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/tbe/dsl/api.py", line 1088, in classify
return shape_classifier.classify(ins, mode, extra_params)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/tbe/common/register/operation_func_mgr.py", line 177, in wrapper
return func(*args, **kwargs)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/tbe/common/register/operation_func_mgr.py", line 177, in wrapper
return func(*args, **kwargs)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/tbe/dsl/classifier/shape_classifier.py", line 83, in classify
return classifier_func(ins, extra_params)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/tbe/dsl/classifier/shape_classifier.py", line 98, in wrapper
return func(*args, **kwargs)
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/tbe/dsl/classifier/concat_classifier.py", line 50, in classify
return classifier.classify()
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/tbe/dsl/classifier/concat_classifier.py", line 79, in classify
self._check_inputs()
File "/usr/local/ma/python3.7/lib/python3.7/site-packages/tbe/dsl/classifier/concat_classifier.py", line 110, in _check_inputs
raise RuntimeError(dict_args, get_error_message(dict_args))
RuntimeError: ({'errCode': 'E90001', 'detailed_cause': 'input numbers error, input numbers must be greater 0 and less equal 63 , now, it is 112'}, 'Compile operator failed, cause: Parameters check failed, detailed information: input numbers error, input numbers must be greater 0 and less equal 63 , now, it is 112.')