test_CSVIter

[leezu]

[2020-05-22T01:08:46.081Z] _________________________________ test_CSVIter _________________________________

[2020-05-22T01:08:46.081Z] [gw0] linux -- Python 3.6.9 /usr/bin/python3

[2020-05-22T01:08:46.081Z] 

[2020-05-22T01:08:46.081Z]     def test_CSVIter():

[2020-05-22T01:08:46.081Z]         def check_CSVIter_synthetic(dtype='float32'):

[2020-05-22T01:08:46.081Z]             cwd = os.getcwd()

[2020-05-22T01:08:46.081Z]             data_path = os.path.join(cwd, 'data.t')

[2020-05-22T01:08:46.081Z]             label_path = os.path.join(cwd, 'label.t')

[2020-05-22T01:08:46.081Z]             entry_str = '1'

[2020-05-22T01:08:46.081Z]             if dtype is 'int32':

[2020-05-22T01:08:46.081Z]                 entry_str = '200000001'

[2020-05-22T01:08:46.081Z]             if dtype is 'int64':

[2020-05-22T01:08:46.081Z]                 entry_str = '2147483648'

[2020-05-22T01:08:46.081Z]             with open(data_path, 'w') as fout:

[2020-05-22T01:08:46.081Z]                 for i in range(1000):

[2020-05-22T01:08:46.081Z]                     fout.write(','.join([entry_str for _ in range(8*8)]) + '\n')

[2020-05-22T01:08:46.081Z]             with open(label_path, 'w') as fout:

[2020-05-22T01:08:46.081Z]                 for i in range(1000):

[2020-05-22T01:08:46.081Z]                     fout.write('0\n')

[2020-05-22T01:08:46.081Z]     

[2020-05-22T01:08:46.081Z]             data_train = mx.io.CSVIter(data_csv=data_path, data_shape=(8, 8),

[2020-05-22T01:08:46.081Z]                                        label_csv=label_path, batch_size=100, dtype=dtype)

[2020-05-22T01:08:46.081Z]             expected = mx.nd.ones((100, 8, 8), dtype=dtype) * int(entry_str)

[2020-05-22T01:08:46.081Z]             for batch in iter(data_train):

[2020-05-22T01:08:46.081Z]                 data_batch = data_train.getdata()

[2020-05-22T01:08:46.081Z]                 assert_almost_equal(data_batch.asnumpy(), expected.asnumpy())

[2020-05-22T01:08:46.081Z]                 assert data_batch.asnumpy().dtype == expected.asnumpy().dtype

[2020-05-22T01:08:46.081Z]     

[2020-05-22T01:08:46.081Z]         for dtype in ['int32', 'int64', 'float32']:

[2020-05-22T01:08:46.081Z] >           check_CSVIter_synthetic(dtype=dtype)

[2020-05-22T01:08:46.081Z] 

[2020-05-22T01:08:46.081Z] tests/python/unittest/test_io.py:455: 

[2020-05-22T01:08:46.081Z] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

[2020-05-22T01:08:46.081Z] tests/python/unittest/test_io.py:447: in check_CSVIter_synthetic

[2020-05-22T01:08:46.081Z]     label_csv=label_path, batch_size=100, dtype=dtype)

[2020-05-22T01:08:46.081Z] python/mxnet/io/io.py:1004: in creator

[2020-05-22T01:08:46.081Z]     return MXDataIter(iter_handle, **kwargs)

[2020-05-22T01:08:46.081Z] python/mxnet/io/io.py:837: in __init__

[2020-05-22T01:08:46.081Z]     self.first_batch = self.next()

[2020-05-22T01:08:46.081Z] python/mxnet/io/io.py:871: in next

[2020-05-22T01:08:46.081Z]     check_call(_LIB.MXDataIterNext(self.handle, ctypes.byref(next_res)))

[2020-05-22T01:08:46.081Z] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

[2020-05-22T01:08:46.081Z] 

[2020-05-22T01:08:46.081Z] ret = -1

[2020-05-22T01:08:46.081Z] 

[2020-05-22T01:08:46.081Z]     def check_call(ret):

[2020-05-22T01:08:46.081Z]         """Check the return value of C API call.

[2020-05-22T01:08:46.081Z]     

[2020-05-22T01:08:46.081Z]         This function will raise an exception when an error occurs.

[2020-05-22T01:08:46.081Z]         Wrap every API call with this function.

[2020-05-22T01:08:46.081Z]     

[2020-05-22T01:08:46.081Z]         Parameters

[2020-05-22T01:08:46.081Z]         ----------

[2020-05-22T01:08:46.081Z]         ret : int

[2020-05-22T01:08:46.081Z]             return value from API calls.

[2020-05-22T01:08:46.081Z]         """

[2020-05-22T01:08:46.081Z]         if ret != 0:

[2020-05-22T01:08:46.081Z] >           raise get_last_ffi_error()

[2020-05-22T01:08:46.081Z] E           mxnet.base.MXNetError: Traceback (most recent call last):

[2020-05-22T01:08:46.081Z] E             [bt] (4) /work/mxnet/python/mxnet/../../build/libmxnet.so(MXDataIterNext+0x31) [0x7f36a8936ca1]

[2020-05-22T01:08:46.081Z] E             [bt] (3) /work/mxnet/python/mxnet/../../build/libmxnet.so(mxnet::io::PrefetcherIter::Next()+0x186) [0x7f36a8c7c896]

[2020-05-22T01:08:46.081Z] E             [bt] (2) /work/mxnet/python/mxnet/../../build/libmxnet.so(dmlc::ThreadedIter<mxnet::DataBatch>::Next(mxnet::DataBatch**)+0x2d6) [0x7f36a8c777b6]

[2020-05-22T01:08:46.081Z] E             [bt] (1) /work/mxnet/python/mxnet/../../build/libmxnet.so(dmlc::ThreadedIter<mxnet::DataBatch>::ThrowExceptionIfSet()+0x23f) [0x7f36a8c7746f]

[2020-05-22T01:08:46.081Z] E             [bt] (0) /work/mxnet/python/mxnet/../../build/libmxnet.so(dmlc::LogMessageFatal::~LogMessageFatal()+0x75) [0x7f36a880c785]

[2020-05-22T01:08:46.081Z] E             [bt] (9) /lib/x86_64-linux-gnu/libpthread.so.0(+0x76db) [0x7f375d13a6db]

[2020-05-22T01:08:46.081Z] E             [bt] (8) /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xbd6df) [0x7f374e4b86df]

[2020-05-22T01:08:46.081Z] E             [bt] (7) /work/mxnet/python/mxnet/../../build/libmxnet.so(std::thread::_State_impl<std::thread::_Invoker<std::tuple<dmlc::ThreadedIter<mxnet::DataBatch>::Init(std::function<bool (mxnet::DataBatch**)>, std::function<void ()>)::{lambda()#1}> > >::_M_run()+0x15) [0x7f36a8c7c6f5]

[2020-05-22T01:08:46.081Z] E             [bt] (6) /work/mxnet/python/mxnet/../../build/libmxnet.so(dmlc::ThreadedIter<mxnet::DataBatch>::Init(std::function<bool (mxnet::DataBatch**)>, std::function<void ()>)::{lambda()#1}::operator()() const+0x1c6) [0x7f36a8c7b906]

[2020-05-22T01:08:46.081Z] E             [bt] (5) /work/mxnet/python/mxnet/../../build/libmxnet.so(std::_Function_handler<bool (mxnet::DataBatch**), mxnet::io::PrefetcherIter::Init(std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&)::{lambda(mxnet::DataBatch**)#1}>::_M_invoke(std::_Any_data const&, mxnet::DataBatch**&&)+0x14) [0x7f36a8c7ac24]

[2020-05-22T01:08:46.081Z] E             [bt] (4) /work/mxnet/python/mxnet/../../build/libmxnet.so(mxnet::io::PrefetcherIter::Init(std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&)::{lambda(mxnet::DataBatch**)#1}::operator()(mxnet::DataBatch**) const+0x42) [0x7f36a8c792b2]

[2020-05-22T01:08:46.081Z] E             [bt] (3) /work/mxnet/python/mxnet/../../build/libmxnet.so(mxnet::io::BatchLoader::Next()+0x6b) [0x7f36a8ceff1b]

[2020-05-22T01:08:46.081Z] E             [bt] (2) /work/mxnet/python/mxnet/../../build/libmxnet.so(mxnet::io::CSVIter::Next()+0x19) [0x7f36a8ce7329]

[2020-05-22T01:08:46.081Z] E             [bt] (1) /work/mxnet/python/mxnet/../../build/libmxnet.so(mxnet::io::CSVIterTyped<int>::Next()+0x333) [0x7f36a8cf3b53]

[2020-05-22T01:08:46.081Z] E             [bt] (0) /work/mxnet/python/mxnet/../../build/libmxnet.so(dmlc::LogMessageFatal::~LogMessageFatal()+0x75) [0x7f36a880c785]

[2020-05-22T01:08:46.081Z] E             File "/work/mxnet/include/dmlc/threadediter.h", line 500

[2020-05-22T01:08:46.081Z] E           MXNetError: [00:55:19] /work/mxnet/src/io/iter_csv.cc:137: Check failed: label_parser_->Next(): Data CSV's row is smaller than the number of rows in label_csv

http://jenkins.mxnet-ci.amazon-ml.com/blue/rest/organizations/jenkins/pipelines/mxnet-validation/pipelines/unix-cpu/branches/PR-18376/runs/8/nodes/363/steps/746/log/?start=0

cc: @zhreshold

[leezu]

Also failed on CD http://jenkins.mxnet-ci.amazon-ml.com/blue/rest/organizations/jenkins/pipelines/restricted-mxnet-cd/pipelines/mxnet-cd-release-job/runs/1153/nodes/222/steps/248/log/?start=0

[leezu]

@zhreshold the test is still flaky

http://jenkins.mxnet-ci.amazon-ml.com/blue/rest/organizations/jenkins/pipelines/mxnet-validation/pipelines/centos-cpu/branches/master/runs/1979/nodes/236/steps/284/log/?start=0

[zhreshold]

@leezu The error is std: :bad_alloc indicating that it's oom?

[leezu]

@ChaiBapchya I looked at the CloudWatch metrics but couldn't find data on the memory usage. Should we activate https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/mon-scripts.html on the instances?

[leezu]

@zhreshold there seem to be more issues with the test

[2020-05-29T14:10:33.690Z] _________________________________ test_CSVIter _________________________________
[2020-05-29T14:10:33.690Z] [gw0] linux -- Python 3.6.9 /opt/rh/rh-python36/root/usr/bin/python
[2020-05-29T14:10:33.690Z] 
[2020-05-29T14:10:33.690Z] tmpdir = local('/tmp/pytest-of-jenkins_slave/pytest-0/popen-gw0/test_CSVIter0')
[2020-05-29T14:10:33.690Z] 
[2020-05-29T14:10:33.690Z]     def test_CSVIter(tmpdir):
[2020-05-29T14:10:33.690Z]         def check_CSVIter_synthetic(dtype='float32'):
[2020-05-29T14:10:33.690Z]             data_path = os.path.join(str(tmpdir), 'data.t')
[2020-05-29T14:10:33.690Z]             label_path = os.path.join(str(tmpdir), 'label.t')
[2020-05-29T14:10:33.690Z]             entry_str = '1'
[2020-05-29T14:10:33.690Z]             if dtype is 'int32':
[2020-05-29T14:10:33.690Z]                 entry_str = '200000001'
[2020-05-29T14:10:33.690Z]             if dtype is 'int64':
[2020-05-29T14:10:33.690Z]                 entry_str = '2147483648'
[2020-05-29T14:10:33.690Z]             with open(data_path, 'w') as fout:
[2020-05-29T14:10:33.690Z]                 for i in range(1000):
[2020-05-29T14:10:33.690Z]                     fout.write(','.join([entry_str for _ in range(8*8)]) + '\n')
[2020-05-29T14:10:33.690Z]             with open(label_path, 'w') as fout:
[2020-05-29T14:10:33.690Z]                 for i in range(1000):
[2020-05-29T14:10:33.690Z]                     fout.write('0\n')
[2020-05-29T14:10:33.690Z]     
[2020-05-29T14:10:33.690Z]             data_train = mx.io.CSVIter(data_csv=data_path, data_shape=(8, 8),
[2020-05-29T14:10:33.690Z]                                        label_csv=label_path, batch_size=100, dtype=dtype)
[2020-05-29T14:10:33.690Z]             expected = mx.nd.ones((100, 8, 8), dtype=dtype) * int(entry_str)
[2020-05-29T14:10:33.690Z]             for batch in iter(data_train):
[2020-05-29T14:10:33.690Z]                 data_batch = data_train.getdata()
[2020-05-29T14:10:33.690Z]                 assert_almost_equal(data_batch.asnumpy(), expected.asnumpy())
[2020-05-29T14:10:33.690Z]                 assert data_batch.asnumpy().dtype == expected.asnumpy().dtype
[2020-05-29T14:10:33.690Z]     
[2020-05-29T14:10:33.690Z]         for dtype in ['int32', 'int64', 'float32']:
[2020-05-29T14:10:33.690Z] >           check_CSVIter_synthetic(dtype=dtype)
[2020-05-29T14:10:33.690Z] 
[2020-05-29T14:10:33.690Z] tests/python/unittest/test_io.py:452: 
[2020-05-29T14:10:33.690Z] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
[2020-05-29T14:10:33.690Z] tests/python/unittest/test_io.py:448: in check_CSVIter_synthetic
[2020-05-29T14:10:33.690Z]     assert_almost_equal(data_batch.asnumpy(), expected.asnumpy())
[2020-05-29T14:10:33.690Z] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
[2020-05-29T14:10:33.690Z] 
[2020-05-29T14:10:33.690Z] a = array([[[2147483648,   47483648,   47483648, ...,   47483648,
[2020-05-29T14:10:33.690Z]            47483648,   47483648],
[2020-05-29T14:10:33.690Z]         [  47483648,  ...7483648,   47483648],
[2020-05-29T14:10:33.690Z]         [  47483648,   47483648,   47483648, ...,   47483648,
[2020-05-29T14:10:33.690Z]            47483648,   47483648]]])
[2020-05-29T14:10:33.690Z] b = array([[[2147483648, 2147483648, 2147483648, ..., 2147483648,
[2020-05-29T14:10:33.690Z]          2147483648, 2147483648],
[2020-05-29T14:10:33.690Z]         [2147483648, 2...7483648, 2147483648],
[2020-05-29T14:10:33.690Z]         [2147483648, 2147483648, 2147483648, ..., 2147483648,
[2020-05-29T14:10:33.690Z]          2147483648, 2147483648]]])
[2020-05-29T14:10:33.690Z] rtol = 1e-05, atol = 1e-20, names = ('a', 'b'), equal_nan = False
[2020-05-29T14:10:33.690Z] use_broadcast = True, mismatches = (10, 10)
[2020-05-29T14:10:33.690Z] 
[2020-05-29T14:10:33.690Z]     def assert_almost_equal(a, b, rtol=None, atol=None, names=('a', 'b'), equal_nan=False,
[2020-05-29T14:10:33.690Z]                             use_broadcast=True, mismatches=(10, 10)):
[2020-05-29T14:10:33.690Z]         """Test that two numpy arrays are almost equal. Raise exception message if not.
[2020-05-29T14:10:33.690Z]     
[2020-05-29T14:10:33.690Z]         Parameters
[2020-05-29T14:10:33.690Z]         ----------
[2020-05-29T14:10:33.690Z]         a : np.ndarray or mx.nd.array
[2020-05-29T14:10:33.690Z]         b : np.ndarray or mx.nd.array
[2020-05-29T14:10:33.690Z]         rtol : None or float
[2020-05-29T14:10:33.690Z]             The relative threshold. Default threshold will be used if set to ``None``.
[2020-05-29T14:10:33.690Z]         atol : None or float
[2020-05-29T14:10:33.690Z]             The absolute threshold. Default threshold will be used if set to ``None``.
[2020-05-29T14:10:33.690Z]         names : tuple of names, optional
[2020-05-29T14:10:33.690Z]             The names used in error message when an exception occurs
[2020-05-29T14:10:33.690Z]         equal_nan : boolean, optional
[2020-05-29T14:10:33.690Z]             The flag determining how to treat NAN values in comparison
[2020-05-29T14:10:33.690Z]         mismatches : tuple of mismatches
[2020-05-29T14:10:33.690Z]             Maximum number of mismatches to be printed (mismatches[0]) and determine (mismatches[1])
[2020-05-29T14:10:33.690Z]         """
[2020-05-29T14:10:33.690Z]         if not use_broadcast:
[2020-05-29T14:10:33.690Z]             checkShapes(a, b)
[2020-05-29T14:10:33.690Z]     
[2020-05-29T14:10:33.690Z]         rtol = get_rtol(rtol)
[2020-05-29T14:10:33.690Z]         atol = get_atol(atol)
[2020-05-29T14:10:33.690Z]         use_np_allclose = isinstance(a, np.ndarray) and isinstance(b, np.ndarray)
[2020-05-29T14:10:33.690Z]         if not use_np_allclose:
[2020-05-29T14:10:33.690Z]             if not (hasattr(a, 'ctx') and hasattr(b, 'ctx') and a.ctx == b.ctx and a.dtype == b.dtype):
[2020-05-29T14:10:33.690Z]                 use_np_allclose = True
[2020-05-29T14:10:33.690Z]                 if isinstance(a, mx.nd.NDArray):
[2020-05-29T14:10:33.690Z]                     a = a.asnumpy()
[2020-05-29T14:10:33.690Z]                 if isinstance(b, mx.nd.NDArray):
[2020-05-29T14:10:33.690Z]                     b = b.asnumpy()
[2020-05-29T14:10:33.690Z]     
[2020-05-29T14:10:33.690Z]         if use_np_allclose:
[2020-05-29T14:10:33.690Z]             if hasattr(a, 'dtype') and a.dtype == np.bool_ and hasattr(b, 'dtype') and b.dtype == np.bool_:
[2020-05-29T14:10:33.690Z]                 np.testing.assert_equal(a, b)
[2020-05-29T14:10:33.690Z]                 return
[2020-05-29T14:10:33.690Z]             if almost_equal(a, b, rtol, atol, equal_nan=equal_nan):
[2020-05-29T14:10:33.690Z]                 return
[2020-05-29T14:10:33.690Z]         else:
[2020-05-29T14:10:33.690Z]             output = mx.nd.contrib.allclose(a, b, rtol, atol, equal_nan)
[2020-05-29T14:10:33.690Z]             if output.asnumpy() == 1:
[2020-05-29T14:10:33.690Z]                 return
[2020-05-29T14:10:33.690Z]     
[2020-05-29T14:10:33.690Z]             a = a.asnumpy()
[2020-05-29T14:10:33.690Z]             b = b.asnumpy()
[2020-05-29T14:10:33.690Z]     
[2020-05-29T14:10:33.690Z]         index, rel = find_max_violation(a, b, rtol, atol)
[2020-05-29T14:10:33.690Z]         indexErr = index
[2020-05-29T14:10:33.690Z]         relErr = rel
[2020-05-29T14:10:33.690Z]     
[2020-05-29T14:10:33.690Z]         print('\n*** Maximum errors for vector of size {}:  rtol={}, atol={}\n'.format(a.size, rtol, atol))
[2020-05-29T14:10:33.690Z]         aTmp = a.copy()
[2020-05-29T14:10:33.690Z]         bTmp = b.copy()
[2020-05-29T14:10:33.690Z]         i = 1
[2020-05-29T14:10:33.690Z]         while i <= a.size:
[2020-05-29T14:10:33.690Z]             if i <= mismatches[0]:
[2020-05-29T14:10:33.690Z]                 print("%3d: Error %f  %s" %(i, rel, locationError(a, b, index, names)))
[2020-05-29T14:10:33.690Z]     
[2020-05-29T14:10:33.690Z]             aTmp[index] = bTmp[index] = 0
[2020-05-29T14:10:33.690Z]             if almost_equal(aTmp, bTmp, rtol, atol, equal_nan=equal_nan):
[2020-05-29T14:10:33.690Z]                 break
[2020-05-29T14:10:33.690Z]     
[2020-05-29T14:10:33.690Z]             i += 1
[2020-05-29T14:10:33.690Z]             if i <= mismatches[1] or mismatches[1] <= 0:
[2020-05-29T14:10:33.690Z]                 index, rel = find_max_violation(aTmp, bTmp, rtol, atol)
[2020-05-29T14:10:33.690Z]             else:
[2020-05-29T14:10:33.690Z]                 break
[2020-05-29T14:10:33.690Z]     
[2020-05-29T14:10:33.690Z]         mismatchDegree = "at least " if mismatches[1] > 0 and i > mismatches[1] else ""
[2020-05-29T14:10:33.690Z]         errMsg = "Error %f exceeds tolerance rtol=%e, atol=%e (mismatch %s%f%%).\n%s" % \
[2020-05-29T14:10:33.690Z]                  (relErr, rtol, atol, mismatchDegree, 100*i/a.size, \
[2020-05-29T14:10:33.690Z]                  locationError(a, b, indexErr, names, maxError=True))
[2020-05-29T14:10:33.690Z]         np.set_printoptions(threshold=4, suppress=True)
[2020-05-29T14:10:33.690Z]         msg = npt.build_err_msg([a, b], err_msg=errMsg)
[2020-05-29T14:10:33.690Z]     
[2020-05-29T14:10:33.690Z] >       raise AssertionError(msg)
[2020-05-29T14:10:33.690Z] E       AssertionError: 
[2020-05-29T14:10:33.690Z] E       Items are not equal:
[2020-05-29T14:10:33.690Z] E       Error 97788.870335 exceeds tolerance rtol=1.000000e-05, atol=1.000000e-20 (mismatch at least 0.171875%).
[2020-05-29T14:10:33.690Z] E       Location of maximum error: (0, 0, 1), a=47483648.00000000, b=2147483648.00000000
[2020-05-29T14:10:33.690Z] E        ACTUAL: array([[[2147483648,   47483648,   47483648, ...,   47483648,
[2020-05-29T14:10:33.690Z] E                  47483648,   47483648],
[2020-05-29T14:10:33.690Z] E               [  47483648,   47483648,   47483648, ...,   47483648,...
[2020-05-29T14:10:33.690Z] E        DESIRED: array([[[2147483648, 2147483648, 2147483648, ..., 2147483648,
[2020-05-29T14:10:33.690Z] E                2147483648, 2147483648],
[2020-05-29T14:10:33.690Z] E               [2147483648, 2147483648, 2147483648, ..., 2147483648,...
[2020-05-29T14:10:33.690Z] 
[2020-05-29T14:10:33.690Z] python/mxnet/test_utils.py:637: AssertionError

http://jenkins.mxnet-ci.amazon-ml.com/blue/rest/organizations/jenkins/pipelines/mxnet-validation/pipelines/centos-cpu/branches/PR-18435/runs/1/nodes/236/steps/284/log/?start=0

[leezu]

I suggest to disable it until the root cause is fixed: #18391