[TOPI] Fix flaky testcase for check round (#4211)

Author: tqchen

topi/tests/python/test_topi_math.py

@@ -52,7 +52,7 @@ def test_apply(
         a_np = np.random.uniform(low=low, high=high, size=shape).astype(A.dtype) * 10
         # avoid round check too close to boundary
         if check_round:
-            a_np += ((np.fmod(a_np, 1) - 0.5) < 1e-6) * 1e-5
+            a_np += ((np.abs(np.fmod(a_np, 1)) - 0.5) < 1e-6) * 1e-5
         b_np = f_numpy(a_np)
 
         def check_device(device):
@@ -100,7 +100,7 @@ def test_isnan(
         a_np.ravel()[np.random.choice(a_np.size, int(a_np.size * 0.5), replace=False)] = np.nan
         # avoid round check too close to boundary
         if check_round:
-            a_np += ((np.fmod(a_np, 1) - 0.5) < 1e-6) * 1e-5
+            a_np += ((np.abs(np.fmod(a_np, 1)) - 0.5) < 1e-6) * 1e-5
         b_np = np.isnan(a_np)
 
         def check_device(device):