test(ALL): use appropriate n_retry to reduce overall flakiness

CONTRIBUTING.md

@@ -109,7 +109,11 @@ This plugin uses 'mini.test' to manage its tests. For a more hands-on introducti
 **Notes**:
 - If new functionality relies on an external dependency (`git` CLI tool, LSP server, etc.), use mocking (writing Lua code which emulates dependency usage as close as reasonably possible). For examples, take a look at tests for 'mini.pick', 'mini.completion', and 'mini.statusline'.
 - There is a certain number of tests that are flaky (i.e. will sometimes report an error due to other reasons than actual functionality being broke). It is usually the ones which test time related functionality (i.e. that certain action was done after specific amount of delay).
+
     A commonly used way to know if the test is flaky is that it fails on non-nightly Neovim version yet there were no changes to its tested module after it had passed in the past. For example, some 'mini.animate' test is shown to break but there were no changes to it since test passed in CI couple of days before.
+
+    This issue is addressed by having test cases being executed several times in case of failure (with more retries in slow context). See ["Retry" section in 'TESTING.md'](TESTING.md#Retry).
+
     In case there is some test breaking which reasonably should not, rerun that test (or the whole file) at least several times.
 - Advice for writing more robust tests:
     - To test asynchronous or slow execution, use common `sleep()` test helper. For a more robust testing code, **never** directly use numbers to compute sleep time. Use precomputed time delay constants, which should always take into account different testing OSs (like be bigger on Windows, etc.). If module testing requires its extensive use and tests can not be made robust enough (examples are 'mini.animate', 'mini.jump', etc.), consider using it with argument that skips entire test case if `sleep()` is called in slow context.

tests/helpers.lua

@@ -202,4 +202,14 @@ Helpers.sleep = function(ms, child, skip_slow)
   if child ~= nil then child.poke_eventloop() end
 end
 
+-- Standardized way of setting number of retries
+Helpers.get_n_retry = function(n)
+  local coef = 1
+  if Helpers.is_ci() then
+    if Helpers.is_windows() then coef = 2 end
+    if Helpers.is_macos() then coef = 4 end
+  end
+  return coef * n
+end
+
 return Helpers

tests/test_ai.lua

@@ -64,6 +64,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_align.lua

@@ -63,6 +63,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_animate.lua

@@ -111,6 +111,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================

tests/test_base16.lua

@@ -70,6 +70,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_basics.lua

@@ -20,6 +20,7 @@ local T = new_set({
     pre_case = child.setup,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_bracketed.lua

@@ -153,6 +153,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_bufremove.lua

@@ -192,6 +192,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_clue.lua

@@ -168,6 +168,7 @@ local T = new_set({
     pre_case = function() child.setup() end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================

tests/test_colors.lua

@@ -31,6 +31,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================
@@ -1616,6 +1617,7 @@ T['animate()'] = new_set({
       child.lua('_G.default_show_duration = ' .. default_show_duration)
     end,
   },
+  n_retry = helpers.get_n_retry(4),
 })
 
 local is_cs_1 = function()
@@ -2209,7 +2211,7 @@ T['simulate_cvd()']['validates arguments'] = function()
 end
 
 -- Integration tests ==========================================================
-T[':Colorscheme'] = new_set()
+T[':Colorscheme'] = new_set({ n_retry = helpers.get_n_retry(4) })
 
 T[':Colorscheme']['works'] = function()
   mock_cs()

tests/test_comment.lua

@@ -60,6 +60,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_completion.lua

@@ -85,6 +85,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================

tests/test_cursorword.lua

@@ -51,6 +51,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================

tests/test_deps.lua

@@ -198,6 +198,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================

tests/test_diff.lua

@@ -229,6 +229,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================

tests/test_doc.lua

@@ -38,6 +38,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_extra.lua

@@ -224,6 +224,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================

tests/test_files.lua

@@ -208,6 +208,7 @@ local T = new_set({
     post_case = function() vim.fn.delete(make_test_path('data'), 'rf') end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================

tests/test_fuzzy.lua

@@ -20,6 +20,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_git.lua

@@ -176,6 +176,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================

tests/test_hipatterns.lua

@@ -72,6 +72,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================

tests/test_hues.lua

@@ -30,6 +30,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_icons.lua

@@ -28,6 +28,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================

tests/test_indentscope.lua

@@ -63,6 +63,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================

tests/test_jump.lua

@@ -39,6 +39,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================

tests/test_jump2d.lua

@@ -97,6 +97,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================

tests/test_map.lua

@@ -107,6 +107,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Data =======================================================================

tests/test_misc.lua

@@ -44,6 +44,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_move.lua

@@ -60,6 +60,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_notify.lua

@@ -73,6 +73,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================

tests/test_operators.lua

@@ -48,6 +48,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_pairs.lua

@@ -263,6 +263,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_pick.lua

@@ -225,6 +225,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================
@@ -982,7 +983,7 @@ T['refresh()']['recomputes window config'] = function()
   child.expect_screenshot()
 end
 
-T['default_match()'] = new_set()
+T['default_match()'] = new_set({ n_retry = helpers.get_n_retry(4) })
 
 local default_match = forward_lua('MiniPick.default_match')
 

tests/test_sessions.lua

@@ -166,6 +166,7 @@ local T = new_set({
       child.stop()
     end,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_splitjoin.lua

@@ -55,6 +55,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_starter.lua

@@ -85,6 +85,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_statusline.lua

@@ -75,6 +75,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_surround.lua

@@ -92,6 +92,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================

tests/test_tabline.lua

@@ -46,6 +46,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_test.lua

@@ -95,6 +95,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================
@@ -486,7 +487,6 @@ T['collect()']['works'] = function()
   local keys = child.lua_get('vim.tbl_keys(_G.cases[1])')
   table.sort(keys)
   eq(keys, { 'args', 'data', 'desc', 'hooks', 'n_retry', 'test' })
-  eq(child.lua_get('_G.cases[1].n_retry'), 1)
 
   local hook_keys = child.lua_get('vim.tbl_keys(_G.cases[1].hooks)')
   table.sort(hook_keys)

tests/test_trailspace.lua

@@ -34,6 +34,7 @@ local T = new_set({
     end,
     post_once = child.stop,
   },
+  n_retry = helpers.get_n_retry(1),
 })
 
 -- Unit tests =================================================================

tests/test_visits.lua

@@ -163,6 +163,7 @@ local T = new_set({
       vim.fn.delete(data_std_path, 'rf')
     end,
   },
+  n_retry = helpers.get_n_retry(2),
 })
 
 -- Unit tests =================================================================