mutest: add --cli-on-error and --pause-on-error support

munet/mutest/__main__.py

@@ -24,6 +24,7 @@
 from munet.args import add_testing_args
 from munet.base import Bridge
 from munet.base import get_event_loop
+from munet.cli import async_cli
 from munet.compat import PytestConfig
 from munet.mutest import userapi as uapi
 from munet.native import L3NodeMixin
@@ -232,7 +233,11 @@ async def execute_test(
     tc = uapi.TestCase(
         str(test_num), test_name, test, targets, args, logger, reslog, args.full_summary
     )
-    passed, failed, e = tc.execute()
+    try:
+        passed, failed, e = tc.execute()
+    except uapi.CLIOnErrorError as error:
+        await async_cli(unet)
+        passed, failed, e = 0, 0, error
 
     run_time = time.time() - tc.info.start_time
 

munet/mutest/userapi.py

@@ -66,6 +66,7 @@
 import pprint
 import re
 import subprocess
+import sys
 import time
 
 from argparse import Namespace
@@ -81,7 +82,44 @@
 class ScriptError(Exception):
     """An unrecoverable script failure."""
 
-    pass
+
+class CLIOnErrorError(Exception):
+    """Enter CLI after error."""
+
+
+def pause_test(desc=""):
+    isatty = sys.stdout.isatty()
+    if not isatty:
+        desc = f" for {desc}" if desc else ""
+        logging.info("NO PAUSE on non-tty terminal%s", desc)
+        return
+
+    while True:
+        if desc:
+            print(f"\n== PAUSING: {desc} ==")
+        try:
+            user = input('PAUSED, "cli" for CLI, "pdb" to debug, "Enter" to continue: ')
+        except EOFError:
+            print("^D...continuing")
+            break
+        user = user.strip()
+        if user == "cli":
+            raise CLIOnErrorError()
+        if user == "pdb":
+            breakpoint()  # pylint: disable=W1515
+        elif user:
+            print(f'Unrecognized input: "{user}"')
+        else:
+            break
+
+
+def act_on_result(success, args, desc=""):
+    if success:
+        return
+    if args.cli_on_error:
+        raise CLIOnErrorError()
+    if args.pause_on_error:
+        pause_test(desc)
 
 
 class TestCaseInfo:
@@ -314,6 +352,8 @@ def __exec_script(self, path, print_header, add_newline):
             # result = await locals()[f"_{name}"](_ok_result)
         except ScriptError as error:
             return error
+        except CLIOnErrorError:
+            raise
         except Exception as error:
             logging.error(
                 "Unexpected exception executing %s: %s", name, error, exc_info=True
@@ -598,6 +638,7 @@ def include(self, pathname: str, new_section: bool = False):
         """
         path = Path(pathname)
         path = self.info.path.parent.joinpath(path)
+        do_cli = False
 
         self.oplogf(
             "include: new path: %s create section: %s currently __in_section: %s",
@@ -617,7 +658,12 @@ def include(self, pathname: str, new_section: bool = False):
             self.info.path = path
             self.oplogf("include: swapped info path: new %s old %s", path, old_path)
 
-        e = self.__exec_script(path, print_header=new_section, add_newline=new_section)
+        try:
+            e = self.__exec_script(
+                path, print_header=new_section, add_newline=new_section
+            )
+        except CLIOnErrorError:
+            do_cli = True
 
         if new_section:
             # Something within the section creating include has also created a section
@@ -643,6 +689,9 @@ def include(self, pathname: str, new_section: bool = False):
             # we are returning to
             self.info.path = old_path
             self.oplogf("include: restored info path: %s", old_path)
+
+        if do_cli:
+            raise CLIOnErrorError()
         if e:
             raise ScriptError(e)
 
@@ -747,6 +796,7 @@ def match_step(
         success, ret = self._match_command(
             target, cmd, match, expect_fail, flags, exact_match
         )
+        act_on_result(success, self.args, desc)
         if desc:
             self.__post_result(target, success, desc)
         return success, ret
@@ -758,6 +808,7 @@ def test_step(self, expr_or_value: Any, desc: str, target: str = "") -> bool:
         """
         success = bool(expr_or_value)
         self.__post_result(target, success, desc)
+        act_on_result(success, self.args, desc)
         return success
 
     def match_step_json(
@@ -790,6 +841,7 @@ def match_step_json(
         )
         if desc:
             self.__post_result(target, success, desc)
+        act_on_result(success, self.args, desc)
         return success, ret
 
     def wait_step(
@@ -838,6 +890,7 @@ def wait_step(
         )
         if desc:
             self.__post_result(target, success, desc)
+        act_on_result(success, self.args, desc)
         return success, ret
 
     def wait_step_json(
@@ -876,6 +929,7 @@ def wait_step_json(
         )
         if desc:
             self.__post_result(target, success, desc)
+        act_on_result(success, self.args, desc)
         return success, ret