Capture failures from auto3dseg related subprocess calls

monai/apps/auto3dseg/auto_runner.py

@@ -13,7 +13,6 @@
 
 import os
 import shutil
-import subprocess
 import warnings
 from copy import deepcopy
 from time import sleep
@@ -31,7 +30,7 @@
 from monai.bundle import ConfigParser
 from monai.transforms import SaveImage
 from monai.utils import AlgoKeys, has_option, look_up_option, optional_import
-from monai.utils.misc import check_kwargs_exist_in_class_init
+from monai.utils.misc import check_kwargs_exist_in_class_init, run_cmd
 
 logger = get_logger(module_name=__name__)
 
@@ -719,15 +718,15 @@ def _train_algo_in_nni(self, history: list[dict[str, Any]]) -> None:
                 logger.info(f"AutoRunner HPO is in dry-run mode. Please manually launch: {cmd}")
                 continue
 
-            subprocess.run(cmd.split(), check=True)
+            run_cmd(cmd.split(), check=True)
 
             n_trainings = len(import_bundle_algo_history(self.work_dir, only_trained=True))
             while n_trainings - last_total_tasks < max_trial:
                 sleep(1)
                 n_trainings = len(import_bundle_algo_history(self.work_dir, only_trained=True))
 
             cmd = "nnictl stop --all"
-            subprocess.run(cmd.split(), check=True)
+            run_cmd(cmd.split(), check=True)
             logger.info(f"NNI completes HPO on {name}")
             last_total_tasks = n_trainings
 

monai/apps/auto3dseg/bundle_gen.py

@@ -33,7 +33,7 @@
 from monai.auto3dseg.utils import algo_to_pickle
 from monai.bundle.config_parser import ConfigParser
 from monai.config import PathLike
-from monai.utils import ensure_tuple
+from monai.utils import ensure_tuple, run_cmd
 from monai.utils.enums import AlgoKeys
 
 logger = get_logger(module_name=__name__)
@@ -243,7 +243,7 @@ def _run_cmd(self, cmd: str, devices_info: str = "") -> subprocess.CompletedProc
 
         logger.info(f"Launching: {' '.join(cmd_list)}")
 
-        return subprocess.run(cmd_list, env=ps_environ, check=True)
+        return run_cmd(cmd_list, env=ps_environ, check=True)
 
     def train(
         self, train_params: None | dict = None, device_setting: None | dict = None

monai/apps/auto3dseg/ensemble_builder.py

@@ -12,7 +12,6 @@
 from __future__ import annotations
 
 import os
-import subprocess
 from abc import ABC, abstractmethod
 from collections.abc import Mapping, Sequence
 from copy import deepcopy
@@ -33,7 +32,7 @@
 from monai.transforms import MeanEnsemble, SaveImage, VoteEnsemble
 from monai.utils import RankFilter, deprecated_arg
 from monai.utils.enums import AlgoKeys
-from monai.utils.misc import check_kwargs_exist_in_class_init, prob2class
+from monai.utils.misc import check_kwargs_exist_in_class_init, prob2class, run_cmd
 from monai.utils.module import look_up_option, optional_import
 
 tqdm, has_tqdm = optional_import("tqdm", name="tqdm")
@@ -672,5 +671,5 @@ def _create_cmd(self) -> None:
             cmd = f"{cmd} -m {base_cmd}"
             cmd_list = cmd.split()
 
-        _ = subprocess.run(cmd_list, env=ps_environ, check=True)
+        run_cmd(cmd_list, env=ps_environ, check=True)
         return

monai/utils/__init__.py

@@ -66,6 +66,7 @@
     MAX_SEED,
     ImageMetaKey,
     MONAIEnvVars,
+    check_kwargs_exist_in_class_init,
     check_parent_dir,
     copy_to_device,
     ensure_tuple,
@@ -84,6 +85,7 @@
     path_to_uri,
     pprint_edges,
     progress_bar,
+    run_cmd,
     sample_slices,
     save_obj,
     set_determinism,

monai/utils/misc.py

@@ -17,6 +17,7 @@
 import pprint
 import random
 import shutil
+import subprocess
 import tempfile
 import types
 import warnings
@@ -73,6 +74,7 @@
     "CheckKeyDuplicatesYamlLoader",
     "ConvertUnits",
     "check_kwargs_exist_in_class_init",
+    "run_cmd",
 ]
 
 _seed = None
@@ -821,3 +823,29 @@ def check_kwargs_exist_in_class_init(cls, kwargs):
     extra_kwargs = input_kwargs - init_params
 
     return extra_kwargs == set(), extra_kwargs
+
+
+def run_cmd(cmd_list: list[str], **kwargs: Any) -> subprocess.CompletedProcess:
+    """
+    Run a command by using ``subprocess.run`` with capture_output=True and stderr=subprocess.STDOUT
+    so that the raise exception will have that information. The argument `capture_output` can be set explicitly
+    if desired, but will be overriden with the debug status from the variable.
+
+    Args:
+        cmd_list: a list of strings describing the command to run.
+        kwargs: keyword arguments supported by the ``subprocess.run`` method.
+
+    Returns:
+        a CompletedProcess instance after the command completes.
+    """
+    debug = MONAIEnvVars.debug()
+    kwargs["capture_output"] = kwargs.get("capture_output", debug)
+
+    try:
+        return subprocess.run(cmd_list, **kwargs)
+    except subprocess.CalledProcessError as e:
+        if not debug:
+            raise
+        output = str(e.stdout.decode(errors="replace"))
+        errors = str(e.stderr.decode(errors="replace"))
+        raise RuntimeError(f"subprocess call error {e.returncode}: {errors}, {output}.") from e

tests/test_monai_utils_misc.py

@@ -11,11 +11,12 @@
 
 from __future__ import annotations
 
+import os
 import unittest
 
 from parameterized import parameterized
 
-from monai.utils.misc import check_kwargs_exist_in_class_init, to_tuple_of_dictionaries
+from monai.utils.misc import check_kwargs_exist_in_class_init, run_cmd, to_tuple_of_dictionaries
 
 TO_TUPLE_OF_DICTIONARIES_TEST_CASES = [
     ({}, tuple(), tuple()),
@@ -72,5 +73,19 @@ def _custom_user_function(self, cls, *args, **kwargs):
         return check_kwargs_exist_in_class_init(cls, kwargs)
 
 
+class TestCommandRunner(unittest.TestCase):
+    def test_run_cmd(self):
+        cmd1 = "python"
+        cmd2 = "-c"
+        cmd3 = 'import sys; print("\\tThis is on stderr\\n", file=sys.stderr); sys.exit(1)'
+        os.environ["MONAI_DEBUG"] = str(True)
+        try:
+            run_cmd([cmd1, cmd2, cmd3], check=True)
+        except RuntimeError as err:
+            self.assertIn("This is on stderr", str(err))
+            self.assertNotIn("\\n", str(err))
+            self.assertNotIn("\\t", str(err))
+
+
 if __name__ == "__main__":
     unittest.main()