pythongh-135124: Change stdout errors in regrtest worker process (python#135138)

Set sys.stdout encoder error handler to backslashreplace in regrtest
workers to avoid UnicodeEncodeError when printing a traceback
or any other non-encodable character.

Move the code from the Regrtest class to setup_process().

Call setup_process() earlier, before displaying regrtest headers.

Author: vstinner

Lib/test/libregrtest/main.py

@@ -543,8 +543,6 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
         self.first_runtests = runtests
         self.logger.set_tests(runtests)
 
-        setup_process()
-
         if (runtests.hunt_refleak is not None) and (not self.num_workers):
             # gh-109739: WindowsLoadTracker thread interferes with refleak check
             use_load_tracker = False
@@ -721,10 +719,7 @@ def _add_python_opts(self) -> None:
         self._execute_python(cmd, environ)
 
     def _init(self):
-        # Set sys.stdout encoder error handler to backslashreplace,
-        # similar to sys.stderr error handler, to avoid UnicodeEncodeError
-        # when printing a traceback or any other non-encodable character.
-        sys.stdout.reconfigure(errors="backslashreplace")
+        setup_process()
 
         if self.junit_filename and not os.path.isabs(self.junit_filename):
             self.junit_filename = os.path.abspath(self.junit_filename)

Lib/test/libregrtest/setup.py

@@ -1,5 +1,6 @@
 import faulthandler
 import gc
+import io
 import os
 import random
 import signal
@@ -52,6 +53,14 @@ def setup_process() -> None:
 
     support.record_original_stdout(sys.stdout)
 
+    # Set sys.stdout encoder error handler to backslashreplace,
+    # similar to sys.stderr error handler, to avoid UnicodeEncodeError
+    # when printing a traceback or any other non-encodable character.
+    #
+    # Use an assertion to fix mypy error.
+    assert isinstance(sys.stdout, io.TextIOWrapper)
+    sys.stdout.reconfigure(errors="backslashreplace")
+
     # Some times __path__ and __file__ are not absolute (e.g. while running from
     # Lib/) and, if we change the CWD to run the tests in a temporary dir, some
     # imports might fail.  This affects only the modules imported before os.chdir().

Lib/test/test_regrtest.py

@@ -768,13 +768,16 @@ def run_command(self, args, input=None, exitcode=0, **kw):
             self.fail(msg)
         return proc
 
-    def run_python(self, args, **kw):
+    def run_python(self, args, isolated=True, **kw):
         extraargs = []
         if 'uops' in sys._xoptions:
             # Pass -X uops along
             extraargs.extend(['-X', 'uops'])
-        args = [sys.executable, *extraargs, '-X', 'faulthandler', '-I', *args]
-        proc = self.run_command(args, **kw)
+        cmd = [sys.executable, *extraargs, '-X', 'faulthandler']
+        if isolated:
+            cmd.append('-I')
+        cmd.extend(args)
+        proc = self.run_command(cmd, **kw)
         return proc.stdout
 
 
@@ -831,8 +834,8 @@ def check_output(self, output):
         self.check_executed_tests(output, self.tests,
                                   randomize=True, stats=len(self.tests))
 
-    def run_tests(self, args, env=None):
-        output = self.run_python(args, env=env)
+    def run_tests(self, args, env=None, isolated=True):
+        output = self.run_python(args, env=env, isolated=isolated)
         self.check_output(output)
 
     def test_script_regrtest(self):
@@ -2276,7 +2279,6 @@ def test_pass(self):
     def test_xml(self):
         code = textwrap.dedent(r"""
             import unittest
-            from test import support
 
             class VerboseTests(unittest.TestCase):
                 def test_failed(self):
@@ -2311,6 +2313,39 @@ def test_failed(self):
         for out in testcase.iter('system-out'):
             self.assertEqual(out.text, r"abc \x1b def")
 
+    def test_nonascii(self):
+        code = textwrap.dedent(r"""
+            import unittest
+
+            class NonASCIITests(unittest.TestCase):
+                def test_docstring(self):
+                    '''docstring:\u20ac'''
+
+                def test_subtest(self):
+                    with self.subTest(param='subtest:\u20ac'):
+                        pass
+
+                def test_skip(self):
+                    self.skipTest('skipped:\u20ac')
+        """)
+        testname = self.create_test(code=code)
+
+        env = dict(os.environ)
+        env['PYTHONIOENCODING'] = 'ascii'
+
+        def check(output):
+            self.check_executed_tests(output, testname, stats=TestStats(3, 0, 1))
+            self.assertIn(r'docstring:\u20ac', output)
+            self.assertIn(r'skipped:\u20ac', output)
+
+        # Run sequentially
+        output = self.run_tests('-v', testname, env=env, isolated=False)
+        check(output)
+
+        # Run in parallel
+        output = self.run_tests('-j1', '-v', testname, env=env, isolated=False)
+        check(output)
+
 
 class TestUtils(unittest.TestCase):
     def test_format_duration(self):