Pass --fail-rerun option on the main branch

master/custom/__init__.py

@@ -0,0 +1,6 @@
+MAIN_BRANCH_VERSION = "3.13"
+CUSTOM_BRANCH_NAME = "custom"
+# The Git branch is called "main", but we give it a different name in buildbot.
+# See git_branches in master/master.cfg.
+MAIN_BRANCH_NAME = "3.x"
+JUNIT_FILENAME = "test-results.xml"

master/custom/builders.py

@@ -20,7 +20,6 @@
     ClangUnixInstalledBuild,
     SharedUnixBuild,
     SlowNonDebugUnixBuild,
-    SlowSharedUnixBuild,
     SlowUnixInstalledBuild,
     LTONonDebugUnixBuild,
     LTOPGONonDebugBuild,
@@ -240,7 +239,7 @@ def get_builders(settings):
 
 # Match builder name (excluding the branch name) of builders that should only
 # run on the master and "custom" branches.
-ONLY_MASTER_BRANCH = (
+ONLY_MAIN_BRANCH = (
     "Alpine Linux",
     # Cygwin is not supported on 2.7, 3.6, 3.7
     "Cygwin",

master/custom/factories.py

@@ -9,6 +9,8 @@
 
 from buildbot.plugins import util
 
+from . import (MAIN_BRANCH_VERSION, CUSTOM_BRANCH_NAME, MAIN_BRANCH_NAME,
+               JUNIT_FILENAME)
 from .steps import (
     Test,
     Clean,
@@ -18,8 +20,13 @@
     UploadTestResults,
 )
 
-main_branch_version = "3.13"
-CUSTOM_BRANCH_NAME = "custom"
+# Python branches with regrtest --fail-rerun option
+# https://github.com/python/cpython/issues/108834
+BRANCH_WITH_FAIL_RERUN = (
+    MAIN_BRANCH_NAME,
+    #"3.12",
+    #"3.11",
+)
 
 # This (default) timeout is for each individual test file.
 # It is a bit more than the default faulthandler timeout in regrtest.py
@@ -40,42 +47,21 @@ def __init__(self, source, *, extra_tags=[], **kwargs):
         self.tags = self.factory_tags + extra_tags
 
 
-class FreezeBuild(TaggedBuildFactory):
-    buildersuffix = ".freeze"  # to get unique directory names on master
-    test_timeout = None
-    factory_tags = ["freeze"]
-
-    def setup(self, **kwargs):
-        self.addStep(Configure(command=["./configure", "--prefix", "$(PWD)/target"]))
-        self.addStep(Compile(command=["make"]))
-        self.addStep(
-            ShellCommand(
-                name="install", description="installing", command=["make", "install"]
-            )
-        )
-        self.addStep(
-            Test(
-                command=[
-                    "make",
-                    "-C",
-                    "Tools/freeze/test",
-                    "PYTHON=../../../target/bin/python3",
-                    "OUTDIR=../../../target/freezetest",
-                ]
-            )
-        )
-
-
 ##############################################################################
 ###############################  UNIX BUILDS  ################################
 ##############################################################################
 
 
+def has_option(option, test_options):
+    # return True for option='-j' and test_options=['-uall', '-j2']
+    return option in ' '.join(test_options)
+
+
 class UnixBuild(TaggedBuildFactory):
     configureFlags = ["--with-pydebug"]
     compile_environ = {}
     interpreterFlags = ""
-    testFlags = "-j2"
+    testFlags = ["-j2"]
     makeTarget = "all"
     test_timeout = None
     test_environ = {}
@@ -106,22 +92,24 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
             Configure(command=configure_cmd, **oot_kwargs)
         )
         compile = ["make", self.makeTarget]
-        testopts = self.testFlags
-        if "-R" not in self.testFlags:
-            testopts += " --junit-xml test-results.xml"
+        testopts = list(self.testFlags)
+        if not has_option("-R", self.testFlags):
+            testopts.extend(("--junit-xml", JUNIT_FILENAME))
         # Timeout for the buildworker process
         self.test_timeout = self.test_timeout or TEST_TIMEOUT
         # Timeout for faulthandler
         faulthandler_timeout = self.test_timeout - 5 * 60
         if parallel:
             compile = ["make", parallel, self.makeTarget]
-            testopts = testopts + " " + parallel
-        if "-j" not in testopts:
-            testopts = "-j2 " + testopts
+            testopts.append(parallel)
+        if not has_option("-j", testopts):
+            testopts.append("-j2")
+        if branch in BRANCH_WITH_FAIL_RERUN:
+            testopts.append("--fail-rerun")
         test = [
             "make",
             "buildbottest",
-            "TESTOPTS=" + testopts + " ${BUILDBOT_TESTOPTS}",
+            "TESTOPTS=" + " ".join(testopts) + " ${BUILDBOT_TESTOPTS}",
             "TESTPYTHONOPTS=" + self.interpreterFlags,
             "TESTTIMEOUT=" + str(faulthandler_timeout),
         ]
@@ -148,8 +136,8 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
                 **oot_kwargs
             )
         )
-        if branch not in ("3",) and "-R" not in self.testFlags:
-            filename = "test-results.xml"
+        if branch not in ("3",) and not has_option("-R", self.testFlags):
+            filename = JUNIT_FILENAME
             if self.build_out_of_tree:
                 filename = os.path.join(out_of_tree_dir, filename)
             self.addStep(UploadTestResults(branch, filename=filename))
@@ -174,7 +162,7 @@ class UnixVintageParserBuild(UnixBuild):
 
 class UnixRefleakBuild(UnixBuild):
     buildersuffix = ".refleak"
-    testFlags = "-R 3:3 -u-cpu"
+    testFlags = ["-R", "3:3", "-u-cpu"]
     # -R 3:3 is supposed to only require timeout x 6, but in practice,
     # it's much more slower. Use timeout x 10 to prevent timeout
     # caused by --huntrleaks.
@@ -199,8 +187,8 @@ class UnixInstalledBuild(TaggedBuildFactory):
     factory_tags = ["installed"]
 
     def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
-        if branch == "3.x":
-            branch = main_branch_version
+        if branch == MAIN_BRANCH_NAME:
+            branch = MAIN_BRANCH_VERSION
         elif branch == "custom":
             branch = "3"
         installed_python = "./target/bin/python%s" % branch
@@ -213,19 +201,23 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
 
         compile = ["make", self.makeTarget]
         install = ["make", self.installTarget]
-        testopts = self.defaultTestOpts[:]
+        testopts = list(self.defaultTestOpts)
         # Timeout for the buildworker process
         self.test_timeout = self.test_timeout or TEST_TIMEOUT
         # Timeout for faulthandler
         faulthandler_timeout = self.test_timeout - 5 * 60
-        testopts += [f"--timeout={faulthandler_timeout}"]
+        testopts.append(f"--timeout={faulthandler_timeout}")
         if parallel:
             compile = ["make", parallel, self.makeTarget]
             install = ["make", parallel, self.installTarget]
-            testopts = testopts + [parallel]
+            testopts.append(parallel)
+        if branch in BRANCH_WITH_FAIL_RERUN:
+            testopts.append("--fail-rerun")
 
-        test = [installed_python] + self.interpreterFlags
-        test += ["-m", "test.regrtest"] + testopts
+        test = [installed_python,
+                *self.interpreterFlags,
+                "-m", "test",
+                *testopts]
 
         self.addStep(Compile(command=compile))
         self.addStep(Install(command=install))
@@ -351,10 +343,6 @@ class SlowNonDebugUnixBuild(NonDebugUnixBuild):
     test_timeout = SLOW_TIMEOUT
 
 
-class SlowSharedUnixBuild(SharedUnixBuild):
-    test_timeout = SLOW_TIMEOUT
-
-
 class SlowUnixInstalledBuild(UnixInstalledBuild):
     test_timeout = SLOW_TIMEOUT
 
@@ -505,7 +493,8 @@ class MacOSArmWithBrewBuild(UnixBuild):
         "LDFLAGS=-L/opt/homebrew/lib",
     ]
     # These tests are known to crash on M1 macs (see bpo-45289).
-    testFlags = UnixBuild.testFlags + " -x test_dbm test_dbm_ndbm test_shelve"
+    testFlags = [*UnixBuild.testFlags,
+                 "-x", "test_dbm", "test_dbm_ndbm", "test_shelve"]
 
 ##############################################################################
 ############################  WINDOWS BUILDS  ################################
@@ -525,12 +514,14 @@ class BaseWindowsBuild(TaggedBuildFactory):
 
     def setup(self, parallel, branch, **kwargs):
         build_command = self.build_command + self.buildFlags
-        test_command = self.test_command + self.testFlags
-        if "-R" not in self.testFlags:
-            test_command += [r"--junit-xml", r"test-results.xml"]
+        test_command = [*self.test_command, *self.testFlags]
+        if not has_option("-R", self.testFlags):
+            test_command.extend((r"--junit-xml", JUNIT_FILENAME))
         clean_command = self.clean_command + self.cleanFlags
         if parallel:
             test_command.append(parallel)
+        if branch in BRANCH_WITH_FAIL_RERUN:
+            test_command.append("--fail-rerun")
         self.addStep(Compile(command=build_command))
         self.addStep(
             ShellCommand(
@@ -541,9 +532,9 @@ def setup(self, parallel, branch, **kwargs):
             )
         )
         timeout = self.test_timeout if self.test_timeout else TEST_TIMEOUT
-        test_command += ["--timeout", timeout - (5 * 60)]
+        test_command.extend(("--timeout", timeout - (5 * 60)))
         self.addStep(Test(command=test_command, timeout=timeout))
-        if branch not in ("3",) and "-R" not in self.testFlags:
+        if branch not in ("3",) and not has_option("-R", self.testFlags):
             self.addStep(UploadTestResults(branch))
         self.addStep(Clean(command=clean_command))
 
@@ -585,7 +576,7 @@ class Windows64BigmemBuild(BaseWindowsBuild):
 
 class Windows64RefleakBuild(Windows64Build):
     buildersuffix = ".refleak"
-    testFlags = ["-p", "x64"] + WindowsRefleakBuild.testFlags
+    testFlags = ["-p", "x64", *WindowsRefleakBuild.testFlags]
     # -R 3:3 is supposed to only require timeout x 6, but in practice,
     # it's much more slower. Use timeout x 10 to prevent timeout
     # caused by --huntrleaks.
@@ -596,7 +587,7 @@ class Windows64RefleakBuild(Windows64Build):
 class Windows64ReleaseBuild(Windows64Build):
     buildersuffix = ".nondebug"
     buildFlags = Windows64Build.buildFlags + ["-c", "Release"]
-    testFlags = Windows64Build.testFlags + ["+d"]
+    testFlags = [*Windows64Build.testFlags, "+d"]
     # keep default cleanFlags, both configurations get cleaned
     factory_tags = ["win64", "nondebug"]
 
@@ -611,7 +602,7 @@ class WindowsARM64Build(BaseWindowsBuild):
 class WindowsARM64ReleaseBuild(WindowsARM64Build):
     buildersuffix = ".nondebug"
     buildFlags = WindowsARM64Build.buildFlags + ["-c", "Release"]
-    testFlags = WindowsARM64Build.testFlags + ["+d"]
+    testFlags = [*WindowsARM64Build.testFlags, "+d"]
     # keep default cleanFlags, both configurations get cleaned
     factory_tags = ["win-arm64", "nondebug"]
 
@@ -703,13 +694,15 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
             )
         )
 
-        testopts = self.testFlags
-        if "-R" not in self.testFlags:
-            testopts += " --junit-xml test-results.xml"
+        testopts = list(self.testFlags)
+        if not has_option("-R", self.testFlags):
+            testopts.extend((" --junit-xml", JUNIT_FILENAME))
         if parallel:
-            testopts = testopts + " " + parallel
-        if "-j" not in testopts:
-            testopts = "-j2 " + testopts
+            testopts.append(parallel)
+        if not has_option("-j", self.testFlags):
+            testopts.append("-j2")
+        if branch in BRANCH_WITH_FAIL_RERUN:
+            testopts.append("--fail-rerun")
 
         # Timeout for the buildworker process
         self.test_timeout = self.test_timeout or TEST_TIMEOUT
@@ -719,7 +712,7 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
         test = [
             "make",
             "buildbottest",
-            "TESTOPTS=" + testopts + " ${BUILDBOT_TESTOPTS}",
+            "TESTOPTS=" + " ".join(testopts) + " ${BUILDBOT_TESTOPTS}",
             "TESTPYTHONOPTS=" + self.interpreterFlags,
             "TESTTIMEOUT=" + str(faulthandler_timeout),
         ]
@@ -756,8 +749,8 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
                     workdir=oot_host_path,
                 )
             )
-            if branch not in ("3",) and "-R" not in self.testFlags:
-                filename = os.path.join(oot_host_path, "test-results.xml")
+            if branch not in ("3",) and not has_option("-R", self.testFlags):
+                filename = os.path.join(oot_host_path, JUNIT_FILENAME)
                 self.addStep(UploadTestResults(branch, filename=filename))
         self.addStep(
             Clean(

master/custom/steps.py

@@ -1,10 +1,13 @@
 import re
 
-from buildbot.steps.shell import ShellCommand, Test as BaseTest
 from buildbot.plugins import steps, util
+from buildbot.process.results import SUCCESS, WARNINGS
+from buildbot.steps.shell import ShellCommand, Test as BaseTest
 from buildbot.steps.source.git import Git as _Git
 from buildbot.steps.source.github import GitHub as _GitHub
 
+from . import JUNIT_FILENAME
+
 
 class Git(_Git):
     # GH-68: If "git clone" fails, mark the whole build as WARNING
@@ -77,6 +80,25 @@ class Test(BaseTest):
     # Give SIGTERM 30 seconds to shut things down before SIGKILL.
     sigtermTime = 30
 
+    # Treat "regrtest --fail-rerun" exit code (5) as WARNINGS
+    # https://github.com/python/cpython/issues/108834
+    decodeRC = {
+        0: SUCCESS,
+
+        # Treat --fail-rerun exit code (5) to WARNINGS, when a test failed but
+        # passed when run again in verbose mode in a fresh process (unstable
+        # test).
+        5: WARNINGS,     # EXITCODE_RERUN_FAIL
+
+        # Any exit code not present in the dictionary is treated as FAILURE.
+        # So there is no need to map each regrtest exit code to FAILURE.
+        #
+        # 2: FAILURE,    # EXITCODE_BAD_TEST
+        # 3: FAILURE,    # EXITCODE_ENV_CHANGED
+        # 4: FAILURE,    # EXITCODE_NO_TESTS_RAN
+        # 130: FAILURE,  # EXITCODE_INTERRUPTED
+    }
+
     def evaluateCommand(self, cmd):
         if cmd.didFail():
             self.setProperty("test_failed_to_build", True)
@@ -139,7 +161,7 @@ class UploadTestResults(steps.FileUpload):
     def _has_the_build_failed(self, build):
         return self.getProperty("test_failed_to_build")
 
-    def __init__(self, branch, filename="test-results.xml"):
+    def __init__(self, branch, filename=JUNIT_FILENAME):
         super().__init__(
             doStepIf=self._has_the_build_failed,
             workersrc=filename,

master/custom/workers.py

@@ -6,6 +6,8 @@
 
 from buildbot.plugins import worker as _worker
 
+from custom.factories import MAIN_BRANCH_NAME
+
 
 # By default, the buildmaster sends a simple, non-blocking message to each
 # worker every hour. These keepalives ensure that traffic is flowing over the
@@ -232,7 +234,7 @@ def get_workers(settings):
         cpw(
             name="ware-alpine",
             tags=['linux', 'unix', 'alpine', 'docker', 'amd64', 'x86-64'],
-            branches=['3.x'],
+            branches=[MAIN_BRANCH_NAME],
         ),
         cpw(
             name="ware-gentoo-x86",