runner switch: pin tests that are specific to the "runner" (legacy)

Many existing tests cover features that are specific to the legacy
runner, and it makes no or little sense to port them to be compatible
with the nrunner.

Examples include:

 * selftests/functional/test_basic.py:RunnerOperationTest.test_unsupported_status
  - The legacy runner contains extra handling after the test process
    ends, and this test depends on that behavior

 * selftests/functional/test_basic.py:RunnerOperationTest.test_hanged_test_with_status
 * selftests/functional/test_basic.py:RunnerOperationTest.test_no_status_reported
  - Relies on Avocado trying to kill test processes, only performed in
    the legacy runner

 * selftests/functional/test_output_check.py:
  - Output check feature is limited to the legacy and data files
    available along side the test
  - avocado-framework#3882

And many other similar examples.

Signed-off-by: Cleber Rosa <crosa@redhat.com>

.cirrus.yml

@@ -30,9 +30,9 @@ redhat_egg_task:
        - python3 setup.py bdist_egg
        - mv dist/avocado_framework-*egg /tmp
        - python3 setup.py clean --all
-       - python3 -c 'import sys; import glob; sys.path.insert(0, glob.glob("/tmp/avocado_framework-*.egg")[0]); from avocado.core.main import main; sys.exit(main())' run /bin/true
+       - python3 -c 'import sys; import glob; sys.path.insert(0, glob.glob("/tmp/avocado_framework-*.egg")[0]); from avocado.core.main import main; sys.exit(main())' run --test-runner=runner /bin/true
        - cd /tmp
-       - python3 -c 'import sys; from pkg_resources import require; require("avocado-framework"); from avocado.core.main import main; sys.exit(main())' run /bin/true
+       - python3 -c 'import sys; from pkg_resources import require; require("avocado-framework"); from avocado.core.main import main; sys.exit(main())' run --test-runner=runner /bin/true
     container:
         matrix:
           - image: fedora:33
@@ -58,9 +58,9 @@ debian_egg_task:
        - python3 setup.py bdist_egg
        - mv dist/avocado_framework-*egg /tmp
        - python3 setup.py clean --all
-       - python3 -c 'import sys; import glob; sys.path.insert(0, glob.glob("/tmp/avocado_framework-*.egg")[0]); from avocado.core.main import main; sys.exit(main())' run /bin/true
+       - python3 -c 'import sys; import glob; sys.path.insert(0, glob.glob("/tmp/avocado_framework-*.egg")[0]); from avocado.core.main import main; sys.exit(main())' run --test-runner=runner /bin/true
        - cd /tmp
-       - python3 -c 'import sys; from pkg_resources import require; require("avocado-framework"); from avocado.core.main import main; sys.exit(main())' run /bin/true
+       - python3 -c 'import sys; from pkg_resources import require; require("avocado-framework"); from avocado.core.main import main; sys.exit(main())' run --test-runner=runner /bin/true
     container:
         matrix:
           - image: debian:10.10

optional_plugins/varianter_yaml_to_mux/tests/test_functional.py

@@ -136,6 +136,7 @@ def test_run_mplex_params(self):
                             ('/run/long', 'This is very long\nmultiline\ntext.')):
             variant, msg = variant_msg
             cmd_line = ('%s run --job-results-dir %s --disable-sysinfo '
+                        '--test-runner=runner '
                         'examples/tests/env_variables.sh '
                         '-m examples/tests/env_variables.sh.data/env_variables.yaml '
                         '--mux-filter-only %s'
@@ -197,6 +198,7 @@ class DryRun(unittest.TestCase):
 
     def test_dry_run(self):
         cmd = ("%s run --disable-sysinfo --dry-run --dry-run-no-cleanup --json - "
+               "--test-runner=runner "
                "--mux-inject foo:1 bar:2 baz:3 foo:foo:a "
                "foo:bar:b foo:baz:c bar:bar:bar "
                "-- examples/tests/passtest.py "

selftests/functional/test_basic.py

@@ -253,6 +253,7 @@ def test_unsupported_status(self):
                                     UNSUPPORTED_STATUS_TEST_CONTENTS,
                                     "avocado_unsupported_status") as tst:
             res = process.run("%s run --disable-sysinfo --job-results-dir %s %s"
+                              " --test-runner=runner"
                               " --json -" % (AVOCADO, self.tmpdir.name, tst),
                               ignore_status=True)
             self.assertEqual(res.exit_status, exit_codes.AVOCADO_TESTS_FAIL)
@@ -273,6 +274,7 @@ def test_hanged_test_with_status(self):
                                     REPORTS_STATUS_AND_HANG,
                                     "hanged_test_with_status") as tst:
             res = process.run("%s run --disable-sysinfo --job-results-dir %s %s "
+                              "--test-runner=runner "
                               "--json - --job-timeout 1" % (AVOCADO, self.tmpdir.name, tst),
                               ignore_status=True)
             self.assertEqual(res.exit_status, exit_codes.AVOCADO_TESTS_FAIL)
@@ -295,6 +297,7 @@ def test_no_status_reported(self):
                                     DIE_WITHOUT_REPORTING_STATUS,
                                     "no_status_reported") as tst:
             res = process.run("%s run --disable-sysinfo --job-results-dir %s %s "
+                              "--test-runner=runner "
                               "--json -" % (AVOCADO, self.tmpdir.name, tst),
                               ignore_status=True)
             self.assertEqual(res.exit_status, exit_codes.AVOCADO_TESTS_FAIL)
@@ -400,6 +403,7 @@ def test_exception_not_in_path(self):
                                RAISE_CUSTOM_PATH_EXCEPTION_CONTENT)
         mytest.save()
         result = process.run("%s --show test run --disable-sysinfo "
+                             "--test-runner=runner "
                              "--job-results-dir %s %s"
                              % (AVOCADO, self.tmpdir.name, mytest))
         self.assertIn(b"mytest.py:SharedLibTest.test -> CancelExc: This "
@@ -432,6 +436,7 @@ def test_runner_abort(self):
         :avocado: tags=parallel:1
         """
         cmd_line = ('%s run --disable-sysinfo --job-results-dir %s '
+                    '--test-runner=runner '
                     '--xunit - examples/tests/abort.py' % (AVOCADO, self.tmpdir.name))
         result = process.run(cmd_line, ignore_status=True)
         excerpt = b'Test died without reporting the status.'
@@ -458,16 +463,17 @@ def test_empty_args_list(self):
                       result.stderr)
 
     def test_empty_test_list(self):
-        cmd_line = '%s run --disable-sysinfo --job-results-dir %s' \
-                   % (AVOCADO, self.tmpdir.name)
+        cmd_line = ('%s run --disable-sysinfo --job-results-dir %s'
+                    '--test-runner=runner ' % (AVOCADO, self.tmpdir.name))
         result = process.run(cmd_line, ignore_status=True)
         self.assertEqual(result.exit_status, exit_codes.AVOCADO_JOB_FAIL)
         self.assertIn(b'No test references provided nor any other arguments '
                       b'resolved into tests', result.stderr)
 
     def test_not_found(self):
-        cmd_line = ('%s run --disable-sysinfo --job-results-dir %s sbrubles'
-                    % (AVOCADO, self.tmpdir.name))
+        cmd_line = ('%s run --disable-sysinfo --job-results-dir %s '
+                    '--test-runner=runner '
+                    'sbrubles' % (AVOCADO, self.tmpdir.name))
         result = process.run(cmd_line, ignore_status=True)
         self.assertEqual(result.exit_status, exit_codes.AVOCADO_JOB_FAIL)
         self.assertIn(b'Unable to resolve reference', result.stderr)
@@ -526,7 +532,9 @@ def test_invalid_python(self):
         test = script.make_script(os.path.join(self.tmpdir.name, 'test.py'),
                                   INVALID_PYTHON_TEST)
         cmd_line = ('%s --show test run --disable-sysinfo '
-                    '--job-results-dir %s %s') % (AVOCADO, self.tmpdir.name, test)
+                    '--job-results-dir %s '
+                    '--test-runner=runner '
+                    '%s') % (AVOCADO, self.tmpdir.name, test)
         result = process.run(cmd_line, ignore_status=True)
         expected_rc = exit_codes.AVOCADO_TESTS_FAIL
         self.assertEqual(result.exit_status, expected_rc,
@@ -575,6 +583,7 @@ def test_other_loggers(self):
                 'avocado_functional_test_other_loggers') as mytest:
 
             cmd_line = ('%s run --disable-sysinfo --job-results-dir %s '
+                        '--test-runner=runner '
                         '-- %s' % (AVOCADO, self.tmpdir.name, mytest))
             result = process.run(cmd_line, ignore_status=True)
             expected_rc = exit_codes.AVOCADO_ALL_OK
@@ -671,6 +680,7 @@ def test_output_cancel(self):
     def test_ugly_echo_cmd(self):
         cmd_line = ('%s --show=test run --external-runner "%s -ne" '
                     '"foo\\\\\\n\\\'\\\\\\"\\\\\\nbar/baz" --job-results-dir %s'
+                    ' --test-runner=runner '
                     ' --disable-sysinfo' %
                     (AVOCADO, GNU_ECHO_BINARY, self.tmpdir.name))
         result = process.run(cmd_line, ignore_status=True)
@@ -699,6 +709,7 @@ def test_replay_skip_skipped(self):
         result = json.loads(result.stdout_text)
         jobid = str(result["job_id"])
         cmd = ("%s run --job-results-dir %s --replay %s "
+               "--test-runner=runner "
                "--replay-test-status PASS" % (AVOCADO, self.tmpdir.name, jobid))
         process.run(cmd)
 
@@ -791,6 +802,7 @@ def test_simplewarning(self):
         # access to an installed location for the libexec scripts
         os.environ['PATH'] += ":" + os.path.join(BASEDIR, 'libexec')
         cmd_line = ('%s --show=test run --job-results-dir %s --disable-sysinfo '
+                    '--test-runner=runner '
                     'examples/tests/simplewarning.sh'
                     % (AVOCADO, self.tmpdir.name))
         result = process.run(cmd_line, ignore_status=True)
@@ -817,6 +829,7 @@ def test_fs_unfriendly_run(self):
                            "[sysinfo.collectibles]\ncommands = %s"
                            % commands_path)
         cmd_line = ("%s --show all --config %s run --job-results-dir %s "
+                    "--test-runner=runner "
                     "--external-runner %s -- \"'\\\"\\/|?*<>'\""
                     % (AVOCADO, config_path, self.tmpdir.name, GNU_ECHO_BINARY))
         process.run(cmd_line)
@@ -848,6 +861,7 @@ def test_non_absolute_path(self):
         os.chdir(test_base_dir)
         test_file_name = os.path.basename(self.pass_script.path)
         cmd_line = ('%s run --job-results-dir %s --disable-sysinfo'
+                    ' --test-runner=runner'
                     ' "%s"' % (AVOCADO, self.tmpdir.name,
                                test_file_name))
         result = process.run(cmd_line, ignore_status=True)
@@ -864,6 +878,7 @@ def test_kill_stopped_sleep(self):
         :avocado: tags=parallel:1
         """
         proc = aexpect.Expect("%s run 60 --job-results-dir %s "
+                              "--test-runner=runner "
                               "--external-runner %s --disable-sysinfo "
                               "--job-timeout 3"
                               % (AVOCADO, self.tmpdir.name, SLEEP_BINARY))
@@ -933,6 +948,7 @@ def test_simpletest_status(self):
                                              'functional')
         warn_script.save()
         cmd_line = ('%s --config %s run --job-results-dir %s --disable-sysinfo'
+                    ' --test-runner=runner'
                     ' %s --json -' % (AVOCADO, self.config_file.path,
                                       self.tmpdir.name, warn_script.path))
         result = process.run(cmd_line, ignore_status=True)
@@ -946,6 +962,7 @@ def test_simpletest_status(self):
                                              'functional')
         skip_script.save()
         cmd_line = ('%s --config %s run --job-results-dir %s --disable-sysinfo'
+                    ' --test-runner=runner'
                     ' %s --json -' % (AVOCADO, self.config_file.path,
                                       self.tmpdir.name, skip_script.path))
         result = process.run(cmd_line, ignore_status=True)
@@ -959,6 +976,7 @@ def test_simpletest_status(self):
                                               'functional')
         skip2_script.save()
         cmd_line = ('%s --config %s run --job-results-dir %s --disable-sysinfo'
+                    ' --test-runner=runner'
                     ' %s --json -' % (AVOCADO, self.config_file.path,
                                       self.tmpdir.name, skip2_script.path))
         result = process.run(cmd_line, ignore_status=True)
@@ -1007,6 +1025,7 @@ def setUp(self):
     def test_simpletest_failure_fields(self):
         fail_test = os.path.join(BASEDIR, 'examples', 'tests', 'failtest.sh')
         cmd_line = ('%s --config %s run --job-results-dir %s --disable-sysinfo'
+                    ' --test-runner=runner'
                     ' -- %s' % (AVOCADO, self.config_file.path,
                                 self.tmpdir.name, fail_test))
         result = process.run(cmd_line, ignore_status=True)
@@ -1037,6 +1056,7 @@ def setUp(self):
 
     def test_externalrunner_pass(self):
         cmd_line = ('%s run --job-results-dir %s --disable-sysinfo '
+                    '--test-runner=runner '
                     '--external-runner=/bin/sh %s'
                     % (AVOCADO, self.tmpdir.name, self.pass_script.path))
         result = process.run(cmd_line, ignore_status=True)
@@ -1047,6 +1067,7 @@ def test_externalrunner_pass(self):
 
     def test_externalrunner_fail(self):
         cmd_line = ('%s run --job-results-dir %s --disable-sysinfo '
+                    '--test-runner=runner '
                     '--external-runner=/bin/sh %s'
                     % (AVOCADO, self.tmpdir.name, self.fail_script.path))
         result = process.run(cmd_line, ignore_status=True)
@@ -1057,6 +1078,7 @@ def test_externalrunner_fail(self):
 
     def test_externalrunner_chdir_no_testdir(self):
         cmd_line = ('%s run --job-results-dir %s --disable-sysinfo '
+                    '--test-runner=runner '
                     '--external-runner=/bin/sh --external-runner-chdir=test %s'
                     % (AVOCADO, self.tmpdir.name, self.pass_script.path))
         result = process.run(cmd_line, ignore_status=True)
@@ -1073,6 +1095,7 @@ def test_externalrunner_chdir_runner_relative(self):
         pass_abs = os.path.abspath(self.pass_script.path)
         os.chdir('/')
         cmd_line = ('%s run --job-results-dir %s --disable-sysinfo '
+                    '--test-runner=runner '
                     '--external-runner=bin/sh --external-runner-chdir=runner -- %s'
                     % (AVOCADO, self.tmpdir.name, pass_abs))
         result = process.run(cmd_line, ignore_status=True)
@@ -1083,6 +1106,7 @@ def test_externalrunner_chdir_runner_relative(self):
 
     def test_externalrunner_no_url(self):
         cmd_line = ('%s run --job-results-dir %s --disable-sysinfo '
+                    '--test-runner=runner '
                     '--external-runner=%s' % (AVOCADO, self.tmpdir.name, TRUE_CMD))
         result = process.run(cmd_line, ignore_status=True)
         expected_output = (b'No test references provided nor any other '
@@ -1301,6 +1325,7 @@ def setUp(self):
     def run_and_check(self, testname, e_rc, e_ntests, e_nerrors,
                       e_nfailures, e_nskip):
         cmd_line = ('%s run --job-results-dir %s --disable-sysinfo'
+                    ' --test-runner=runner'
                     ' --xunit - %s' % (AVOCADO, self.tmpdir.name, testname))
         result = process.run(cmd_line, ignore_status=True)
         xml_output = result.stdout
@@ -1374,8 +1399,9 @@ class PluginsJSONTest(TestCaseTmpDir):
 
     def run_and_check(self, testname, e_rc, e_ntests, e_nerrors,
                       e_nfailures, e_nskip, e_ncancel=0, external_runner=None):
-        cmd_line = ('%s run --job-results-dir %s --disable-sysinfo --json - '
-                    '--archive %s' % (AVOCADO, self.tmpdir.name, testname))
+        cmd_line = ('%s run --job-results-dir %s --disable-sysinfo '
+                    '--test-runner=runner '
+                    '--json - --archive %s' % (AVOCADO, self.tmpdir.name, testname))
         if external_runner is not None:
             cmd_line += " --external-runner '%s'" % external_runner
         result = process.run(cmd_line, ignore_status=True)

selftests/functional/test_export_variables.py

@@ -36,6 +36,7 @@ def setUp(self):
     def test_environment_vars(self):
         os.chdir(BASEDIR)
         cmd_line = ('%s run --job-results-dir %s %s'
+                    ' --test-runner=runner'
                     % (AVOCADO, self.tmpdir.name, self.script.path))
         result = process.run(cmd_line, ignore_status=True)
         expected_rc = exit_codes.AVOCADO_ALL_OK

selftests/functional/test_fetch_asset.py

@@ -61,9 +61,11 @@ def test_asset_fetch_find_success(self):
         test_file.close()
 
         expected_rc = exit_codes.AVOCADO_ALL_OK
-        cmd_line = "%s --config %s run %s " % (AVOCADO,
-                                               self.config_file.name,
-                                               test_file.name)
+        cmd_line = ("%s --config %s run "
+                    "--test-runner=runner "
+                    "%s" % (AVOCADO,
+                            self.config_file.name,
+                            test_file.name))
         result = process.run(cmd_line)
         os.remove(localpath)
         self.assertEqual(expected_rc, result.exit_status)
@@ -92,9 +94,11 @@ def test_asset_fetch_find_fail(self):
         expected_rc = exit_codes.AVOCADO_TESTS_FAIL
         expected_stdout = "not found in the cache"
 
-        cmd_line = "%s --config %s run %s " % (AVOCADO,
-                                               self.config_file.name,
-                                               test_file.name)
+        cmd_line = ("%s --config %s run "
+                    "--test-runner=runner "
+                    "%s" % (AVOCADO,
+                            self.config_file.name,
+                            test_file.name))
 
         result = process.run(cmd_line, ignore_status=True)
         self.assertEqual(expected_rc, result.exit_status)
@@ -123,9 +127,11 @@ def test_asset_fetch_find_fail_cancel(self):
         expected_rc = exit_codes.AVOCADO_ALL_OK
         expected_stdout = "Missing asset"
 
-        cmd_line = "%s --config %s run %s " % (AVOCADO,
-                                               self.config_file.name,
-                                               test_file.name)
+        cmd_line = ("%s --config %s run "
+                    "--test-runner=runner "
+                    "%s" % (AVOCADO,
+                            self.config_file.name,
+                            test_file.name))
 
         result = process.run(cmd_line, ignore_status=True)
         self.assertEqual(expected_rc, result.exit_status)
@@ -153,9 +159,11 @@ def test_asset_fetch_cancel(self):
         expected_rc = exit_codes.AVOCADO_ALL_OK
         expected_stdout = "Missing asset"
 
-        cmd_line = "%s --config %s run %s " % (AVOCADO,
-                                               self.config_file.name,
-                                               test_file.name)
+        cmd_line = ("%s --config %s run "
+                    "--test-runner=runner "
+                    "%s " % (AVOCADO,
+                             self.config_file.name,
+                             test_file.name))
 
         result = process.run(cmd_line, ignore_status=True)
         self.assertEqual(expected_rc, result.exit_status)

selftests/functional/test_getdata.py

@@ -20,6 +20,7 @@ def test_varianter(self):
         test_variants_path = os.path.join(BASEDIR, "selftests", ".data",
                                           "get_data.py.data", "get_data.json")
         cmd_line = ("%s run --disable-sysinfo --job-results-dir '%s' "
+                    "--test-runner=runner "
                     "--json-variants-load %s -- %s")
         cmd_line %= (AVOCADO, self.tmpdir.name, test_variants_path, test_path)
         result = process.run(cmd_line)

selftests/functional/test_interrupt.py

@@ -112,8 +112,9 @@ def test_badly_behaved_sigint(self):
         bad_test.save()
         self.test_module = bad_test.path
         os.chdir(BASEDIR)
-        cmd = ('%s run %s --disable-sysinfo --job-results-dir %s ' %
-               (AVOCADO, self.test_module, self.tmpdir.name))
+        cmd = ('%s run %s --disable-sysinfo --job-results-dir %s '
+               '--test-runner=runner'
+               % (AVOCADO, self.test_module, self.tmpdir.name))
         proc = subprocess.Popen(cmd.split(),
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)