testsuite: check warnings in gnatcov output

Check warnings for every gnatcov command rather than for "gnatcov
instrument" only. Fix the fallout regressions by adapting the testcases.

Also remove unused auto_languages parameter from the xcov function.

Author: Jugst3r

testsuite/Qualif/Common/UnitsOfInterest/GPR/Robustness/BadUnitInAttr/test.py

@@ -21,8 +21,9 @@
     ),
     covlevel="stmt",
     mains=["main"],
-    tolerate_instrument_messages="no unit.*in project",
+    tolerate_instrument_messages="no unit .* in project",
     extra_coverage_args=["-axcov"],
+    tolerate_coverage_messages="no unit .* in project",
 )
 
 log_file = (

testsuite/Qualif/Common/UnitsOfInterest/GPR/Robustness/BadUnitInSwitch/test.py

@@ -18,8 +18,9 @@
     ),
     covlevel="stmt",
     mains=["main"],
-    tolerate_instrument_messages="no unit.*in the projects of interest",
+    tolerate_instrument_messages="no unit .* in the projects of interest",
     extra_coverage_args=["-axcov"],
+    tolerate_coverage_messages="no unit .* in the projects of interest",
 )
 
 log_file = (

testsuite/Qualif/Common/UnitsOfInterest/GPR/Robustness/MissingLI/test.py

@@ -41,7 +41,7 @@
     what = "ALI"
 
 log_file = "coverage.log"
-xcov(xcov_args, out=log_file)
+xcov(xcov_args, out=log_file, tolerate_messages=".")
 thistest.fail_if_not_equal(
     "gnatcov output",
     "warning: no {} file found for unit helper".format(what),

testsuite/Qualif/Common/UnitsOfInterest/GPR/Robustness/NoContribPrj/test.py

@@ -103,8 +103,9 @@ def run_test(
         covlevel="stmt",
         mains=["main"],
         gpr_obj_dir="obj-main",
-        tolerate_instrument_messages="project.*provides no unit",
+        tolerate_instrument_messages="project .* provides no unit",
         extra_coverage_args=["-axcov"],
+        tolerate_coverage_messages="project .* provides no unit",
     )
 
     log_file = (

testsuite/Qualif/Common/UnitsOfInterest/GPR/Robustness/SourcesAttr/test.py

@@ -44,6 +44,7 @@ def try_one(subdir, extra_covargs, xreports, xwarnings):
         covlevel="stmt",
         mains=["test_t"],
         extra_coverage_args=["--annotate=xcov"] + extra_covargs,
+        tolerate_coverage_messages=".",
     )
 
     check_xcov_reports("obj", xreports)

testsuite/Qualif/Common/UnitsOfInterest/GPR/Robustness/UnitNotInProjects/test.py

@@ -43,8 +43,9 @@
     covlevel="stmt",
     mains=["main"],
     gpr_obj_dir="obj-main",
-    tolerate_instrument_messages="no unit.*in the projects",
+    tolerate_instrument_messages="no unit .* in the projects",
     extra_coverage_args=["-axcov"],
+    tolerate_coverage_messages="no unit .* in the projects",
 )
 
 log_file = (

testsuite/SCOV/instr.py

@@ -7,7 +7,7 @@
 
 from SUITE.context import thistest
 from SUITE.control import env
-from SUITE.cutils import contents_of, copy_to_dir, ext, indent
+from SUITE.cutils import contents_of, ext, copy_to_dir
 from SUITE.tutils import RUNTIME_INFO, GNATCOV_INFO, locate_gpr_file, xcov
 
 
@@ -58,7 +58,6 @@ def xcov_instrument(
     register_failure=True,
     auto_config_args=True,
     auto_target_args=True,
-    auto_languages=True,
 ):
     """
     Run "gnatcov instrument" on a project.
@@ -87,7 +86,6 @@ def xcov_instrument(
     :param bool register_failure: See SUITE.tutils.xcov.
     :param bool auto_config_args: See SUITE.tutils.xcov.
     :param bool auto_target_args: See SUITE.tutils.xcov.
-    :param bool auto_languages: See SUITE.tutils.xcov.
 
     See SUITE.tutils.xcov for the other supported options.
     """
@@ -137,20 +135,9 @@ def xcov_instrument(
     if thistest.options.block:
         args.append("--instrument-block")
 
-    out = out or "instrument.log"
-    result = xcov(
-        args,
-        out=out,
-        err=err,
-        register_failure=register_failure,
-        auto_config_args=auto_config_args,
-        auto_target_args=auto_target_args,
-        auto_languages=auto_languages,
-    )
-
     # When no message is to be tolerated, fallback to an actual regexp
     # that will never match:
-    re_tolerate_messages = tolerate_messages or "__NEVER_IN_A_WARNING___"
+    re_tolerate_messages = tolerate_messages or ""
 
     # For qualification purposes, tolerate possible warnings about
     # inexistant object dirs from older gnatcov versions, typically
@@ -163,36 +150,16 @@ def xcov_instrument(
             for mre in ["object directory.*not found", re_tolerate_messages]
         )
 
-    if register_failure:
-        output = contents_of(out)
-
-        # Check for unexpected messages. Beware that the "warning:"
-        # indication at least is not necessarily at the beginning of
-        # a line, as in
-        #
-        #    app.gpr:4:23: warning: object directory "obj" not found
-
-        messages = re.findall(
-            pattern=r"(?:!!!|\*\*\*|warning:).*$",
-            string=output,
-            flags=re.MULTILINE,
-        )
-
-        unexpected_messages = [
-            w
-            for w in messages
-            if not re.search(pattern=re_tolerate_messages, string=w)
-        ]
-        thistest.fail_if(
-            unexpected_messages,
-            f"Unexpected messages in the output of 'gnatcov instrument':"
-            f"\n{indent(output)}"
-            + (
-                f"\n(allowed: {tolerate_messages})"
-                if tolerate_messages
-                else ""
-            ),
-        )
+    out = out or "instrument.log"
+    result = xcov(
+        args,
+        out=out,
+        err=err,
+        register_failure=register_failure,
+        auto_config_args=auto_config_args,
+        auto_target_args=auto_target_args,
+        tolerate_messages=re_tolerate_messages,
+    )
 
     return result
 

testsuite/SCOV/minicheck.py

@@ -68,7 +68,6 @@ def build_and_run(
     program_env=None,
     tolerate_instrument_messages=None,
     exec_args=None,
-    auto_languages=True,
     manual_prj_name=None,
     auto_config_args=True,
 ):
@@ -142,7 +141,6 @@ def build_and_run(
         output.
     :param None|list[str] exec_args: List of arguments to pass to the
         executable. This will only work for native configurations.
-    :param bool auto_languages: See SUITE.tutils.xcov.
     :param None|str manual_prj_name: When the dump trigger is manual, several
         traces files (one per project) can be emitted if there are dump buffers
         procedure calls in at least two distinct projects. This is the name of
@@ -283,7 +281,6 @@ def gprbuild_wrapper(root_project):
             out="instrument.log",
             register_failure=register_failure,
             tolerate_messages=tolerate_instrument_messages,
-            auto_languages=auto_languages,
             auto_config_args=auto_config_args,
         )
         gprbuild_wrapper(gprsw.root_project)
@@ -431,6 +428,7 @@ def build_run_and_coverage(
     err=None,
     register_failure=True,
     auto_config_args=True,
+    tolerate_coverage_messages=None,
     **kwargs,
 ):
     """
@@ -442,6 +440,10 @@ def build_run_and_coverage(
     `out` and `err` are forwarded to `xcov`, `register_failure` and
     `auto_config_args` are reported to `xcov` and `build_and_run`, other
     arguments are forwarded to `build_and_run`.
+
+    If tolerate_coverage_messages is not None, filter out error or warning
+    messages from the "gnatcov coverage" output matching the regular
+    expression.
     """
     xcov_args = build_and_run(
         register_failure=register_failure,
@@ -454,6 +456,7 @@ def build_run_and_coverage(
         out=out,
         err=err,
         register_failure=register_failure,
+        tolerate_messages=tolerate_coverage_messages,
     )
 
 

testsuite/SUITE/tutils.py

@@ -34,12 +34,12 @@
 )
 from SUITE.context import ROOT_DIR, thistest
 
-
 # Then mind our own business
 
 from SUITE.cutils import (
     FatalError,
     contents_of,
+    indent,
     list_to_file,
     text_to_file,
     to_list,
@@ -874,20 +874,27 @@ def xcov(
     auto_config_args=True,
     auto_target_args=True,
     force_project_args=False,
-    auto_languages=True,
+    tolerate_messages=None,
 ):
     """
-    Run xcov with arguments ARGS, timeout control, valgrind control if
-    available and enabled, output directed to OUT and failure registration if
-    register_failure is True. Return the process status descriptor. ARGS may be
-    a list or a whitespace separated strings.
-
-    If AUTO_LANGUAGES is True, the gnatcov sub-command is "instrument" and the
-    testsuite is not in qualification mode, automatically pass the
-    --restricted-to-languages argument to enable all the languages to test.
-
-    See xcov_suite_args for the meaning of AUTO_*_ARGS and FORCE_PROJECT_ARGS
-    arguments.
+    Run "gnatcov".
+
+    :param list[str] args: arguments to pass to gnatcov.
+    :param None|str out: if not None, redirect gnatcov output to out.
+    :param None|str err: if not None, redirect gnatcov error output to err.
+    :param None|str inp: if not None, redirect gnatcov input to inp.
+    :param None|dict[str, str] env: If not none, environment variables for the
+        program to run.
+    :param bool register_failure: If True, register execution failure.
+    :param bool auto_config_args: see xcov_suite_args.
+    :param bool auto_target_args: see xcov_suite_args.
+    :param bool force_project_args: see xcov_suite_args.
+    :param None|str tolerate_messages: If not None, a re pattern of warning
+        or error messsages tolerated in the tool output. Messages not matching
+        this pattern will cause a test failure when register_failure is True.
+
+    :rtype: Run
+    :return: Process status descriptor for the gnatcov invocation.
     """
 
     # Defensive code: running "gnatcov setup" with no prefix will install
@@ -941,6 +948,43 @@ def xcov(
         for_pgm=(covcmd == "run"),
     )
 
+    # When no message is to be tolerated, fallback to an actual regexp
+    # that will never match:
+    re_tolerate_messages = tolerate_messages or "__NEVER_IN_A_WARNING___"
+
+    # Do not check warnings when running the testsuite in binary traces mode,
+    # as it would require modifying / adapting too many tests.
+    if register_failure and thistest.options.trace_mode == "src":
+        output = contents_of(out) if out else p.out
+
+        # Check for unexpected messages. Beware that the "warning:"
+        # indication at least is not necessarily at the beginning of
+        # a line, as in
+        #
+        #    app.gpr:4:23: warning: object directory "obj" not found
+
+        messages = re.findall(
+            pattern=r"(?:!!!|\*\*\*|warning:).*$",
+            string=output,
+            flags=re.MULTILINE,
+        )
+
+        unexpected_messages = [
+            w
+            for w in messages
+            if not re.search(pattern=re_tolerate_messages, string=w)
+        ]
+        thistest.fail_if(
+            unexpected_messages,
+            f"Unexpected messages in the output of 'gnatcov {covcmd}':"
+            f"\n{indent(output)}"
+            + (
+                f"\n(allowed: {tolerate_messages})"
+                if tolerate_messages
+                else ""
+            ),
+        )
+
     if thistest.options.enable_valgrind == "memcheck":
         memcheck_log = contents_of(MEMCHECK_LOG)
         thistest.fail_if(
@@ -1008,6 +1052,7 @@ def xrun(
         register_failure=register_failure,
         auto_config_args=auto_config_args,
         auto_target_args=auto_target_args,
+        tolerate_messages=".",
     )
 
 
@@ -1271,6 +1316,7 @@ def xcov_annotate(
     register_failure=True,
     auto_config_args=True,
     auto_target_args=True,
+    tolerate_messages=None,
 ):
     """
     Invoke "gnatcov annotate" with the correct arguments to generate
@@ -1288,6 +1334,7 @@ def xcov_annotate(
         will be added. Defaults to None.
     :param list[str] | None  extra_args: extra arguments passed on the command
         line. Defaults to None.
+    :param str | None tolerate_messages: see documentation of xcov.
     """
     args = ["add-annotation"]
     args.extend(annotation.cmd_line_args())
@@ -1306,10 +1353,11 @@ def xcov_annotate(
         register_failure=register_failure,
         auto_config_args=auto_config_args,
         auto_target_args=auto_target_args,
+        tolerate_messages=tolerate_messages,
     )
 
 
-def generate_annotations(annotations, subdir=""):
+def generate_annotations(annotations, subdir="", tolerate_messages=None):
     """
     Setup a temporary working directory in which an annotation file
     will be generated from annotations, using gnatcov add-annotation
@@ -1334,6 +1382,9 @@ def generate_annotations(annotations, subdir=""):
     # Generate the annotations
     for annotation in annotations:
         xcov_annotate(
-            annotation, annot_in_files=[annot_file], annot_out_file=annot_file
+            annotation,
+            annot_in_files=[annot_file],
+            annot_out_file=annot_file,
+            tolerate_messages=tolerate_messages,
         )
     return annot_file

testsuite/tests/14-unused-separate/uncompilable/test.py

@@ -22,6 +22,7 @@
     covlevel="stmt",
     mains=["main"],
     extra_coverage_args=["--annotate=xcov"],
+    tolerate_coverage_messages="no ALI file found for unit pkg.bar",
 )
 
 thistest.fail_if(

testsuite/tests/15-non-instr-cons/test.py

@@ -40,7 +40,13 @@
 xcov(xcov_args + ["--save-checkpoint=test_f.ckpt", trace_f])
 
 # Try to consolidate the checkpoint
-xcov(xcov_args + ["-Ctest_t.ckpt", "-Ctest_f.ckpt", "-axcov"])
+xcov(
+    xcov_args + ["-Ctest_t.ckpt", "-Ctest_f.ckpt", "-axcov"],
+    tolerate_messages=(
+        "warning: Specifying units of interest through --units"
+        " has no effect on checkpoints"
+    ),
+)
 
 check_xcov_reports("obj", {"pkg.ads.xcov": {"?": {10}, "+": {5, 6, 7}}})
 

testsuite/tests/354-source-encoding/test.py

@@ -211,7 +211,11 @@ def check_report(
     argv = xcov_args + [f"--output-dir=out-{label}"]
     if encoding is not None:
         argv.append(f"--source-encoding={encoding}")
-    xcov(argv, out=log_file)
+
+    tolerate_messages = (
+        "|".join(expected_warnings) if expected_warnings else None
+    )
+    xcov(argv, out=log_file, tolerate_messages=tolerate_messages)
 
     # Check that we have exacly the expected warnings
     thistest.fail_if_not_equal(

testsuite/tests/IA21-009-nested-scos/test.opt

@@ -0,0 +1 @@
+src-traces DEAD Test specific to binary traces

testsuite/tests/IA21-009-nested-scos/test.py

@@ -5,5 +5,5 @@
 
 Wdir("tmp_")
 gprbuild(gprfor(["nested_same_sloc.adb"], srcdirs="../src"))
-xcov(["map-routines", "--scos=obj/nested_same_sloc.ali"])
+xcov(["map-routines", "--scos=obj/nested_same_sloc.ali", "nested_same_sloc"])
 thistest.result()

testsuite/tests/L613-012/test.opt

@@ -0,0 +1 @@
+src-traces DEAD Test specific to binary traces