Merge branch 'topic/419-prepro-comments' into 'master'

Instrument.C: Fix pre-processing of sources where comments cannot be preserved

Closes #419

See merge request eng/das/cov/gnatcoverage!846

Some macros can perform token concatenation, but do not allow concatenating comments with other tokens. The comments are normally replaced by white spaces before macro expansion, but gnatcov needs to preserve comments to process annotations. In this case the pre-processor will error out when asked to preserve the comments.

To fix this, this change introduces a retry without the -C argument. This allows gnatcov to properly instrument the source, but annotations will be ignored.

Closes eng/das/cov/gnatcoverage#419

Author: RenjiSann

doc/gnatcov/src_traces.rst

@@ -642,6 +642,13 @@ it is required to remove any :cmd-option:`-include` switch that is passed to
 the compiler invocation, by having a dedicated scenario variable for a coverage
 build for instance.
 
+To support annotations such as :ref:`exemptions<exemptions>` or :ref:`buffer
+output indications<manual_dump>`, the |gcvins| attempts to preserve comments
+when preprocessing the sources. If this is not possible (e.g. due to illegal
+concatenation of a comment with another token), the tool will emit a warning
+and ignore any in-source annotation for the mentioned file, as well as any
+other transitively included file.
+
 Function and call coverage limitations
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

testsuite/tests/instr-cov/419-comment-preprocessing/src/pkg.h

@@ -0,0 +1,9 @@
+#define CONCAT(X, Y) X##Y
+
+inline int
+ident (int x)
+{
+  /* GNATCOV_EXEMPT_ON unused*/
+  return 0; // # noeval
+  /*GNATCOV_EXEMPT_OFF */
+}

testsuite/tests/instr-cov/419-comment-preprocessing/src/test_pkg.c

@@ -0,0 +1,16 @@
+#include "pkg.h"
+
+int
+main ()
+{
+  int CONCAT (/*empty*/, my_var) = 0;         // # st
+  return CONCAT (/*something else*/, my_var); // # st
+}
+
+//# test_pkg.c
+//
+// /st/ l+ ## 0
+
+//# pkg.h
+//
+// /noeval/ l- ## s-

testsuite/tests/instr-cov/419-comment-preprocessing/test.opt

@@ -0,0 +1 @@
+bin-traces DEAD Check instrumentation-specific behavior

testsuite/tests/instr-cov/419-comment-preprocessing/test.py

@@ -0,0 +1,34 @@
+"""
+Test that gnatcov does not crash when instrumenting a source that cannot be
+preprocessed while preserving comments, and instead emits a warning stating
+that annotations are ignored.
+"""
+
+import os
+
+from SCOV.tc import TestCase
+from SCOV.tctl import CAT
+from SUITE.context import thistest
+from SUITE.cutils import contents_of
+
+wd = "s_pkg"
+
+warning_pat = (
+    r"warning: Could not preserve comments while pre-processing .*test_pkg\.c"
+    r", annotations in comments within this file or included headers will not"
+    r" be taken into account"
+)
+
+TestCase(
+    category=CAT.stmt,
+    tolerate_messages=warning_pat,
+).run()
+
+# Actually check that the warning got emitted
+thistest.fail_if_no_match(
+    what="missing warning in the output of 'gnatcov instrument'",
+    regexp=warning_pat,
+    actual=contents_of(os.path.join(wd, "xinstr.out")),
+)
+
+thistest.result()

tools/gnatcov/instrument-c.adb

@@ -3019,8 +3019,11 @@ package body Instrument.C is
       PP_Filename  : out Unbounded_String;
       Options      : in out Analysis_Options)
    is
+      Base_Cmd : Command_Type;
+      --  The command to preprocess the file that we save
+
       Cmd : Command_Type;
-      --  The command to preprocess the file
+      --  The actual command to preprocess the file
 
       Success : Boolean;
       --  Whether this command is successful
@@ -3050,38 +3053,42 @@ package body Instrument.C is
          return;
       end if;
 
-      Cmd :=
+      Base_Cmd :=
         (Command => Prj.Compiler_Driver (Instrumenter.Language),
          others  => <>);
 
       --  Add the preprocessing flag
 
-      Append_Arg (Cmd, "-E");
-
-      --  Keep the code comments as they may contain exemption / coverage
-      --  disabling markers.
-
-      Append_Arg (Cmd, "-C");
+      Append_Arg (Base_Cmd, "-E");
 
       --  Add all of the compiler / compiler driver options to the
       --  preprocessing command line.
 
-      Append_Args (Cmd, Options.Raw_Switches);
+      Append_Args (Base_Cmd, Options.Raw_Switches);
 
-      Append_Arg (Cmd, Filename);
+      Append_Arg (Base_Cmd, Filename);
 
       --  Register the preprocessing command. We need it to preprocess the file
       --  when producing the report, and getting the text of macro expansions.
       --  We don't need the options added afterwards, as they are just there
       --  for the instrumentation process (and we do not want to pass a -o
       --  option, as it would make paths too opaque at gnatcov coverage time).
 
+      --  Keep the code comments as they may contain exemption / coverage
+      --  disabling markers. This argument needs to be kept last in Base_Cmd as
+      --  we may need to remove it later if preprocessing with comments
+      --  preserved fails.
+
+      Append_Arg (Base_Cmd, "-C");
+
       PP_Cmds.Insert
         (Get_Index_From_Generic_Name
            (Filename,
             Kind                => Files_Table.Source_File,
             Indexed_Simple_Name => True),
-         Cmd);
+         Base_Cmd);
+
+      Cmd := Base_Cmd;
 
       --  To get the include paths, we use the verbose output of cpp -E
 
@@ -3098,6 +3105,60 @@ package body Instrument.C is
          Output_File         => Preprocessor_Output_Filename,
          Ignore_Error        => True);
 
+      if not Success then
+
+         --  If it did not succeed, it might be due to comment preservation
+         --  (-C), which can result in invalid tokens being generated by
+         --  token-manipulating macros. E.G:
+         --
+         --  #define FOO (X, Y) X ## Y
+         --
+         --  int FOO(/*my*/, integer);
+         --
+         --  Without preserving comments the above declaration expands to int
+         --  integer;
+         --
+         --  But preserving comments makes the preprocessor error out as a
+         --  comment token is concatenated to something else, which is not
+         --  allowed.
+         --
+         --  In that case, retry preprocessing without preserving comment, but
+         --  emit a warning as we'll loose any annotations in the preprocessed
+         --  files or included headers.
+
+         Base_Cmd.Arguments.Delete_Last;
+
+         PP_Cmds.Replace
+           (Get_Index_From_Generic_Name
+              (Filename,
+               Kind                => Files_Table.Source_File,
+               Indexed_Simple_Name => True),
+            Base_Cmd);
+
+         Cmd := Base_Cmd;
+
+         Append_Arg (Cmd, "-v");
+         Append_Arg (Cmd, "-o");
+         Append_Arg (Cmd, Preprocessed_Filename);
+
+         Success :=
+           Run_Command
+             (Command             => Cmd,
+              Origin_Command_Name => "Preprocessing",
+              Output_File         => Preprocessor_Output_Filename,
+              Ignore_Error        => True);
+
+         --  If this preprocessing command succeeded, warn the user that we
+         --  will not be able to process in-source annotations for this file
+         --  and included headers.
+
+         if Success then
+            Warn
+              ("Could not preserve comments while pre-processing " & Filename
+               & ", annotations in comments within this file or included"
+               & " headers will not be taken into account");
+         end if;
+      end if;
       --  Clear the search path so that we populate it from the include search
       --  paths in the logs.