From 1fa955d5870ea8e2e5bab2c08c2084b33df02caf Mon Sep 17 00:00:00 2001
From: Max Horn <max@quendi.de>
Date: Fri, 5 Oct 2018 12:24:35 +0200
Subject: [PATCH] TestDirectory reports number of failures and failed files

With this, you may get output like this, with the
relevant change in the "total" line near the end.

    gap> TestDirectory("misc");
    Architecture: x86_64-apple-darwin15.6.0-default64-kv5

    testing: misc/err.tst
           0 ms (0 ms GC) and 46.6KB allocated for err.tst
    testing: misc/foo.tst
    ########> Diff in misc/foo.tst:3
    # Input is:
    1+1;
    # Expected output:
    3
    # But found:
    2
    ########
           0 ms (0 ms GC) and 6.62KB allocated for foo.tst
    -----------------------------------
    total         0 ms (0 ms GC) and 53.2KB allocated, 1 failures in 1 out of 2 files

    #I  Errors detected while testing

    false
    gap>

Resolves #2861
---
 lib/test.gi | 44 +++++++++++++++++++++++++++++++-------------
 1 file changed, 31 insertions(+), 13 deletions(-)

diff --git a/lib/test.gi b/lib/test.gi
index 7cb01dfdd5..7b8aa364e6 100644
--- a/lib/test.gi
+++ b/lib/test.gi
@@ -273,6 +273,11 @@ end;
 ##  line breaks "\r\n" by UNIX style line breaks "\n" after reading the test
 ##  file. (default is <K>true</K>).</Item>
 ##  </List>
+##  <Mark><C>returnNumFailures</C></Mark>
+##  <Item>If this is <K>true</K> then &GAP; returns the number of input
+##  lines of the test file which had differences in their output, instead
+##  of returning <K>true</K> or <K>false</K>.</Item>
+##  </List>
 ## 
 ##  <Log><![CDATA[
 ##  gap> tnam := Filename(DirectoriesLibrary(), "../doc/ref/demo.tst");;
@@ -324,7 +329,7 @@ end;
 ##  <#/GAPDoc>
 ##  
 InstallGlobalFunction("Test", function(arg)
-  local fnam, nopts, opts, size, full, pf, ret, lines, ign, new, n, 
+  local fnam, nopts, opts, size, full, pf, failures, lines, ign, new, n,
         cT, ok, oldtimes, thr, delta, len, c, i, j, d;
   
   # get arguments and set options
@@ -374,6 +379,7 @@ InstallGlobalFunction("Test", function(arg)
              Print("########\n");
            end,
            subsWindowsLineBreaks := true,
+           returnNumFailures := false,
          );
   if not IS_OUTPUT_TTY() then
     opts.showProgress := false;
@@ -432,12 +438,12 @@ InstallGlobalFunction("Test", function(arg)
   SizeScreen(size);
 
   # check for and report differences
-  ret := true;
+  failures := 0;
   for i in [1..Length(pf[1])] do
     if opts.compareFunction(pf[2][i], pf[4][i]) <> true then
       if not opts.ignoreSTOP_TEST or 
          PositionSublist(pf[1][i], "STOP_TEST") <> 1 then
-        ret := false;
+        failures := failures + 1;
         opts.reportDiff(pf[1][i], pf[2][i], pf[4][i], fnam, pf[3][i], pf[5][i]);
       else
         # print output of STOP_TEST
@@ -521,8 +527,13 @@ InstallGlobalFunction("Test", function(arg)
   # store internal test data in TEST
   TEST.lastTestData := pf;
 
+  # if requested, return number of failures
+  if opts.returnNumFailures then
+    return failures;
+  fi;
+
   # return true/false
-  return ret;
+  return (failures = 0);
 end);
 
 
@@ -575,14 +586,15 @@ end);
 ##  </ManSection>
 ##  <#/GAPDoc>
 InstallGlobalFunction( "TestDirectory", function(arg)
-    local  testTotal, totalTime, totalMem, STOP_TEST_CPY, 
+    local  testTotalFailures, testFailedFiles, totalTime, totalMem, STOP_TEST_CPY,
            basedirs, nopts, opts, testOptions, earlyStop, 
            showProgress, suppressStatusMessage, exitGAP, c, files, 
            filetimes, filemems, recurseFiles, f, i, startTime, 
            startMem, testResult, time, mem, startGcTime, gctime,
            totalGcTime, filegctimes, GcTime;
 
-  testTotal := true;
+  testTotalFailures := 0;
+  testFailedFiles := 0;
   totalTime := 0;
   totalMem := 0;
   totalGcTime := 0;
@@ -621,7 +633,7 @@ InstallGlobalFunction( "TestDirectory", function(arg)
     opts.(c) := nopts.(c);
   od;
   opts.exclude := Set(opts.exclude);
-  
+  opts.testOptions.returnNumFailures := true;
   
   if opts.exitGAP then
     GAP_EXIT_CODE(1);
@@ -690,7 +702,7 @@ InstallGlobalFunction( "TestDirectory", function(arg)
       opts.testOptions.rewriteToFile := files[i].name;
     fi;
     testResult := Test(files[i].name, opts.testOptions);
-    if not(testResult) and opts.earlyStop then
+    if (testResult <> 0) and opts.earlyStop then
       STOP_TEST := STOP_TEST_CPY;
       if not opts.suppressStatusMessage then
         # Do not change the next line - it is needed for testing scrips
@@ -701,7 +713,10 @@ InstallGlobalFunction( "TestDirectory", function(arg)
       fi;
       return false;
     fi;
-    testTotal := testTotal and testResult;
+    if testResult <> 0 then
+      testFailedFiles := testFailedFiles + 1;
+    fi;
+    testTotalFailures := testTotalFailures + testResult;
     
     time := Runtime() - startTime;
     mem := TotalMemoryAllocated() - startMem;    
@@ -725,10 +740,13 @@ InstallGlobalFunction( "TestDirectory", function(arg)
   Print("-----------------------------------\n");
   Print( "total",
          String( totalTime, 10 ), " ms (",String( totalGcTime )," ms GC) and ", 
-         StringOfMemoryAmount( totalMem )," allocated\n\n" );
+         StringOfMemoryAmount( totalMem )," allocated, ",
+         testTotalFailures, " failures in ",
+         testFailedFiles, " of ", Length(files), " files",
+         "\n\n" );
 
   if not opts.suppressStatusMessage then
-    if testTotal then
+    if testTotalFailures = 0 then
       # Do not change the next line - it is needed for testing scrips
       Print( "#I  No errors detected while testing\n\n" );
     else
@@ -738,14 +756,14 @@ InstallGlobalFunction( "TestDirectory", function(arg)
   fi;
 
   if opts.exitGAP then
-    if testTotal then
+    if testTotalFailures = 0 then
       QUIT_GAP(0);
     else
       QUIT_GAP(1);
     fi;
   fi;
   
-  return testTotal;
+  return testTotalFailures = 0;
 end);
 
 #############################################################################