update test reporting

cice.setup

@@ -941,9 +941,6 @@ EOF2
         exit -1
       endif
 
-#      # Initial test_output file
-#      echo "#---" >! test_output
-#      echo "PEND ${testname_noid} " >> test_output
     endif
 
     #------------------------------------------------------------
@@ -1004,31 +1001,55 @@ EOF0
   # Add code to results.csh to count the number of failures
   cat >> ${tsdir}/results.csh << EOF
 cat ./results.log
-set pends = \`cat ./results.log | grep PEND | wc -l\`
-set failures = \`cat ./results.log | grep FAIL | wc -l\`
-set success = \`cat ./results.log | grep 'PASS\|COPY' | wc -l\`
-set comments = \`cat ./results.log | grep "#" | wc -l\`
-set alltotal = \`cat ./results.log | wc -l\`
+set pends     = \`cat ./results.log | grep PEND | wc -l\`
+set misses    = \`cat ./results.log | grep MISS | wc -l\`
+set failures  = \`cat ./results.log | grep FAIL | wc -l\`
+set failbuild = \`cat ./results.log | grep FAIL | grep " build " | wc -l\`
+set failrun   = \`cat ./results.log | grep FAIL | grep " run " | wc -l\`
+set failtest  = \`cat ./results.log | grep FAIL | grep " test " | wc -l\`
+set failcomp  = \`cat ./results.log | grep FAIL | grep " compare " | wc -l\`
+set failbfbc  = \`cat ./results.log | grep FAIL | grep " bfbcomp " | wc -l\`
+set failgen   = \`cat ./results.log | grep FAIL | grep " generate " | wc -l\`
+set success   = \`cat ./results.log | grep 'PASS\|COPY' | wc -l\`
+set comments  = \`cat ./results.log | grep "#" | wc -l\`
+set alltotal  = \`cat ./results.log | wc -l\`
 @ total = \$alltotal - \$comments
+@ chkcnt = \$pends + \$misses + \$failures + \$success
 
 echo "#------- " >> results.log
 echo " " >> results.log
-echo "#totl = \$total" >> results.log
+echo "#totl = \$total total" >> results.log
+echo "#chkd = \$chkcnt checked" >> results.log
 echo "#pass = \$success" >> results.log
-echo "#fail = \$failures" >> results.log
 echo "#pend = \$pends" >> results.log
+echo "#miss = \$misses" >> results.log
+echo "#fail = \$failures" >> results.log
+echo "    #failbuild = \$failbuild" >> results.log
+echo "    #failrun   = \$failrun" >> results.log
+echo "    #failtest  = \$failtest" >> results.log
+echo "    #failcomp  = \$failcomp" >> results.log
+echo "    #failbfbc  = \$failbfbc" >> results.log
+echo "    #failgen   = \$failgen" >> results.log
 
 echo ""
 echo "Descriptors:"
 echo " PASS - successful completion"
 echo " COPY - previously compiled code was copied for new test"
 echo " MISS - comparison data is missing"
-echo " PEND - run has been submitted to queue and is waiting or failed submission"
-echo " FAIL - test is still executing, did not complete, or completed and failed"
+echo " PEND - status is undertermined; test may still be queued, running, or timed out"
+echo " FAIL - test failed"
 echo ""
-echo "\$success of \$total tests PASSED"
-echo "\$failures of \$total tests FAILED"
-echo "\$pends of \$total tests PENDING"
+echo "\$chkcnt measured results of \$total total results"
+echo "\$success of \$chkcnt tests PASSED"
+echo "\$pends of \$chkcnt tests PENDING"
+echo "\$misses of \$chkcnt tests MISSING data"
+echo "\$failures of \$chkcnt tests FAILED"
+#echo "  \$failbuild of \$failures FAILED build"
+#echo "  \$failrun of \$failures FAILED run"
+#echo "  \$failtest of \$failures FAILED test"
+#echo "  \$failcomp of \$failures FAILED compare"
+#echo "  \$failbfbc of \$failures FAILED bfbcomp"
+#echo "  \$failgen of \$failures FAILED generate"
 exit \$failures
 EOF
 

configuration/scripts/cice.test.setup.csh

@@ -40,6 +40,13 @@ if ( ! -f ${ICE_RUNDIR}/cice ) then
   exit 99
 endif
 
+# Initial test results and Reset test results for rerun
+mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev
+echo "#---" >! ${ICE_CASEDIR}/test_output
+cat ${ICE_CASEDIR}/test_output.prev | grep -i "${ICE_TESTNAME} build" >> ${ICE_CASEDIR}/test_output
+echo "PEND ${ICE_TESTNAME} run" >> ${ICE_CASEDIR}/test_output
+rm -f ${ICE_CASEDIR}/test_output.prev
+
 EOF2
 
 if ( -f ${ICE_SCRIPTS}/tests/test_${ICE_TEST}.script) then

configuration/scripts/tests/report_results.csh

@@ -1,5 +1,19 @@
 #!/bin/csh -f
 
+if ($#argv == 0) then
+  echo "${0}: Running results.csh"
+  ./results.csh >& /dev/null
+else if ($#argv == 1) then
+  if ("$argv[1]" =~ "-n") then
+    #continue
+  else
+    echo "$0 usage:"
+    echo "$0 [-n]"
+    echo "   -n : do NOT run results.csh (by default it does)"
+    exit -1
+  endif
+endif
+
 if (! -e results.log) then
   echo " "
   echo "${0}: ERROR results.log does not exist, try running results.csh"
@@ -79,12 +93,21 @@ unset noglob
 
 foreach compiler ( ${compilers} )
 
-  set ofile = "${shhash}.${mach}.${compiler}.${xcdat}.${xctim}"
-  set outfile = "${wikiname}/${tsubdir}/${ofile}.md"
+  set cnt = 0
+  set found = 1
+  while ($found == 1)
+    set ofile = "${shhash}.${mach}.${compiler}.${xcdat}.${xctim}.$cnt"
+    set outfile = "${wikiname}/${tsubdir}/${ofile}.md"
+    if (-e ${outfile}) then
+      @ cnt = $cnt + 1
+    else
+      set found = 0
+    endif
+  end
+
   mkdir -p ${wikiname}/${tsubdir}
   echo "${0}: writing to ${outfile}"
 
-  if (-e ${outfile}) rm -f ${outfile}
 
 cat >! ${outfile} << EOF
 
@@ -103,7 +126,7 @@ EOF
 foreach case ( ${cases} )
 if ( ${case} =~ *_${compiler}_* ) then
 
-# check thata case results are meaningful
+# check that case results are meaningful
   set fbuild = `grep " ${case} " results.log | grep " build"   | cut -c 1-4`
   set frun   = `grep " ${case} " results.log | grep " run"     | cut -c 1-4`
   set ftest  = `grep " ${case} " results.log | grep " test"    | cut -c 1-4`

configuration/scripts/tests/test_logbfb.script

@@ -4,11 +4,6 @@
 # This is identical to a smoke test, but triggers bfbcompare with log files instead of restarts
 # cice.run returns -1 if run did not complete successfully
 
-mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev
-cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} run" >! ${ICE_CASEDIR}/test_output
-rm -f ${ICE_CASEDIR}/test_output.prev
-echo "RUN  ${ICE_TESTNAME} run " >> ${ICE_CASEDIR}/test_output
-
 ./cice.run
 set res="$status"
 

configuration/scripts/tests/test_restart.script

@@ -7,14 +7,6 @@ cp ice_in ice_in.0
 ${ICE_CASEDIR}/casescripts/parse_namelist.sh ice_in ${ICE_CASEDIR}/casescripts/test_nml.restart1
 cp ice_in ice_in.1
 
-mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev
-cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} run" >! ${ICE_CASEDIR}/test_output
-mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev
-cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} test" >! ${ICE_CASEDIR}/test_output
-rm -f ${ICE_CASEDIR}/test_output.prev
-echo "RUN  ${ICE_TESTNAME} run " >> ${ICE_CASEDIR}/test_output
-echo "PEND ${ICE_TESTNAME} test " >> ${ICE_CASEDIR}/test_output
-
 ./cice.run
 set res="$status"
 
@@ -27,8 +19,6 @@ if ( $res != 0 ) then
   echo "FAIL ${ICE_TESTNAME} run" >> ${ICE_CASEDIR}/test_output
   echo "FAIL ${ICE_TESTNAME} test " >> ${ICE_CASEDIR}/test_output
   exit 99
-else
-  echo "PASS ${ICE_TESTNAME} initialrun" >> ${ICE_CASEDIR}/test_output
 endif
 
 # Prepend 'base_' to the final restart file to save for comparison

configuration/scripts/tests/test_smoke.script

@@ -3,11 +3,6 @@
 # Run the CICE model
 # cice.run returns -1 if run did not complete successfully
 
-mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev
-cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} run" >! ${ICE_CASEDIR}/test_output
-rm -f ${ICE_CASEDIR}/test_output.prev
-echo "RUN  ${ICE_TESTNAME} run " >> ${ICE_CASEDIR}/test_output
-
 ./cice.run
 set res="$status"
 

doc/source/user_guide/ug_testing.rst

@@ -645,7 +645,11 @@ To post results, once a test suite is complete, run ``results.csh`` and
   ./results.csh
   ./report_results.csh
 
-The reporting can also be automated by adding ``--report`` to ``cice.setup``
+``report_results.csh`` will run ``results.csh`` by default automatically, but
+we recommmend running it manually first to verify results before publishing
+them.  ``report_results.csh -n`` will turn off automatic running of ``results.csh``.
+
+The reporting can also be automated in a test suite by adding ``--report`` to ``cice.setup``
 ::
 
   ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a --report