Merge pull request #393 from elebeaup/fix/fix-false-negative

Fix false negative with `set -e`

Author: Chemaclass

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Fixed false negative with `set -e`
 - Fixed name rendered when having `test_test_*`
 - Fixed duplicate function detection
 - Fixed display test with multiple outputs in multiline

src/runner.sh

@@ -146,16 +146,19 @@ function runner::run_test() {
   exec 3>&1
 
   local test_execution_result=$(
+    trap '
+      state::set_test_exit_code $?
+      runner::run_tear_down
+      runner::clear_mocks
+      state::export_subshell_context
+    ' EXIT
     state::initialize_assertions_count
     runner::run_set_up
 
     # 2>&1: Redirects the std-error (FD 2) to the std-output (FD 1).
     # points to the original std-output.
     "$function_name" "$@" 2>&1
 
-    runner::run_tear_down
-    runner::clear_mocks
-    state::export_subshell_context
   )
 
   # Closes FD 3, which was used temporarily to hold the original stdout.
@@ -216,8 +219,9 @@ function runner::run_test() {
   runner::parse_result "$function_name" "$test_execution_result" "$@"
 
   local total_assertions="$(state::calculate_total_assertions "$test_execution_result")"
+  local test_exit_code="$(state::get_test_exit_code)"
 
-  if [[ -n $runtime_error ]]; then
+  if [[ -n $runtime_error || $test_exit_code -ne 0 ]]; then
     state::add_tests_failed
     console_results::print_error_test "$function_name" "$runtime_error"
     reports::add_test_failed "$test_file" "$function_name" "$duration" "$total_assertions"
@@ -360,13 +364,20 @@ function runner::parse_result_sync() {
     sed -E -e 's/.*##ASSERTIONS_SNAPSHOT=([0-9]*)##.*/\1/g'\
   )
 
+  local test_exit_code=$(\
+    echo "$execution_result" |\
+    tail -n 1 |\
+    sed -E -e 's/.*##TEST_EXIT_CODE=([0-9]*)##.*/\1/g'\
+  )
+
   log "debug" "[SYNC]" "function_name:$function_name" "execution_result:$execution_result"
 
   ((_ASSERTIONS_PASSED += assertions_passed)) || true
   ((_ASSERTIONS_FAILED += assertions_failed)) || true
   ((_ASSERTIONS_SKIPPED += assertions_skipped)) || true
   ((_ASSERTIONS_INCOMPLETE += assertions_incomplete)) || true
   ((_ASSERTIONS_SNAPSHOT += assertions_snapshot)) || true
+  ((_TEST_EXIT_CODE += test_exit_code)) || true
 }
 
 function runner::write_failure_result_output() {

src/state.sh

@@ -14,6 +14,7 @@ _DUPLICATED_FUNCTION_NAMES=""
 _FILE_WITH_DUPLICATED_FUNCTION_NAMES=""
 _DUPLICATED_TEST_FUNCTIONS_FOUND=false
 _TEST_OUTPUT=""
+_TEST_EXIT_CODE=0
 
 function state::get_tests_passed() {
   echo "$_TESTS_PASSED"
@@ -123,6 +124,14 @@ function state::add_test_output() {
   _TEST_OUTPUT+="$1"
 }
 
+function state::get_test_exit_code() {
+  echo "$_TEST_EXIT_CODE"
+}
+
+function state::set_test_exit_code() {
+  _TEST_EXIT_CODE="$1"
+}
+
 function state::set_duplicated_functions_merged() {
   state::set_duplicated_test_functions_found
   state::set_file_with_duplicated_function_names "$1"
@@ -155,6 +164,7 @@ function state::export_subshell_context() {
 ##ASSERTIONS_SKIPPED=$_ASSERTIONS_SKIPPED\
 ##ASSERTIONS_INCOMPLETE=$_ASSERTIONS_INCOMPLETE\
 ##ASSERTIONS_SNAPSHOT=$_ASSERTIONS_SNAPSHOT\
+##TEST_EXIT_CODE=$_TEST_EXIT_CODE\
 ##TEST_OUTPUT=$encoded_test_output\
 ##
 EOF

tests/acceptance/bashunit_fail_test.sh

@@ -45,6 +45,13 @@ function test_bashunit_with_multiple_failing_tests() {
   assert_general_error "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$test_file" --simple)"
 }
 
+function test_bashunit_with_a_test_fail_and_exit_immediately() {
+  local test_file=./tests/acceptance/fixtures/test_bashunit_when_exit_immediately_after_execution_error.sh
+
+  assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$test_file")"
+  assert_general_error "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" "$test_file" --simple)"
+}
+
 function test_different_simple_snapshots_matches() {
   todo "The different snapshots for these tests should also be identical to each other, option to choose snapshot name?"
 }

tests/acceptance/fixtures/test_bashunit_when_exit_immediately_after_execution_error.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+function test_error() {
+  set -e
+  invalid_function_name arg1 arg2 &>/dev/null
+}

tests/acceptance/snapshots/bashunit_fail_test_sh.test_bashunit_with_a_test_fail_and_exit_immediately.snapshot

@@ -0,0 +1,12 @@
+Running ./tests/acceptance/fixtures/test_bashunit_when_exit_immediately_after_execution_error.sh
+✗ Error: Error
+    
+
+There was 1 failure:
+
+|1) ./tests/acceptance/fixtures/test_bashunit_when_exit_immediately_after_execution_error.sh
+
+Tests:      1 failed, 1 total
+Assertions: 0 failed, 0 total
+
+ Some tests failed 

tests/unit/state_test.sh

@@ -254,6 +254,7 @@ function test_initialize_assertions_count() {
 ##ASSERTIONS_SKIPPED=0\
 ##ASSERTIONS_INCOMPLETE=0\
 ##ASSERTIONS_SNAPSHOT=0\
+##TEST_EXIT_CODE=0\
 ##TEST_OUTPUT=\
 ##"\
     "$export_assertions_count"
@@ -270,6 +271,7 @@ function test_export_assertions_count() {
     _ASSERTIONS_INCOMPLETE=12
     _ASSERTIONS_SNAPSHOT=33
     _ASSERTIONS_SNAPSHOT=33
+    _TEST_EXIT_CODE=1
     _TEST_OUTPUT="something"
 
     state::export_subshell_context
@@ -281,6 +283,7 @@ function test_export_assertions_count() {
 ##ASSERTIONS_SKIPPED=42\
 ##ASSERTIONS_INCOMPLETE=12\
 ##ASSERTIONS_SNAPSHOT=33\
+##TEST_EXIT_CODE=1\
 ##TEST_OUTPUT=$(echo -n "something" | base64)##"\
     "$export_assertions_count"
 }
@@ -291,6 +294,7 @@ function test_calculate_total_assertions() {
   ##ASSERTIONS_SKIPPED=3\
   ##ASSERTIONS_INCOMPLETE=4\
   ##ASSERTIONS_SNAPSHOT=5\
+  ##TEST_EXIT_CODE=0\
   ##TEST_OUTPUT=3zhbEncodedBase64##"
 
   assert_same 15 "$(state::calculate_total_assertions "$input")"