-
Notifications
You must be signed in to change notification settings - Fork 80
/
Copy pathtestframework.sh
2230 lines (2105 loc) · 67.6 KB
/
testframework.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
#
# Serval Project testing framework for Bash shell
# Copyright 2012 Serval Project, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
# This file is sourced by all testing scripts. A typical test script looks
# like this:
#
# #!/bin/bash
# source testframework.sh
# setup() {
# export BLAH_CONFIG=$TFWTMP/blah.conf
# echo "username=$LOGNAME" >$BLAH_CONFIG
# }
# teardown() {
# # $TFWTMP is always removed after every test, so no need to
# # remove blah.conf ourselves.
# }
# doc_feature1='Feature one works'
# test_feature1() {
# execute programUnderTest --feature1 arg1 arg2
# assertExitStatus '==' 0
# assertRealTime --message='ran in under half a second' '<=' 0.5
# assertStdoutIs ""
# assertStderrIs ""
# tfw_cat arg1
# }
# doc_feature2='Feature two fails with status 1'
# setup_feature2() {
# # Overrides setup(), so we have to call it ourselves explicitly
# # here if we still want it.
# setup
# echo "option=specialValue" >>$BLAH_CONFIG
# }
# test_feature2() {
# execute programUnderTest --feature2 arg1 arg2
# assertExitStatus '==' 1
# assertStdoutIs -e "Response:\tok\n"
# assertStderrGrep "^ERROR: missing arg3$"
# }
# runTests "$@"
AWK=awk
SED=sed
GREP=grep
TSFMT='+%Y-%m-%d %H:%M:%S'
SYSTYPE="$(uname -s)"
case "$SYSTYPE" in
SunOS | Darwin)
AWK=gawk
SED=gsed
GREP=ggrep
;;
Linux)
# Get nanosecond resolution
TSFMT='+%Y-%m-%d %H:%M:%S.%N'
;;
esac
usage() {
echo -n "\
Usage: $0 [options] [--]
Options:
-h --help Print this usage message
-t --trace Log shell "set -x" trace during tests
-v --verbose Send test log to output during execution
-j 0 --jobs Run all tests in parallel
-j N --jobs=N Run tests in parallel, at most N at a time (default
N=1)
-E --stop-on-error Do not execute any tests after an ERROR occurs
-F --stop-on-failure Do not execute any tests after a FAIL occurs
-t N --timeout=N Override default timeout, make it N seconds instead
of 60
-f PRE --filter=PRE Only execute tests whose names start with PRE
-f N --filter=N Only execute test number N
-f M-N --filter=M-N Only execute tests with numbers in range M-N inclusive
-f -N --filter=-N Only execute tests with numbers <= N
-f N- --filter=N- Only execute tests with numbers >= N
-f ... --filter=M,N,... Only execute tests with number M or N or ...
-c --coverage Collect test coverage data
-cg --geninfo Invoke geninfo(1) to produce one coverage.info file
per test case (requires at least one --gcno-dir)
-cd DIR --gcno-dir=DIR Use test coverage GCNO files under DIR (overrides
TFW_GCNO_PATH env var)
"
}
# Utility functions for setting shopt variables and restoring their original
# value:
# local oo
# tfw_shopt oo -s extglob -u extdebug
# ...
# tfw_shopt_restore oo
tfw_shopt() {
local _var="$1"
shift
local op=s
local restore=
while [ $# -ne 0 ]
do
case "$1" in
-s) op=s;;
-u) op=u;;
*)
local opt="$1"
restore="${restore:+$restore; }shopt -$(shopt -q $opt && echo s || echo u) $opt"
shopt -$op $opt
;;
esac
shift
done
eval $_var='"$restore"'
}
tfw_shopt_restore() {
local _var="$1"
[ -n "${!_var}" ] && eval "${!_var}"
}
declare -a _tfw_included_tests=()
declare -a _tfw_test_names=()
declare -a _tfw_test_sourcefiles=()
declare -a _tfw_job_pgids=()
declare -a _tfw_forked_pids=()
declare -a _tfw_forked_labels=()
# The rest of this file is parsed for extended glob patterns.
tfw_shopt _tfw_orig_shopt -s extglob
includeTests() {
local arg
for arg; do
local path
case "$arg" in
/*) path="$(abspath "$arg")";;
*) path="$(abspath "$_tfw_script_dir/arg")";;
esac
local n=$((${#BASH_LINENO[*]} - 1))
while [ $n -gt 0 -a ${BASH_LINENO[$n]} -eq 0 ]; do
let n=n-1
done
_tfw_included_tests+=("${BASH_LINENO[$n]} $arg")
done
}
runTests() {
[ -n "${_tfw_recursive_source}" ] && return 0
_tfw_stdout=1
_tfw_stderr=2
_tfw_log_fd=
_tfw_checkBashVersion
export PATH="$(_tfw_abspath "${BASH_SOURCE%/*}"):$PATH"
_tfw_checkTerminfo
_tfw_checkCommandInPATH tfw_createfile
_tfw_invoking_script=$(abspath "${BASH_SOURCE[1]}")
_tfw_script_name="${_tfw_invoking_script##*/}"
_tfw_script_dir="${_tfw_invoking_script%/*}"
_tfw_cwd=$(abspath "$PWD")
_tfw_tmpdir="$(_tfw_abspath -P "${TFW_TMPDIR:-${TMPDIR:-/tmp}}")"
_tfw_tmpmain="$_tfw_tmpdir/_tfw-$$"
_tfw_njobs=1
_tfw_log_noise=true
_tfw_assert_noise=true
_tfw_logdir="${TFW_LOGDIR:-$_tfw_cwd/testlog}"
_tfw_logdir_script="$_tfw_logdir/$_tfw_script_name"
_tfw_list=false
_tfw_trace=false
_tfw_verbose=false
_tfw_stop_on_error=false
_tfw_stop_on_failure=false
_tfw_default_execute_timeout=60
_tfw_default_wait_until_timeout=60
_tfw_timeout_override=
_tfw_coverage=false
_tfw_geninfo=false
_tfw_gcno_path=()
local allargs="$*"
local -a filters=()
local oo
tfw_shopt oo -s extglob
while [ $# -ne 0 ]; do
case "$1" in
-h|--help) usage; exit 0;;
-l|--list) _tfw_list=true;;
-t|--trace) _tfw_trace=true;;
-v|--verbose) _tfw_verbose=true;;
-E|--stop-on-error) _tfw_stop_on_error=true;;
-F|--stop-on-failure) _tfw_stop_on_failure=true;;
-f) [ -n "$2" ] || _tfw_fatal "missing argument after option: $1"
filters+=("$2")
shift
;;
-f*) filters+=("${1#-?}");;
--filter=*) filters+=("${1#*=}");;
-j) [ -n "$2" ] || _tfw_fatal "missing argument after option: $1"
_tfw_is_uint "${2?}" || _tfw_fatal "invalid option: $1 $2"
_tfw_njobs=${2?}
shift
;;
-j+([0-9])) _tfw_njobs="${1#-?}";;
-j*) _tfw_fatal "invalid option: $1";;
--jobs=+([0-9])) _tfw_njobs="${1#*=}";;
--jobs=*) _tfw_fatal "invalid option: $1";;
--jobs) _tfw_njobs=0;;
-t) [ -n "$2" ] || _tfw_fatal "missing argument after option: $1"
_tfw_is_float "${2?}" || _tfw_fatal "invalid option: $1 $2"
_tfw_timeout_override="${2?}"
shift
;;
-t*)
_tfw_is_float "${1#-?}" || _tfw_fatal "invalid option: $1"
_tfw_timeout_override="${1#-?}"
;;
--timeout=*)
_tfw_is_float "${1#*=}" || _tfw_fatal "invalid option: $1"
_tfw_timeout_override="${1#*=}"
;;
-c|--coverage) _tfw_coverage=true;;
-cg|--geninfo) _tfw_coverage=true; _tfw_geninfo=true;;
-cd) [ -n "$2" ] || _tfw_fatal "missing argument after option: $1"
_tfw_gcno_path+=("$2")
shift
;;
-cd*) _tfw_gcno_path+=("${1#-?}");;
--gcno-dir=*) _tfw_gcno_path+=("${1#*=}");;
--) shift; break;;
-*) _tfw_fatal "unsupported option: $1";;
*) _tfw_fatal "spurious argument: $1";;
esac
shift
done
tfw_shopt_restore oo
if $_tfw_verbose && [ $_tfw_njobs -ne 1 ]; then
_tfw_fatal "--verbose is incompatible with --jobs=$_tfw_njobs"
fi
# Handle --gcno-dir arguments, or if none given, $TFW_GCNO_PATH env var.
# Convert into a list of absolute directory paths.
if [ ${#_tfw_gcno_path[*]} -eq -0 ]; then
local oIFS="$IFS"
IFS=:
_tfw_gcno_path=($TFW_GCNO_PATH)
IFS="$oIFS"
else
local pathdir
for pathdir in "${_tfw_gcno_path[@]}"; do
[ -d "$pathdir" ] || _tfw_fatal "--gcno-dir: no such directory: '$pathdir'"
done
fi
_tfw_gcno_dirs=()
local pathdir
for pathdir in "${_tfw_gcno_path[@]}"; do
[ -d "$pathdir" ] && _tfw_gcno_dirs+=("$(abspath "$pathdir")")
done
# Handle --geninfo option.
if $_tfw_geninfo; then
if [ ${#_tfw_gcno_dirs[*]} -eq 0 ]; then
_tfw_fatal "--geninfo: requires at least one --gcno-dir=DIR or \$TFW_GCNO_PATH env var"
fi
_tfw_checkCommandInPATH geninfo
_tfw_checkCommandInPATH gcov _tfw_gcov_path
# Check that all source files are available.
_tfw_extract_source_files_from_gcno "${_tfw_gcno_dirs[@]}"
_tfw_coverage_source_basedir=.
if [ -n "$TFW_COVERAGE_SOURCE_BASE_DIR" ]; then
[ -d "$TFW_COVERAGE_SOURCE_BASE_DIR" ] || _tfw_fatal "--geninfo: no such directory '$TFW_COVERAGE_SOURCE_BASE_DIR' (\$TFW_COVERAGE_SOURCE_BASE_DIR)"
_tfw_coverage_source_basedir="$TFW_COVERAGE_SOURCE_BASE_DIR"
fi
local src
for src in "${_tfw_coverage_source_files[@]}"; do
local path="$_tfw_coverage_source_basedir/$src"
[ -r "$path" ] || _tfw_fatal "--geninfo: missing source file $path"
done
fi
# Enumerate all the test cases.
_tfw_list_tests
# If we are only asked to list them, then do so and finish.
if $_tfw_list; then
local testNumber
for ((testNumber = 1; testNumber <= ${#_tfw_test_names[*]}; ++testNumber)); do
local testSourceFile="${_tfw_test_sourcefiles[$(($testNumber - 1))]}"
local testName="${_tfw_test_names[$(($testNumber - 1))]}"
if _tfw_filter_predicate "$testNumber" "$testName" "${filters[@]}"; then
echo "$testNumber $testName ${testSourceFile#$_tfw_script_dir/}"
fi
done
return 0
fi
# Create base temporary working directory.
trap '_tfw_status=$?; _tfw_killtests 2>/dev/null; rm -rf "$_tfw_tmpmain"; trap - EXIT; exit $_tfw_status' EXIT SIGHUP SIGINT SIGTERM
rm -rf "$_tfw_tmpmain"
mkdir -p "$_tfw_tmpmain" || return $?
# Create an empty results directory.
_tfw_results_dir="$_tfw_tmpmain/results"
mkdir "$_tfw_results_dir" || return $?
# Create an empty log directory.
mkdir -p "$_tfw_logdir_script" || return $?
rm -r -f "$_tfw_logdir_script"/*
# Enable job control.
set -m
# Iterate through all test cases, starting a new test whenever the number of
# running tests is less than the job limit.
_tfw_testcount=0
_tfw_passcount=0
_tfw_failcount=0
_tfw_errorcount=0
_tfw_fatalcount=0
_tfw_job_pgids=()
_tfw_job_pgids[0]=
_tfw_test_number_watermark=0
local testNumber
local testPosition=0
for ((testNumber = 1; testNumber <= ${#_tfw_test_names[*]}; ++testNumber)); do
local testSourceFile="${_tfw_test_sourcefiles[$(($testNumber - 1))]}"
local testName="${_tfw_test_names[$(($testNumber - 1))]}"
local scriptName="${testSourceFile#$_tfw_script_dir/}"
_tfw_filter_predicate "$testNumber" "$testName" "${filters[@]}" || continue
let ++testPosition
let ++_tfw_testcount
# Wait for any existing child process to finish.
while [ $_tfw_njobs -ne 0 -a $(_tfw_count_running_jobs) -ge $_tfw_njobs ]; do
_tfw_wait_job_finish
_tfw_harvest_jobs
done
[ $_tfw_fatalcount -ne 0 ] && break
$_tfw_stop_on_error && [ $_tfw_errorcount -ne 0 ] && break
$_tfw_stop_on_failure && [ $_tfw_failcount -ne 0 ] && break
# Start the next test in a child process.
_tfw_echo_progress $testPosition $testNumber "$testSourceFile" $testName
if $_tfw_verbose || [ $_tfw_njobs -ne 1 ]; then
echo
fi
echo "$testPosition $testNumber $testName" NONE "$testSourceFile" >"$_tfw_results_dir/$testNumber"
( #)#<-- fixes Vim syntax highlighting
# The directory where this test's log.txt and other artifacts are
# deposited.
_tfw_logdir_test="$_tfw_logdir_script/$testNumber.$testName"
mkdir "$_tfw_logdir_test" || _tfw_fatalexit
# Pick a unique decimal number that must not coincide with other tests
# being run concurrently, _including tests being run in other test
# scripts by other users on the same host_. We cannot simply use
# $testNumber. The subshell process ID is ideal. We don't use
# $BASHPID because MacOS only has Bash-3.2, and $BASHPID was introduced
# in Bash-4.
_tfw_unique=$($BASH -c 'echo $PPID')
# All files created by this test belong inside a temporary directory.
# The path name must be kept short because it is used to construct
# named socket paths, which have a limited length.
_tfw_tmp=$_tfw_tmpdir/_tfw-$_tfw_unique
trap '_tfw_status=$?; rm -rf "$_tfw_tmp"; exit $_tfw_status' EXIT SIGHUP SIGINT SIGTERM
mkdir $_tfw_tmp || _tfw_fatalexit
# Set up test coverage data directory, which contains all the .gcno
# files of the executable(s) under test. If using geninfo(1) to
# generate coverage info files, then link to all the source files, to
# ensure that temporary .gcov files are created in this directory and
# not in the repository's base directory (which would cause race
# conditions).
if $_tfw_coverage; then
export GCOV_PREFIX="$_tfw_logdir_test/gcov"
export GCOV_PREFIX_STRIP=0
mkdir "$GCOV_PREFIX" || _tfw_fatalexit
# Link to GCNO files.
if [ ${#_tfw_gcno_dirs[*]} -ne 0 ]; then
find "${_tfw_gcno_dirs[@]}" -type f -name '*.gcno' -print0 | cpio -0pdl --quiet "$GCOV_PREFIX"
fi
# Link source files to where geninfo(1) will always find them before
# finding the original source files.
if $_tfw_geninfo; then
pushd "$_tfw_coverage_source_basedir" >/dev/null || _tfw_fatalexit
find "${_tfw_coverage_source_files[@]}" -maxdepth 0 -print0 | cpio -0pdl --quiet "$GCOV_PREFIX"
popd >/dev/null
fi
fi
## XXX _tfw_geninfo_initial "$scriptName/$testName" >$_tfw_tmp/log.geninfo 2>&1
local start_time=$(_tfw_timestamp)
local finish_time=unknown
( #)#<-- fixes Vim syntax highlighting
_tfw_result=FATAL
_tfw_stdout=5
_tfw_stderr=5
_tfw_log_fd=6
BASH_XTRACEFD=7
# Disable job control.
set +m
# If the test is from a different source script than the one
# invoking us, then source that file to pull in all the test
# definitions.
if [ "$testSourceFile" != "$_tfw_invoking_script" ]; then
_tfw_recursive_source=true
source "$testSourceFile"
unset _tfw_recursive_source
fi
declare -f test_$testName >/dev/null || _tfw_fatal "test_$testName not defined"
# Where per-forked-process files get stored -- see fork().
_tfw_process_tmp="$_tfw_tmp"
# Environment variables and temporary directories that test cases
# depend upon.
export COLUMNS=80 # for ls(1) multi-column output
export TFWSOURCE="$testSourceFile"
export TFWLOG="$_tfw_logdir_test"
export TFWUNIQUE=$_tfw_unique
export TFWVAR="$_tfw_tmp/var"
mkdir $TFWVAR || _tfw_fatalexit
export TFWTMP="$_tfw_tmp/tmp"
mkdir $TFWTMP || _tfw_fatalexit
cd $TFWTMP || _tfw_fatalexit
if $_tfw_verbose; then
# Find the PID of the current subshell process. Cannot use $BASHPID
# because MacOS only has Bash-3.2, and $BASHPID was introduced in Bash-4.
local mypid=$($BASH -c 'echo $PPID')
# These tail processes will die when the current subshell exits.
tail --pid=$mypid --follow $_tfw_tmp/log.stdout >&$_tfw_stdout 2>/dev/null &
tail --pid=$mypid --follow $_tfw_tmp/log.stderr >&$_tfw_stderr 2>/dev/null &
fi
# Execute the test case.
_tfw_phase=setup
_tfw_result=ERROR
_tfw_status=
trap "_tfw_status=\$?; _tfw_finalise $testName; _tfw_teardown $testName; _tfw_exit" EXIT SIGHUP SIGINT SIGTERM
_tfw_setup $testName
_tfw_result=FAIL
_tfw_phase=testcase
tfw_log "# CALL test_$testName()"
$_tfw_trace && set -x
test_$testName
set +x
case $_tfw_phase in
testcase-setup) _tfw_error "test terminated within fixture (missing a end_fixture call?)";;
testcase);;
*) _tfw_fatal "internal error: _tfw_phase=$_tfw_phase";;
esac
_tfw_result=PASS
exit 0
) <&- 5>&1 5>&2 6>"$_tfw_tmp/log.stdout" 1>&6 2>"$_tfw_tmp/log.stderr" 7>"$_tfw_tmp/log.xtrace"
local stat=$?
finish_time=$(_tfw_timestamp)
local result=FATAL
case $stat in
254) result=ERROR;;
1) result=FAIL;;
0) result=PASS;;
esac
echo "$testPosition $testNumber $testName $result $testSourceFile" >"$_tfw_results_dir/$testNumber"
{
echo "Name: $testName ($scriptName)"
echo "Result: $result"
echo "Started: $start_time"
echo "Finished: $finish_time"
echo '++++++++++ log.stdout ++++++++++'
cat $_tfw_tmp/log.stdout
echo '++++++++++'
echo '++++++++++ log.stderr ++++++++++'
cat $_tfw_tmp/log.stderr
echo '++++++++++'
if $_tfw_trace; then
echo '++++++++++ log.xtrace ++++++++++'
cat $_tfw_tmp/log.xtrace
echo '++++++++++'
fi
} >"$_tfw_logdir_test/log.txt"
mv "$_tfw_logdir_test" "$_tfw_logdir_test.$result"
_tfw_logdir_test="$_tfw_logdir_test.$result"
if $_tfw_geninfo; then
local testname=$(_tfw_string_to_identifier "$scriptName/$testName")
local result="$_tfw_tmp/result.info"
local coverage="$_tfw_logdir_test/coverage.info"
{
echo '++++++++++ log.geninfo ++++++++++'
_tfw_run_geninfo "$coverage" --test-name "$testname" 2>&1
echo '++++++++++'
} >>"$_tfw_logdir_test/log.txt"
fi
exit 0
) </dev/null &
local job=$(jobs %% 2>/dev/null | $SED -n -e '1s/^\[\([0-9]\{1,\}\)\].*/\1/p')
if [ -n "${_tfw_job_pgids[$job]}" ]; then
_tfw_harvest_job $job
fi
_tfw_job_pgids[$job]=$(jobs -p %$job 2>/dev/null)
ln -f -s "$_tfw_results_dir/$testNumber" "$_tfw_results_dir/job-$job"
done
# Wait for all child processes to finish.
while _tfw_any_running_jobs; do
_tfw_wait_job_finish
_tfw_harvest_jobs
done
# Clean up working directory.
rm -rf "$_tfw_tmpmain"
trap - EXIT SIGHUP SIGINT SIGTERM
# Echo result summary and exit with success if no failures or errors.
s=$([ $_tfw_testcount -eq 1 ] || echo s)
echo "$_tfw_testcount test$s, $_tfw_passcount pass, $_tfw_failcount fail, $_tfw_errorcount error"
[ $_tfw_fatalcount -eq 0 -a $_tfw_failcount -eq 0 -a $_tfw_errorcount -eq 0 ]
}
_tfw_killtests() {
if [ $_tfw_njobs -eq 1 ]; then
echo " killing..."
else
echo -n -e "\r\rKilling tests..."
fi
trap '' SIGHUP SIGINT SIGTERM
local pgid
for pgid in $(jobs -p); do
kill -TERM -$pgid
done
wait
}
_tfw_count_running_jobs() {
local count=0
local job
for ((job = 1; job < ${#_tfw_job_pgids[*]}; ++job)); do
[ -n "${_tfw_job_pgids[$job]}" ] && let count=count+1
done
echo $count
}
_tfw_any_running_jobs() {
local job
for ((job = 1; job < ${#_tfw_job_pgids[*]}; ++job)); do
[ -n "${_tfw_job_pgids[$job]}" ] && return 0
done
return 1
}
_tfw_wait_job_finish() {
if [ $_tfw_njobs -eq 1 ]; then
wait >/dev/null 2>/dev/null
else
# This is the only way known to get the effect of a 'wait' builtin that
# will return when _any_ child dies (or after a one-second timeout).
set -m
sleep 1 &
local spid=$!
trap "kill -TERM $spid 2>/dev/null" SIGCHLD
wait $spid >/dev/null 2>/dev/null
trap - SIGCHLD
fi
}
_tfw_harvest_jobs() {
local job
for ((job = 1; job < ${#_tfw_job_pgids[*]}; ++job)); do
if [ -n "${_tfw_job_pgids[$job]}" ]; then
jobs %$job >/dev/null 2>/dev/null || _tfw_harvest_job $job
fi
done
}
_tfw_harvest_job() {
local job="$1"
# Kill any residual processes from the test case.
local pgid=${_tfw_job_pgids[$job]}
[ -n "$pgid" ] && kill -TERM -$pgid 2>/dev/null
_tfw_job_pgids[$job]=
# Report the test script outcome.
if [ -s "$_tfw_results_dir/job-$job" ]; then
local testPosition
local testNumber
local testName
local result
local testSourceFile
_tfw_unpack_words "$(<"$_tfw_results_dir/job-$job")" testPosition testNumber testName result testSourceFile
case "$result" in
ERROR)
let _tfw_errorcount=_tfw_errorcount+1
;;
PASS)
let _tfw_passcount=_tfw_passcount+1
;;
FAIL)
let _tfw_failcount=_tfw_failcount+1
;;
*)
result=FATAL
let _tfw_fatalcount=_tfw_fatalcount+1
;;
esac
local lines
if ! $_tfw_verbose && [ $_tfw_njobs -eq 1 ]; then
_tfw_echo_progress $testPosition $testNumber "$testSourceFile" $testName $result
echo
elif ! $_tfw_verbose && lines=$($_tfw_tput lines); then
local travel=$(($_tfw_test_number_watermark - $testPosition + 1))
if [ $travel -gt 0 -a $travel -lt $lines ] && $_tfw_tput cuu $travel ; then
_tfw_echo_progress $testPosition $testNumber "$testSourceFile" $testName $result
echo
travel=$(($_tfw_test_number_watermark - $testPosition))
[ $travel -gt 0 ] && $_tfw_tput cud $travel
fi
else
_tfw_echo_progress $testPosition $testNumber "$testSourceFile" $testName $result
echo
fi
else
_tfw_echoerr "${BASH_SOURCE[1]}: job %$job terminated without result"
fi
rm -f "$_tfw_results_dir/job-$job"
}
_tfw_echo_progress() {
(
local script=
if [ "$testSourceFile" != "$_tfw_invoking_script" ]; then
_tfw_recursive_source=true
source "$testSourceFile"
unset _tfw_recursive_source
script=" (${3#$_tfw_script_dir/})"
fi
local docvar="doc_$4"
echo -n -e '\r'
echo -n "$2 ["
_tfw_echo_result "$5"
echo -n "]$script ${!docvar:-$4}"
)
[ $1 -gt $_tfw_test_number_watermark ] && _tfw_test_number_watermark=$1
}
_tfw_echo_result() {
local result="$1"
case "$result" in
ERROR | FATAL)
$_tfw_tput setaf 1
$_tfw_tput rev
echo -n "$result"
$_tfw_tput sgr0
$_tfw_tput op
;;
PASS)
$_tfw_tput setaf 2
echo -n "$result"
$_tfw_tput op
echo -n "."
;;
FAIL)
$_tfw_tput setaf 1
echo -n "$result"
$_tfw_tput op
echo -n "."
;;
*)
result="$result....."
echo -n "${result:0:5}"
;;
esac
}
_tfw_extract_source_files_from_gcno() {
# This should possibly be done by creating a binary utility that knows how to
# disassemble GCNO files. In the meantime, this approach seems to work:
# simply extract all strings from all GCNO files that match *.c or *.h.
local IFS='
'
_tfw_coverage_source_files=($(find "$@" -type f -name '*.gcno' -print0 | xargs -0 strings | grep '\.[ch]$' | sort -u))
}
_tfw_run_geninfo() {
local infofile="$1"
shift
geninfo \
--rc lcov_tmp_dir="$_tfw_tmp" \
--gcov-tool "$_tfw_gcov_path" \
--output-file "$infofile" \
--no-external \
"$@" \
"$_tfw_logdir_test/gcov"
# Cook the absolute source file paths in the info file to refer to the
# original source files, not the links we placed into the gcov subdirectory
# in order to avoid race conditions.
local basedir="$(abspath "$_tfw_coverage_source_basedir")"
$SED -i -e "/^SF:/s:$_tfw_logdir_test/gcov:$basedir:" "$infofile"
}
_tfw_string_to_identifier() {
echo "$1" | $SED -e 's/\//__/g' -e 's/[^0-9a-zA-Z_]/_/g'
}
# Internal (private) functions that are not to be invoked directly from test
# scripts.
# Add shell quotation to the given arguments, so that when expanded using
# 'eval', the exact same argument results. This makes argument handling fully
# immune to spaces and shell metacharacters.
_tfw_shellarg() {
local arg
_tfw_args=()
for arg; do
case "$arg" in
'' | *[^A-Za-z_0-9.,:=+\/-]* ) _tfw_args+=("'${arg//'/'\\''}'");;
*) _tfw_args+=("$arg");;
esac
done
}
# Echo the absolute path of the given path, using only Bash builtins.
_tfw_abspath() {
cdopt=-L
if [ $# -gt 1 -a "${1:0:1}" = - ]; then
cdopt="$1"
shift
fi
case "$1" in
*/)
builtin echo $(_tfw_abspath $cdopt "${1%/}")/
;;
/*/*)
if [ -d "$1" ]; then
(CDPATH= builtin cd $cdopt "$1" && builtin echo "$PWD")
else
builtin echo $(_tfw_abspath $cdopt "${1%/*}")/"${1##*/}"
fi
;;
/*)
echo "$1"
;;
*/*)
if [ -d "$1" ]; then
(CDPATH= builtin cd $cdopt "$1" && builtin echo "$PWD")
else
builtin echo $(_tfw_abspath $cdopt "${1%/*}")/"${1##*/}"
fi
;;
. | ..)
(CDPATH= builtin cd $cdopt "$1" && builtin echo "$PWD")
;;
*)
(CDPATH= builtin cd $cdopt . && builtin echo "$PWD/$1")
;;
esac
}
_tfw_timestamp() {
local ts=$(date "$TSFMT")
echo "${ts%[0-9][0-9][0-9][0-9][0-9][0-9]}"
}
_tfw_setup() {
local testName="$1"
tfw_log '# SETUP'
case `type -t setup_$testName` in
function)
tfw_log "# call setup_$testName()"
$_tfw_trace && set -x
setup_$testName $testName
set +x
;;
*)
tfw_log "# call setup($testName)"
$_tfw_trace && set -x
setup $testName
set +x
;;
esac
tfw_log '# END SETUP'
}
_tfw_finalise() {
local testName="$1"
_tfw_phase=finalise
tfw_log '# FINALISE'
case `type -t finally_$testName` in
function)
tfw_log "# CALL finally_$testName()"
$_tfw_trace && set -x
finally_$testName
set +x
;;
*)
tfw_log "# CALL finally($testName)"
$_tfw_trace && set -x
finally $testName
set +x
;;
esac
fork_terminate_all
fork_wait_all
tfw_log '# END FINALLY'
}
_tfw_teardown() {
local testName="$1"
_tfw_phase=teardown
tfw_log '# TEARDOWN'
case `type -t teardown_$testName` in
function)
tfw_log "# CALL teardown_$testName()"
$_tfw_trace && set -x
teardown_$testName
set +x
;;
*)
tfw_log "# CALL teardown($testName)"
$_tfw_trace && set -x
teardown $testName
set +x
;;
esac
tfw_log '# END TEARDOWN'
}
_tfw_exit() {
case $_tfw_status:$_tfw_result in
254:* | *:ERROR ) exit 254;;
1:* | *:FAIL ) exit 1;;
0:PASS ) exit 0;;
esac
_tfw_fatal "_tfw_status='$_tfw_status' _tfw_result='$_tfw_result'"
}
# Executes $_tfw_executable with the given arguments.
_tfw_execute() {
executed=$(shellarg "${_tfw_executable##*/}" "$@")
if $_tfw_opt_core_backtrace; then
ulimit -S -c unlimited
rm -f core
fi
export TFWSTDOUT="${_tfw_stdout_file:-$_tfw_process_tmp/stdout}"
export TFWSTDERR="${_tfw_stderr_file:-$_tfw_process_tmp/stderr}"
echo "$executed" >"$_tfw_process_tmp/executing"
{
time -p "$_tfw_executable" "$@" >"$TFWSTDOUT" 2>"$TFWSTDERR"
} 2>"$_tfw_process_tmp/times" &
local subshell_pid=$!
local timer_pid=
local timeout=${_tfw_timeout_override:-${_tfw_opt_timeout:-${TFW_EXECUTE_TIMEOUT:-$_tfw_default_execute_timeout}}}
if [ -n "$timeout" ]; then
_tfw_is_float "$timeout" || error "invalid timeout '$timeout'"
fi
if [ -n "$timeout" ]; then
if type pgrep >/dev/null 2>/dev/null; then
( #)#( <<- fixes Vim syntax colouring
# For some reason, set -e does not work here. So all the following
# commands are postfixed with || exit $?
local executable_pid=$(pgrep -P $subshell_pid) || exit $?
[ -n "$executable_pid" ] || exit $?
if [ -n "$timeout" ]; then
sleep $timeout || exit $?
fi
kill -0 $executable_pid || exit $?
tfw_log "# timeout after $timeout seconds, sending SIGABRT to pid $executable_pid ($executed)" || exit $?
kill -ABRT $executable_pid || exit $?
sleep 2 || exit $?
kill -0 $executable_pid || exit $?
tfw_log "# sending second SIGABRT to pid $executable_pid ($executed)" || exit $?
kill -ABRT $executable_pid || exit $?
sleep 2 || exit $?
kill -0 $executable_pid || exit $?
tfw_log "# sending SIGKILL to pid $executable_pid ($executed)" || exit $?
kill -KILL $executable_pid || exit $?
exit 0
) 2>/dev/null &
timer_pid=$!
else
tfw_log "# execution timeout ($timeout seconds) not supported because pgrep(1) not available"
fi
fi
wait $subshell_pid
_tfw_exitStatus=$?
if [ -n "$timer_pid" ]; then
while kill -0 $timer_pid 2>/dev/null; do
pkill -P $timer_pid 2>/dev/null
done
wait $timer_pid
fi
rm -f "$_tfw_process_tmp/executing"
# Deal with core dump.
if $_tfw_opt_core_backtrace && [ -s core ]; then
tfw_core_backtrace "$_tfw_executable" core
fi
# Deal with exit status.
if [ -n "$_tfw_opt_exit_status" ]; then
_tfw_message="exit status ($_tfw_exitStatus) of ($executed) is $_tfw_opt_exit_status"
_tfw_assert [ "$_tfw_exitStatus" -eq "$_tfw_opt_exit_status" ] || _tfw_failexit || return $?
$_tfw_assert_noise && tfw_log "# assert $_tfw_message"
else
$_tfw_assert_noise && tfw_log "# exit status of ($executed) = $_tfw_exitStatus"
fi
# Parse execution time report.
if true || [ -s "$_tfw_process_tmp/times" ]; then
if ! _tfw_parse_times_to_milliseconds real realtime_ms ||
! _tfw_parse_times_to_milliseconds user usertime_ms ||
! _tfw_parse_times_to_milliseconds sys systime_ms
then
tfw_log '# malformed output from time:'
tfw_cat -v "$_tfw_process_tmp/times"
fi
else
realtime_ms=
usertime_ms=
systime_ms=
fi
return 0
}
_tfw_parse_times_to_milliseconds() {
local label="$1"
local var="$2"
local milliseconds=$($AWK '$1 == "'"$label"'" {
value = $2
minutes = 0
if (match(value, "[0-9]+m")) {
minutes = substr(value, RSTART, RLENGTH - 1)
value = substr(value, 1, RSTART - 1) substr(value, RSTART + RLENGTH)
}
if (substr(value, length(value)) == "s") {
value = substr(value, 1, length(value) - 1)
}
if (match(value, "^[0-9]+(\\.[0-9]+)?$")) {
seconds = value + 0
print (minutes * 60 + seconds) * 1000
}
}' $_tfw_process_tmp/times)
[ -z "$milliseconds" ] && return 1
[ -n "$var" ] && eval $var=$milliseconds
return 0
}
_tfw_assert() {
if ! tfw_run "$@"; then
_tfw_failmsg "assertion failed: ${_tfw_message:-$*}"
_tfw_backtrace
return 1
fi
return 0
}
declare -a _tfw_opt_dump_on_fail
_tfw_dump_on_fail() {
local arg
for arg; do
local _found=false
local _f
for _f in "${_tfw_opt_dump_on_fail[@]}"; do
if [ "$_f" = "$arg" ]; then
_found=true
break
fi
done
$_found || _tfw_opt_dump_on_fail+=("$arg")
done
}
_tfw_getopts() {
local context="$1"
shift
_tfw_executable=
_tfw_stdout_file=
_tfw_stderr_file=
_tfw_opt_core_backtrace=false
_tfw_message=
_tfw_opt_dump_on_fail=()
_tfw_opt_error_on_fail=false
_tfw_opt_exit_status=
_tfw_opt_timeout=
_tfw_opt_sleep=
_tfw_opt_matches=
_tfw_opt_line=
_tfw_opt_line_sed=
_tfw_opt_line_msg=
_tfw_opt_grepopts=()
_tfw_getopts_shift=0
local oo
tfw_shopt oo -s extglob
while [ $# -ne 0 ]; do
case "$context:$1" in
*:--stdout) _tfw_dump_on_fail --stdout;;
*:--stderr) _tfw_dump_on_fail --stderr;;
assert*:--dump-on-fail=*) _tfw_dump_on_fail "${1#*=}";;
@(assert*|execute*|fork_wait*):--error-on-fail) _tfw_opt_error_on_fail=true;;
assert*:--message=*) _tfw_message="${1#*=}";;
execute:--exit-status=+([0-9])) _tfw_opt_exit_status="${1#*=}";;
execute:--exit-status=*) _tfw_error "invalid value: $1";;
execute*:--executable=) _tfw_error "missing value: $1";;
execute*:--executable=*) _tfw_executable="${1#*=}";;
execute*:--stdout-file=) _tfw_error "missing value: $1";;
execute*:--stdout-file=*) _tfw_stdout_file="${1#*=}";;
execute*:--stderr-file=) _tfw_error "missing value: $1";;
execute*:--stderr-file=*) _tfw_stderr_file="${1#*=}";;
execute*:--core-backtrace) _tfw_opt_core_backtrace=true;;
@(execute*|wait_until):--timeout=@(+([0-9])?(.+([0-9]))|*([0-9]).+([0-9]))) _tfw_opt_timeout="${1#*=}";;
@(execute*|wait_until):--timeout=*) _tfw_error "invalid value: $1";;
wait_until:--sleep=*) _tfw_is_float "${1#*=}" || _tfw_error "invalid value: $1"; _tfw_opt_sleep="${1#*=}";;
*grep:--fixed-strings) _tfw_opt_grepopts+=(-F);;
assertcontentgrep:--matches=+([0-9])) _tfw_opt_matches="${1#*=}";;
assertcontentgrep:--matches=*) _tfw_error "invalid value: $1";;