Skip to content

Commit c0b9927

Browse files
egavrinegavrin
committed
Add parallel test262 runner.
JerryScript-DCO-1.0-Signed-off-by: Evgeny Gavrin e.gavrin@samsung.com
1 parent 71bac03 commit c0b9927

File tree

1 file changed

+89
-92
lines changed

1 file changed

+89
-92
lines changed

‎tools/runners/run-test-suite-test262.sh

Lines changed: 89 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22

3-
# Copyright 2014-2015 Samsung Electronics Co., Ltd.
3+
# Copyright 2015 Samsung Electronics Co., Ltd.
44
#
55
# Licensed under the Apache License, Version 2.0 (the "License");
66
# you may not use this file except in compliance with the License.
@@ -14,108 +14,105 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616

17-
TARGET=$1
18-
shift
19-
20-
PARSE_ONLY=""
21-
VERBOSE_REPORT=0
22-
TIMEOUT=5
23-
24-
function usage() {
25-
echo "Usage: $0 {path to engine} [-p] [-v] [-t {timeout in seconds}]" >&2
26-
echo " -p - parser only mode" >&2
27-
echo " -v - verbose report mode" >&2
28-
echo " -t - timeout for each test" >&2
17+
ENGINE="$1"
18+
PATH_TO_TEST262="$2"
19+
REPORT_PREFIX="report"
20+
RUN_PIDS=""
21+
RESULT_OK=1
22+
TIMEOUT="5s"
2923

24+
if [ $# -lt 2 ]
25+
then
26+
echo "This script performs parallel test262 compliance testing of the specified engine."
27+
echo ""
28+
echo "Usage:"
29+
echo " 1st parameter: JavaScript engine to be tested."
30+
echo " 2nd parameter: path to the directory with official test262 testsuite."
31+
echo " 3rd parameter: (optional) call this script with the '--sub-chapters' flag to print the detailed report."
32+
echo ""
33+
echo "Example:"
34+
echo " ./run-test-suite-test262.sh <engine> <test262_dir> --sub-chapters"
3035
exit 1
31-
}
36+
fi
3237

33-
[ -x "$TARGET" ] || usage
34-
35-
while getopts pvt: OPTION; do
36-
case "$OPTION" in
37-
p)
38-
PARSE_ONLY="--parse-only" ;;
39-
v)
40-
VERBOSE_REPORT=1 ;;
41-
t)
42-
TIMEOUT="$OPTARG"; [ "$TIMEOUT" -eq "$TIMEOUT" ] || usage;;
43-
[?])
44-
usage ;;
45-
esac
46-
done
4738

48-
TMP_DIR=`mktemp -d --tmpdir=./`
39+
rm "${REPORT_PREFIX}".* &> /dev/null
40+
41+
declare -a CHAPTER07_TO_TEST=("7.1" "7.2" "7.3" "7.4" "7.5" "7.6" "7.6.1" "7.7" \
42+
"7.8" "7.8.1" "7.8.2" "7.8.3" "7.8.4" "7.8.5" "7.9" "7.9.2")
43+
declare -a CHAPTER08_TO_TEST=("8.1" "8.2" "8.3" "8.4" "8.5" "8.6" "8.6.1" "8.6.2" "8.7" "8.7.1" "8.7.2" "8.8" "8.12" \
44+
"8.12.1" "8.12.3" "8.12.4" "8.12.5" "8.12.6" "8.12.7" "8.12.8" "8.12.9")
45+
declare -a CHAPTER09_TO_TEST=("9.1" "9.2" "9.3" "9.3.1" "9.4" "9.5" "9.6" "9.7" "9.8" "9.8.1" "9.9")
46+
declare -a CHAPTER10_TO_TEST=("10.1" "10.1.1" "10.2" "10.2.1" "10.2.2" "10.2.3" "10.3" "10.3.1" "10.4" "10.4.1" \
47+
"10.4.2" "10.4.3" "10.5" "10.6")
48+
declare -a CHAPTER11_TO_TEST=("11.1" "11.1.1" "11.1.2" "11.1.3" "11.1.4" "11.1.5" "11.1.6" "11.2" "11.2.1" "11.2.2" \
49+
"11.2.3" "11.2.4" "11.3" "11.3.1" "11.3.2" "11.4" "11.4.1" "11.4.2" "11.4.3" \
50+
"11.4.4" "11.4.5" "11.4.6" "11.4.7" "11.4.8" "11.4.9" "11.5" "11.5.1" "11.5.2" "11.5.3" \
51+
"11.6" "11.6.1" "11.6.2" "11.7" "11.7.1" "11.7.2" "11.7.3" "11.8" "11.8.1" "11.8.2" \
52+
"11.8.3" "11.8.4" "11.8.6" "11.8.7" "11.9" "11.9.1" "11.9.2" "11.9.4" "11.9.5" "11.10" \
53+
"11.11" "11.12" "11.13" "11.13.1" "11.13.2" "11.14")
54+
declare -a CHAPTER12_TO_TEST=("12.1" "12.2" "12.2.1" "12.3" "12.4" "12.5" "12.6" "12.6.1" "12.6.2" "12.6.3" "12.6.4" \
55+
"12.7" "12.8" "12.9" "12.10" "12.10.1" "12.11" "12.12" "12.13" "12.14" "12.14.1")
56+
declare -a CHAPTER13_TO_TEST=("13.1" "13.2" "13.2.1" "13.2.2" "13.2.3")
57+
declare -a CHAPTER14_TO_TEST=("14.1")
58+
declare -a CHAPTER14_TO_TEST=("12.6.4")
59+
declare -a CHAPTER15_TO_TEST=("15.1" "15.1.1" "15.1.2" "15.1.3" "15.1.4" "15.1.5" "15.2" "15.2.1" "15.2.2" "15.2.3" \
60+
"15.2.4" "15.2.5" "15.3" "15.3.1" "15.3.2" "15.3.3" "15.3.4" "15.3.5" "15.4" "15.4.1" \
61+
"15.4.2" "15.4.3" "15.4.4" "15.4.5" "15.5" "15.5.1" "15.5.2" "15.5.3" "15.5.4" "15.5.5" \
62+
"15.6" "15.6.1" "15.6.2" "15.6.3" "15.6.4" "15.6.5" "15.7" "15.7.1" "15.7.2" "15.7.3" \
63+
"15.7.4" "15.7.5" "15.8" "15.8.1" "15.8.2" "15.9" "15.9.1" "15.9.2" "15.9.3" "15.9.4" \
64+
"15.9.5" "15.9.6" "15.10" "15.10.1" "15.10.2" "15.10.3" "15.10.4" "15.10.5" "15.10.6" \
65+
"15.10.7" "15.11" "15.11.1" "15.11.2" "15.11.3" "15.11.4" "15.11.5" "15.11.6" "15.11.7" \
66+
"15.12" "15.12.1" "15.12.2" "15.12.3")
67+
declare -a FULL_CHAPTERS_TO_TEST=("ch06" "ch07" "ch08" "ch09" "ch10" "ch11" "ch12" "ch13" "ch14" "ch15")
68+
declare -a SUB_CHAPTERS_TO_TEST=("${CHAPTER07_TO_TEST[@]}" "${CHAPTER08_TO_TEST[@]}" "${CHAPTER09_TO_TEST[@]}" \
69+
"${CHAPTER10_TO_TEST[@]}" "${CHAPTER11_TO_TEST[@]}" "${CHAPTER12_TO_TEST[@]}" \
70+
"${CHAPTER13_TO_TEST[@]}" "${CHAPTER14_TO_TEST[@]}" "${CHAPTER15_TO_TEST[@]}")
71+
72+
declare -a CHAPTERS_TO_TEST=("${FULL_CHAPTERS_TO_TEST[@]}")
73+
74+
if [[ $* == *--sub-chapters* ]]
75+
then
76+
declare -a CHAPTERS_TO_TEST=("${SUB_CHAPTERS_TO_TEST[@]}")
77+
fi
4978

50-
echo $TMP_DIR | grep -q tmp || exit 1
79+
function run_test262 () {
80+
ARG_ENGINE="$1"
81+
ARG_TEST262_PATH="$2"
82+
ARG_CHAPTER="$3"
5183

52-
trap clean_on_exit INT
84+
"${ARG_TEST262_PATH}"/tools/packaging/test262.py --command "timeout ${TIMEOUT} ${ARG_ENGINE}" \
85+
--tests="${ARG_TEST262_PATH}" --full-summary "${ARG_CHAPTER}" \
86+
> "${REPORT_PREFIX}"."${ARG_CHAPTER}"
87+
}
5388

54-
function clean_on_exit() {
55-
rm -rf $TMP_DIR
89+
function show_report_results () {
90+
ARG_CHAPTER="$1"
5691

57-
[[ $1 == "OK" ]] || exit 1
92+
echo ""
93+
echo "Chapter ${ARG_CHAPTER}:"
94+
grep -A3 "=== Summary ===" "${REPORT_PREFIX}"."${ARG_CHAPTER}"
95+
echo "==============="
5896
}
5997

60-
TEST_SUITE_PATH="./tests/test262/test"
61-
STA_JS="$TEST_SUITE_PATH/harness/sta-jerry.js"
62-
MAX_THREADS=${MAX_CPUS:-32}
63-
64-
[ $VERBOSE_REPORT -eq 0 ] || rm -f jerry.error.*
65-
rm -f test262.report
66-
67-
find $TEST_SUITE_PATH/suite -name *.js -printf "%p %D%i\0" | xargs -0 -n 1 -P $MAX_THREADS bash -c '
68-
target=$0
69-
sta_js_path=$1;
70-
timeout=$2;
71-
tmp_dir=$3;
72-
test=`echo $4 | cut -d " " -f 1`;
73-
test_id=`echo $4 | cut -d " " -f 2`;
74-
chapter=`echo $test | cut -d "/" -f 6`;
75-
76-
grep "\* @negative" $test 2>&1 >/dev/null;
77-
negative=$?;
78-
79-
cmd="$target $test $sta_js_path";
80-
output=`ulimit -t $timeout; $cmd 2>&1`;
81-
status=$?;
82-
output=`echo "$output" | sed -e "s/^/ /"`
83-
84-
if [[ $status -eq 0 && $negative -eq 0 || $status -ne 0 && $negative -ne 0 ]];
85-
then
86-
(echo "====================";
87-
echo "Run \"$cmd\" failed: $status";
88-
echo "---------------------";
89-
echo "$output";
90-
echo;) >> $tmp_dir/jerry.error."$chapter"."$test_id";
91-
else
92-
(echo "====================";
93-
echo "Run \"$cmd\" successful";
94-
echo "---------------------";
95-
echo "$output";
96-
echo;) >> $tmp_dir/jerry.ok."$chapter"."$test_id";
97-
fi;
98-
' "$TARGET $PARSE" $STA_JS $TIMEOUT $TMP_DIR 2>/dev/null;
99-
100-
if [ $VERBOSE_REPORT -eq 1 ]
101-
then
102-
for CHAPTER in `ls $TEST_SUITE_PATH/suite`;
103-
do
104-
cat $TMP_DIR/jerry.error."$CHAPTER".* > jerry.error."$CHAPTER"
105-
done
106-
fi
98+
echo "Starting test262 testing for ${ENGINE}."
10799

108-
cat $TMP_DIR/jerry.ok.* $TMP_DIR/jerry.error.* | grep "^Run" > test262.report
109-
110-
ok_num=`grep "successful$" test262.report | wc -l`
111-
error_num=`grep "Run .* failed: [0-9]*$" test262.report | wc -l`
100+
for TEST_NAME in "${CHAPTERS_TO_TEST[@]}"
101+
do
102+
run_test262 "${ENGINE}" "${PATH_TO_TEST262}" "$TEST_NAME" &
103+
RUN_PIDS="$RUN_PIDS $!";
104+
done
112105

113-
ok_percent=`echo $ok_num $error_num | awk '{print 100.0 * $1 / ($1 + $2);}'`
114-
error_percent=`echo $ok_num $error_num | awk '{print 100.0 * $2 / ($1 + $2);}'`
106+
for RUN_PIDS in $RUN_PIDS
107+
do
108+
wait "$RUN_PIDS" || RESULT_OK=0
109+
done;
110+
#[ $RESULT_OK -eq 1 ] || exit 1
115111

116-
echo -e "OK:\t$ok_num ($ok_percent %)"
117-
echo -e "FAIL:\t$error_num ($error_percent %)"
118-
echo
119-
echo -e "Report:\t$PWD/test262.report"
112+
echo "Testing is completed."
120113

121-
clean_on_exit OK
114+
for TEST_NAME in "${CHAPTERS_TO_TEST[@]}"
115+
do
116+
echo "$TEST_NAME"
117+
show_report_results "$TEST_NAME"
118+
done

0 commit comments

Comments
 (0)