|
1 | 1 | #!/bin/bash
|
2 | 2 |
|
3 |
| -# Copyright 2014-2015 Samsung Electronics Co., Ltd. |
| 3 | +# Copyright 2015 Samsung Electronics Co., Ltd. |
4 | 4 | #
|
5 | 5 | # Licensed under the Apache License, Version 2.0 (the "License");
|
6 | 6 | # you may not use this file except in compliance with the License.
|
|
14 | 14 | # See the License for the specific language governing permissions and
|
15 | 15 | # limitations under the License.
|
16 | 16 |
|
17 |
| -TARGET=$1 |
18 |
| -shift |
19 |
| - |
20 |
| -PARSE_ONLY="" |
21 |
| -VERBOSE_REPORT=0 |
22 |
| -TIMEOUT=5 |
23 |
| - |
24 |
| -function usage() { |
25 |
| - echo "Usage: $0 {path to engine} [-p] [-v] [-t {timeout in seconds}]" >&2 |
26 |
| - echo " -p - parser only mode" >&2 |
27 |
| - echo " -v - verbose report mode" >&2 |
28 |
| - echo " -t - timeout for each test" >&2 |
| 17 | +ENGINE="$1" |
| 18 | +PATH_TO_TEST262="$2" |
| 19 | +REPORT_PREFIX="report" |
| 20 | +RUN_PIDS="" |
| 21 | +RESULT_OK=1 |
| 22 | +TIMEOUT="5s" |
29 | 23 |
|
| 24 | +if [ $# -lt 2 ] |
| 25 | +then |
| 26 | + echo "This script performs parallel test262 compliance testing of the specified engine." |
| 27 | + echo "" |
| 28 | + echo "Usage:" |
| 29 | + echo " 1st parameter: JavaScript engine to be tested." |
| 30 | + echo " 2nd parameter: path to the directory with official test262 testsuite." |
| 31 | + echo " 3rd parameter: (optional) call this script with the '--sub-chapters' flag to print the detailed report." |
| 32 | + echo "" |
| 33 | + echo "Example:" |
| 34 | + echo " ./run-test-suite-test262.sh <engine> <test262_dir> --sub-chapters" |
30 | 35 | exit 1
|
31 |
| -} |
| 36 | +fi |
32 | 37 |
|
33 |
| -[ -x "$TARGET" ] || usage |
34 |
| - |
35 |
| -while getopts pvt: OPTION; do |
36 |
| - case "$OPTION" in |
37 |
| - p) |
38 |
| - PARSE_ONLY="--parse-only" ;; |
39 |
| - v) |
40 |
| - VERBOSE_REPORT=1 ;; |
41 |
| - t) |
42 |
| - TIMEOUT="$OPTARG"; [ "$TIMEOUT" -eq "$TIMEOUT" ] || usage;; |
43 |
| - [?]) |
44 |
| - usage ;; |
45 |
| - esac |
46 |
| -done |
47 | 38 |
|
48 |
| -TMP_DIR=`mktemp -d --tmpdir=./` |
| 39 | +rm "${REPORT_PREFIX}".* &> /dev/null |
| 40 | + |
| 41 | +declare -a CHAPTER07_TO_TEST=("7.1" "7.2" "7.3" "7.4" "7.5" "7.6" "7.6.1" "7.7" \ |
| 42 | + "7.8" "7.8.1" "7.8.2" "7.8.3" "7.8.4" "7.8.5" "7.9" "7.9.2") |
| 43 | +declare -a CHAPTER08_TO_TEST=("8.1" "8.2" "8.3" "8.4" "8.5" "8.6" "8.6.1" "8.6.2" "8.7" "8.7.1" "8.7.2" "8.8" "8.12" \ |
| 44 | + "8.12.1" "8.12.3" "8.12.4" "8.12.5" "8.12.6" "8.12.7" "8.12.8" "8.12.9") |
| 45 | +declare -a CHAPTER09_TO_TEST=("9.1" "9.2" "9.3" "9.3.1" "9.4" "9.5" "9.6" "9.7" "9.8" "9.8.1" "9.9") |
| 46 | +declare -a CHAPTER10_TO_TEST=("10.1" "10.1.1" "10.2" "10.2.1" "10.2.2" "10.2.3" "10.3" "10.3.1" "10.4" "10.4.1" \ |
| 47 | + "10.4.2" "10.4.3" "10.5" "10.6") |
| 48 | +declare -a CHAPTER11_TO_TEST=("11.1" "11.1.1" "11.1.2" "11.1.3" "11.1.4" "11.1.5" "11.1.6" "11.2" "11.2.1" "11.2.2" \ |
| 49 | + "11.2.3" "11.2.4" "11.3" "11.3.1" "11.3.2" "11.4" "11.4.1" "11.4.2" "11.4.3" \ |
| 50 | + "11.4.4" "11.4.5" "11.4.6" "11.4.7" "11.4.8" "11.4.9" "11.5" "11.5.1" "11.5.2" "11.5.3" \ |
| 51 | + "11.6" "11.6.1" "11.6.2" "11.7" "11.7.1" "11.7.2" "11.7.3" "11.8" "11.8.1" "11.8.2" \ |
| 52 | + "11.8.3" "11.8.4" "11.8.6" "11.8.7" "11.9" "11.9.1" "11.9.2" "11.9.4" "11.9.5" "11.10" \ |
| 53 | + "11.11" "11.12" "11.13" "11.13.1" "11.13.2" "11.14") |
| 54 | +declare -a CHAPTER12_TO_TEST=("12.1" "12.2" "12.2.1" "12.3" "12.4" "12.5" "12.6" "12.6.1" "12.6.2" "12.6.3" "12.6.4" \ |
| 55 | + "12.7" "12.8" "12.9" "12.10" "12.10.1" "12.11" "12.12" "12.13" "12.14" "12.14.1") |
| 56 | +declare -a CHAPTER13_TO_TEST=("13.1" "13.2" "13.2.1" "13.2.2" "13.2.3") |
| 57 | +declare -a CHAPTER14_TO_TEST=("14.1") |
| 58 | +declare -a CHAPTER14_TO_TEST=("12.6.4") |
| 59 | +declare -a CHAPTER15_TO_TEST=("15.1" "15.1.1" "15.1.2" "15.1.3" "15.1.4" "15.1.5" "15.2" "15.2.1" "15.2.2" "15.2.3" \ |
| 60 | + "15.2.4" "15.2.5" "15.3" "15.3.1" "15.3.2" "15.3.3" "15.3.4" "15.3.5" "15.4" "15.4.1" \ |
| 61 | + "15.4.2" "15.4.3" "15.4.4" "15.4.5" "15.5" "15.5.1" "15.5.2" "15.5.3" "15.5.4" "15.5.5" \ |
| 62 | + "15.6" "15.6.1" "15.6.2" "15.6.3" "15.6.4" "15.6.5" "15.7" "15.7.1" "15.7.2" "15.7.3" \ |
| 63 | + "15.7.4" "15.7.5" "15.8" "15.8.1" "15.8.2" "15.9" "15.9.1" "15.9.2" "15.9.3" "15.9.4" \ |
| 64 | + "15.9.5" "15.9.6" "15.10" "15.10.1" "15.10.2" "15.10.3" "15.10.4" "15.10.5" "15.10.6" \ |
| 65 | + "15.10.7" "15.11" "15.11.1" "15.11.2" "15.11.3" "15.11.4" "15.11.5" "15.11.6" "15.11.7" \ |
| 66 | + "15.12" "15.12.1" "15.12.2" "15.12.3") |
| 67 | +declare -a FULL_CHAPTERS_TO_TEST=("ch06" "ch07" "ch08" "ch09" "ch10" "ch11" "ch12" "ch13" "ch14" "ch15") |
| 68 | +declare -a SUB_CHAPTERS_TO_TEST=("${CHAPTER07_TO_TEST[@]}" "${CHAPTER08_TO_TEST[@]}" "${CHAPTER09_TO_TEST[@]}" \ |
| 69 | + "${CHAPTER10_TO_TEST[@]}" "${CHAPTER11_TO_TEST[@]}" "${CHAPTER12_TO_TEST[@]}" \ |
| 70 | + "${CHAPTER13_TO_TEST[@]}" "${CHAPTER14_TO_TEST[@]}" "${CHAPTER15_TO_TEST[@]}") |
| 71 | + |
| 72 | +declare -a CHAPTERS_TO_TEST=("${FULL_CHAPTERS_TO_TEST[@]}") |
| 73 | + |
| 74 | +if [[ $* == *--sub-chapters* ]] |
| 75 | +then |
| 76 | + declare -a CHAPTERS_TO_TEST=("${SUB_CHAPTERS_TO_TEST[@]}") |
| 77 | +fi |
49 | 78 |
|
50 |
| -echo $TMP_DIR | grep -q tmp || exit 1 |
| 79 | +function run_test262 () { |
| 80 | + ARG_ENGINE="$1" |
| 81 | + ARG_TEST262_PATH="$2" |
| 82 | + ARG_CHAPTER="$3" |
51 | 83 |
|
52 |
| -trap clean_on_exit INT |
| 84 | + "${ARG_TEST262_PATH}"/tools/packaging/test262.py --command "timeout ${TIMEOUT} ${ARG_ENGINE}" \ |
| 85 | + --tests="${ARG_TEST262_PATH}" --full-summary "${ARG_CHAPTER}" \ |
| 86 | + > "${REPORT_PREFIX}"."${ARG_CHAPTER}" |
| 87 | +} |
53 | 88 |
|
54 |
| -function clean_on_exit() { |
55 |
| - rm -rf $TMP_DIR |
| 89 | +function show_report_results () { |
| 90 | + ARG_CHAPTER="$1" |
56 | 91 |
|
57 |
| - [[ $1 == "OK" ]] || exit 1 |
| 92 | + echo "" |
| 93 | + echo "Chapter ${ARG_CHAPTER}:" |
| 94 | + grep -A3 "=== Summary ===" "${REPORT_PREFIX}"."${ARG_CHAPTER}" |
| 95 | + echo "===============" |
58 | 96 | }
|
59 | 97 |
|
60 |
| -TEST_SUITE_PATH="./tests/test262/test" |
61 |
| -STA_JS="$TEST_SUITE_PATH/harness/sta-jerry.js" |
62 |
| -MAX_THREADS=${MAX_CPUS:-32} |
63 |
| - |
64 |
| -[ $VERBOSE_REPORT -eq 0 ] || rm -f jerry.error.* |
65 |
| -rm -f test262.report |
66 |
| - |
67 |
| -find $TEST_SUITE_PATH/suite -name *.js -printf "%p %D%i\0" | xargs -0 -n 1 -P $MAX_THREADS bash -c ' |
68 |
| - target=$0 |
69 |
| - sta_js_path=$1; |
70 |
| - timeout=$2; |
71 |
| - tmp_dir=$3; |
72 |
| - test=`echo $4 | cut -d " " -f 1`; |
73 |
| - test_id=`echo $4 | cut -d " " -f 2`; |
74 |
| - chapter=`echo $test | cut -d "/" -f 6`; |
75 |
| -
|
76 |
| - grep "\* @negative" $test 2>&1 >/dev/null; |
77 |
| - negative=$?; |
78 |
| -
|
79 |
| - cmd="$target $test $sta_js_path"; |
80 |
| - output=`ulimit -t $timeout; $cmd 2>&1`; |
81 |
| - status=$?; |
82 |
| - output=`echo "$output" | sed -e "s/^/ /"` |
83 |
| -
|
84 |
| - if [[ $status -eq 0 && $negative -eq 0 || $status -ne 0 && $negative -ne 0 ]]; |
85 |
| - then |
86 |
| - (echo "===================="; |
87 |
| - echo "Run \"$cmd\" failed: $status"; |
88 |
| - echo "---------------------"; |
89 |
| - echo "$output"; |
90 |
| - echo;) >> $tmp_dir/jerry.error."$chapter"."$test_id"; |
91 |
| - else |
92 |
| - (echo "===================="; |
93 |
| - echo "Run \"$cmd\" successful"; |
94 |
| - echo "---------------------"; |
95 |
| - echo "$output"; |
96 |
| - echo;) >> $tmp_dir/jerry.ok."$chapter"."$test_id"; |
97 |
| - fi; |
98 |
| -' "$TARGET $PARSE" $STA_JS $TIMEOUT $TMP_DIR 2>/dev/null; |
99 |
| - |
100 |
| -if [ $VERBOSE_REPORT -eq 1 ] |
101 |
| -then |
102 |
| - for CHAPTER in `ls $TEST_SUITE_PATH/suite`; |
103 |
| - do |
104 |
| - cat $TMP_DIR/jerry.error."$CHAPTER".* > jerry.error."$CHAPTER" |
105 |
| - done |
106 |
| -fi |
| 98 | +echo "Starting test262 testing for ${ENGINE}." |
107 | 99 |
|
108 |
| -cat $TMP_DIR/jerry.ok.* $TMP_DIR/jerry.error.* | grep "^Run" > test262.report |
109 |
| - |
110 |
| -ok_num=`grep "successful$" test262.report | wc -l` |
111 |
| -error_num=`grep "Run .* failed: [0-9]*$" test262.report | wc -l` |
| 100 | +for TEST_NAME in "${CHAPTERS_TO_TEST[@]}" |
| 101 | +do |
| 102 | + run_test262 "${ENGINE}" "${PATH_TO_TEST262}" "$TEST_NAME" & |
| 103 | + RUN_PIDS="$RUN_PIDS $!"; |
| 104 | +done |
112 | 105 |
|
113 |
| -ok_percent=`echo $ok_num $error_num | awk '{print 100.0 * $1 / ($1 + $2);}'` |
114 |
| -error_percent=`echo $ok_num $error_num | awk '{print 100.0 * $2 / ($1 + $2);}'` |
| 106 | +for RUN_PIDS in $RUN_PIDS |
| 107 | +do |
| 108 | + wait "$RUN_PIDS" || RESULT_OK=0 |
| 109 | +done; |
| 110 | +#[ $RESULT_OK -eq 1 ] || exit 1 |
115 | 111 |
|
116 |
| -echo -e "OK:\t$ok_num ($ok_percent %)" |
117 |
| -echo -e "FAIL:\t$error_num ($error_percent %)" |
118 |
| -echo |
119 |
| -echo -e "Report:\t$PWD/test262.report" |
| 112 | +echo "Testing is completed." |
120 | 113 |
|
121 |
| -clean_on_exit OK |
| 114 | +for TEST_NAME in "${CHAPTERS_TO_TEST[@]}" |
| 115 | +do |
| 116 | + echo "$TEST_NAME" |
| 117 | + show_report_results "$TEST_NAME" |
| 118 | +done |
0 commit comments