Update spec test cases to latest version (#889)

Update spec test cases to commit 2460ad02b51fb5ed5824f44de287a8638b19a5f8, and modify wamr test suite script as the SIMD cases have been added into spec main repo by default, no need to clone SIMD repo again when testing SIMD.
bytecodealliance · Dec 15, 2021 · e70867c · e70867c
1 parent bbaf0a3
commit e70867c
Show file tree

Hide file tree

Showing 3 changed files with 68 additions and 111 deletions.
diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c
@@ -3342,6 +3342,7 @@ create_sections(const uint8 *buf, uint32 size, WASMSection **p_section_list,
                 if (last_section_index != (uint8)-1
                     && (section_index <= last_section_index)) {
                     set_error_buf(error_buf, error_buf_size,
+                                  "unexpected content after last section or "
                                   "junk after last section");
                     return false;
                 }

diff --git a/tests/wamr-test-suites/spec-test-script/all.py b/tests/wamr-test-suites/spec-test-script/all.py
@@ -204,70 +204,59 @@ def test_suite(
     xip_flag=False,
     clean_up_flag=True,
     verbose_flag=True,
+    parl_flag=False,
 ):
     suite_path = pathlib.Path(SPEC_TEST_DIR).resolve()
     if not suite_path.exists():
         print(f"can not find spec test cases at {suite_path}")
         return False
 
-    case_list = sorted(suite_path.glob("**/*.wast"))
+    case_list = sorted(suite_path.glob("*.wast"))
+    if simd_flag:
+        simd_case_list = sorted(suite_path.glob("simd/*.wast"))
+        case_list.extend(simd_case_list)
+
     case_count = len(case_list)
     failed_case = 0
     successful_case = 0
-    for case_path in case_list:
-        try:
-            test_case(
-                str(case_path),
-                target,
-                aot_flag,
-                sgx_flag,
-                multi_module_flag,
-                multi_thread_flag,
-                simd_flag,
-                xip_flag,
-                clean_up_flag,
-                verbose_flag,
-            )
-            successful_case += 1
-        except Exception:
-            failed_case += 1
-            break
-
-    print(
-        f"IN ALL {case_count} cases: {successful_case} PASS, {failed_case} FAIL, {case_count - successful_case - failed_case} SKIP"
-    )
-
-    return 0 == failed_case
 
+    if parl_flag:
+        print(f"----- Run the whole spec test suite on {mp.cpu_count()} cores -----")
+        with mp.Pool() as pool:
+            results = {}
+            for case_path in case_list:
+                results[case_path.stem] = pool.apply_async(
+                    test_case,
+                    [
+                        str(case_path),
+                        target,
+                        aot_flag,
+                        sgx_flag,
+                        multi_module_flag,
+                        multi_thread_flag,
+                        simd_flag,
+                        xip_flag,
+                        clean_up_flag,
+                        verbose_flag,
+                    ],
+                )
 
-def test_suite_parallelly(
-    target,
-    aot_flag=False,
-    sgx_flag=False,
-    multi_module_flag=False,
-    multi_thread_flag=False,
-    simd_flag=False,
-    xip_flag=False,
-    clean_up_flag=False,
-    verbose_flag=False,
-):
-
-    suite_path = pathlib.Path(SPEC_TEST_DIR).resolve()
-    if not suite_path.exists():
-        print(f"can not find spec test cases at {suite_path}")
-        return False
-
-    case_list = sorted(suite_path.glob("**/*.wast"))
-    case_count = len(case_list)
-    failed_case = 0
-    successful_case = 0
-    print(f"----- Run the whole spec test suite on {mp.cpu_count()} cores -----")
-    with mp.Pool() as pool:
-        results = {}
+            for case_name, result in results.items():
+                try:
+                    # 5 min / case
+                    result.wait(300)
+                    if not result.successful():
+                        failed_case += 1
+                    else:
+                        successful_case += 1
+                except mp.TimeoutError:
+                    print(f"{case_name} meets TimeoutError")
+                    failed_case += 1
+    else:
+        print(f"----- Run the whole spec test suite -----")
         for case_path in case_list:
-            results[case_path.stem] = pool.apply_async(
-                test_case,
-                [
+            try:
+                test_case(
                     str(case_path),
                     target,
                     aot_flag,
@@ -278,20 +267,11 @@ def test_suite_parallelly(
                     xip_flag,
                     clean_up_flag,
                     verbose_flag,
-                ],
-            )
-
-        for case_name, result in results.items():
-            try:
-                # 5 min / case
-                result.wait(300)
-                if not result.successful():
-                    failed_case += 1
-                else:
-                    successful_case += 1
-            except mp.TimeoutError:
-                print(f"{case_name} meets TimeoutError")
+                )
+                successful_case += 1
+            except Exception:
                 failed_case += 1
+                break
 
     print(
         f"IN ALL {case_count} cases: {successful_case} PASS, {failed_case} FAIL, {case_count - successful_case - failed_case} SKIP"
@@ -396,37 +376,23 @@ def main():
             options.clean_up_flag = False
             options.verbose_flag = False
 
-            start = time.time_ns()
-            ret = test_suite_parallelly(
-                options.target,
-                options.aot_flag,
-                options.sgx_flag,
-                options.multi_module_flag,
-                options.multi_thread_flag,
-                options.simd_flag,
-                options.xip_flag,
-                options.clean_up_flag,
-                options.verbose_flag,
-            )
-            end = time.time_ns()
-            print(
-                f"It takes {((end - start) / 1000000):,} ms to run test_suite_parallelly"
-            )
-        else:
-            start = time.time_ns()
-            ret = test_suite(
-                options.target,
-                options.aot_flag,
-                options.sgx_flag,
-                options.multi_module_flag,
-                options.multi_thread_flag,
-                options.simd_flag,
-                options.xip_flag,
-                options.clean_up_flag,
-                options.verbose_flag,
-            )
-            end = time.time_ns()
-            print(f"It takes {((end - start) / 1000000):,} ms to run test_suite")
+        start = time.time_ns()
+        ret = test_suite(
+            options.target,
+            options.aot_flag,
+            options.sgx_flag,
+            options.multi_module_flag,
+            options.multi_thread_flag,
+            options.simd_flag,
+            options.xip_flag,
+            options.clean_up_flag,
+            options.verbose_flag,
+            options.parl_flag,
+        )
+        end = time.time_ns()
+        print(
+            f"It takes {((end - start) / 1000000):,} ms to run test_suite {'parallelly' if options.parl_flag else ''}"
+        )
     else:
         try:
             for case in options.cases:

diff --git a/tests/wamr-test-suites/test_wamr.sh b/tests/wamr-test-suites/test_wamr.sh
@@ -279,8 +279,12 @@ function spec_test()
     # restore from XX_ignore_cases.patch
     # resotre branch
     git checkout -B master
-    git reset --hard 397399a70565609bf142d211891724e21bffd01f
+    # [spec] Fix instruction table (#1402) Thu Dec 2 17:21:54 2021 +0100
+    git reset --hard 2460ad02b51fb5ed5824f44de287a8638b19a5f8
     git apply ../../spec-test-script/ignore_cases.patch
+    if [[ ${ENABLE_SIMD} == 1 ]]; then
+        git apply ../../spec-test-script/simd_ignore_cases.patch
+    fi
 
     # udpate thread cases
     if [ ${ENABLE_MULTI_THREAD} == 1 ]; then
@@ -298,20 +302,6 @@ function spec_test()
         git apply ../../spec-test-script/thread_proposal_ignore_cases.patch
     fi
 
-    # udpate SIMD cases
-    if [[ ${ENABLE_SIMD} == 1 ]]; then
-        echo "checkout spec for SIMD proposal"
-        # check spec test cases for simd
-        if [[ -z $(git remote | grep "\<simd\>") ]]; then
-            git remote add simd https://github.com/WebAssembly/simd.git
-        fi
-
-        git fetch simd
-        git checkout simd/main -- test/core/simd
-
-        git apply ../../spec-test-script/simd_ignore_cases.patch
-    fi
-
     popd
     echo $(pwd)