wamr-test-suites: Add support for ARM/RISCV by QEMU (bytecodealliance…

…#1704)

Enhance wamr-test-suites to add QEMU and Firmware options:
`./test_wamr.sh -Q <qemu> -F <firmware>`

product-mini/platforms/posix/main.c

@@ -6,6 +6,7 @@
 #ifndef _GNU_SOURCE
 #define _GNU_SOURCE
 #endif
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
@@ -149,7 +150,9 @@ app_instance_repl(wasm_module_inst_t module_inst)
     size_t len = 0;
     ssize_t n;
 
-    while ((printf("webassembly> "), n = getline(&cmd, &len, stdin)) != -1) {
+    while ((printf("webassembly> "), fflush(stdout),
+            n = getline(&cmd, &len, stdin))
+           != -1) {
         bh_assert(n > 0);
         if (cmd[n - 1] == '\n') {
             if (n == 1)

tests/wamr-test-suites/spec-test-script/all.py

@@ -5,14 +5,9 @@
 #
 
 import argparse
-import hashlib
 import multiprocessing as mp
 import os
 import pathlib
-import random
-import shlex
-import shutil
-import string
 import subprocess
 import sys
 import time
@@ -24,6 +19,7 @@
 PLATFORM_NAME = os.uname().sysname.lower()
 IWASM_CMD = "../../../product-mini/platforms/" + PLATFORM_NAME + "/build/iwasm"
 IWASM_SGX_CMD = "../../../product-mini/platforms/linux-sgx/enclave-sample/iwasm"
+IWASM_QEMU_CMD = "iwasm"
 SPEC_TEST_DIR = "spec/test/core"
 WAST2WASM_CMD = "./wabt/out/gcc/Release/wat2wasm"
 WAMRC_CMD = "../../../wamr-compiler/build/wamrc"
@@ -32,8 +28,11 @@
 class TargetAction(argparse.Action):
     TARGET_MAP = {
         "ARMV7_VFP": "armv7",
-        "RISCV64": "riscv64_lp64d",
-        "RISCV64_LP64": "riscv64_lp64d",
+        "RISCV32": "riscv32_ilp32",
+        "RISCV32_ILP32": "riscv32_ilp32",
+        "RISCV32_ILP32D": "riscv32_ilp32d",
+        "RISCV64": "riscv64_lp64",
+        "RISCV64_LP64": "riscv64_lp64",
         "RISCV64_LP64D": "riscv64_lp64",
         "THUMBV7_VFP": "thumbv7",
         "X86_32": "i386",
@@ -105,6 +104,8 @@ def test_case(
     xip_flag=False,
     clean_up_flag=True,
     verbose_flag=True,
+    qemu_flag=False,
+    qemu_firmware='',
 ):
     case_path = pathlib.Path(case_path).resolve()
     case_name = case_path.stem
@@ -125,14 +126,23 @@ def test_case(
     CMD.append("--wast2wasm")
     CMD.append(WAST2WASM_CMD)
     CMD.append("--interpreter")
-    CMD.append(IWASM_CMD if not sgx_flag else IWASM_SGX_CMD)
+    if sgx_flag:
+        CMD.append(IWASM_SGX_CMD)
+    elif qemu_flag:
+        CMD.append(IWASM_QEMU_CMD)
+    else:
+        CMD.append(IWASM_CMD)
     CMD.append("--aot-compiler")
     CMD.append(WAMRC_CMD)
 
     if aot_flag:
         CMD.append("--aot")
-        CMD.append("--aot-target")
-        CMD.append(target)
+
+    CMD.append("--target")
+    CMD.append(target)
+
+    if multi_module_flag:
+        CMD.append("--multi-module")
 
     if multi_thread_flag:
         CMD.append("--multi-thread")
@@ -146,6 +156,11 @@ def test_case(
     if xip_flag:
         CMD.append("--xip")
 
+    if qemu_flag:
+        CMD.append("--qemu")
+        CMD.append("--qemu-firmware")
+        CMD.append(qemu_firmware)
+
     if not clean_up_flag:
         CMD.append("--no_cleanup")
 
@@ -206,6 +221,8 @@ def test_suite(
     clean_up_flag=True,
     verbose_flag=True,
     parl_flag=False,
+    qemu_flag=False,
+    qemu_firmware=''
 ):
     suite_path = pathlib.Path(SPEC_TEST_DIR).resolve()
     if not suite_path.exists():
@@ -239,13 +256,19 @@ def test_suite(
                         xip_flag,
                         clean_up_flag,
                         verbose_flag,
+                        qemu_flag,
+                        qemu_firmware,
                     ],
                 )
 
             for case_name, result in results.items():
                 try:
-                    # 5 min / case
-                    result.wait(300)
+                    if qemu_flag:
+                        # 60 min / case, testing on QEMU may be very slow
+                        result.wait(7200)
+                    else:
+                        # 5 min / case
+                        result.wait(300)
                     if not result.successful():
                         failed_case += 1
                     else:
@@ -268,6 +291,8 @@ def test_suite(
                     xip_flag,
                     clean_up_flag,
                     verbose_flag,
+                    qemu_flag,
+                    qemu_firmware,
                 )
                 successful_case += 1
             except Exception as e:
@@ -349,6 +374,19 @@ def main():
         dest="parl_flag",
         help="To run whole test suite parallelly",
     )
+    parser.add_argument(
+        "--qemu",
+        action="store_true",
+        default=False,
+        dest="qemu_flag",
+        help="To run whole test suite in qemu",
+    )
+    parser.add_argument(
+        "--qemu-firmware",
+        default="",
+        dest="qemu_firmware",
+        help="Firmware required by qemu",
+    )
     parser.add_argument(
         "--quiet",
         action="store_false",
@@ -389,6 +427,8 @@ def main():
             options.clean_up_flag,
             options.verbose_flag,
             options.parl_flag,
+            options.qemu_flag,
+            options.qemu_firmware,
         )
         end = time.time_ns()
         print(
@@ -408,6 +448,8 @@ def main():
                     options.xip_flag,
                     options.clean_up_flag,
                     options.verbose_flag,
+                    options.qemu_flag,
+                    options.qemu_firmware
                 )
             else:
                 ret = True