chore: move wasm-testsuite to a seperate repo & update tests

Signed-off-by: Henry Gressmann <mail@henrygressmann.de>

Author: explodingcamera

CHANGELOG.md

@@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - **Breaking:**: New backwards-incompatible version of the twasm format (upgraded `rkyv` to 0.8.0)
-- Increased MSRV to 1.81.0
+- Increased MSRV to 1.83.0
 
 ### Fixed
 

Cargo.lock

@@ -20,7 +20,8 @@ tinywasm-types={version="0.9.0-alpha.0", path="../types", default-features=false
 libm={version="0.2", default-features=false}
 
 [dev-dependencies]
-wasm-testsuite={path="../wasm-testsuite"}
+wasm-testsuite={version="0.3.2"}
+indexmap="2.7"
 wast={workspace=true}
 eyre={workspace=true}
 pretty_env_logger={workspace=true}
@@ -67,7 +68,7 @@ harness=false
 test=false
 
 [[test]]
-name="test-wasm-simd"
+name="test-wasm-relaxed-simd"
 harness=false
 test=false
 

crates/tinywasm/Cargo.toml

@@ -1,2 +1,2 @@
 0.8.0,142,0,[{"name":"annotations.wast","passed":74,"failed":0},{"name":"annotations/simd_lane.wast (skipped)","passed":0,"failed":0},{"name":"id.wast","passed":7,"failed":0},{"name":"token.wast","passed":61,"failed":0}]
-0.9.0-alpha.0,142,0,[{"name":"annotations.wast","passed":74,"failed":0},{"name":"annotations/simd_lane.wast (skipped)","passed":0,"failed":0},{"name":"id.wast","passed":7,"failed":0},{"name":"token.wast","passed":61,"failed":0}]
+0.9.0-alpha.0,142,0,[{"name":"annotations.wast","passed":74,"failed":0},{"name":"id.wast","passed":7,"failed":0},{"name":"simd_lane.wast (skipped)","passed":0,"failed":0},{"name":"token.wast","passed":61,"failed":0}]

crates/tinywasm/tests/generated/wasm-2.csv

@@ -1,2 +1,2 @@
 0.8.0,1872,0,[{"name":"address0.wast","passed":92,"failed":0},{"name":"address1.wast","passed":127,"failed":0},{"name":"align.wast","passed":160,"failed":0},{"name":"align0.wast","passed":5,"failed":0},{"name":"binary.wast","passed":126,"failed":0},{"name":"binary0.wast","passed":7,"failed":0},{"name":"data.wast","passed":61,"failed":0},{"name":"data0.wast","passed":7,"failed":0},{"name":"data1.wast","passed":14,"failed":0},{"name":"data_drop0.wast","passed":11,"failed":0},{"name":"exports0.wast","passed":8,"failed":0},{"name":"float_exprs0.wast","passed":14,"failed":0},{"name":"float_exprs1.wast","passed":3,"failed":0},{"name":"float_memory0.wast","passed":30,"failed":0},{"name":"imports.wast","passed":175,"failed":0},{"name":"imports0.wast","passed":8,"failed":0},{"name":"imports1.wast","passed":5,"failed":0},{"name":"imports2.wast","passed":20,"failed":0},{"name":"imports3.wast","passed":10,"failed":0},{"name":"imports4.wast","passed":16,"failed":0},{"name":"linking0.wast","passed":6,"failed":0},{"name":"linking1.wast","passed":14,"failed":0},{"name":"linking2.wast","passed":11,"failed":0},{"name":"linking3.wast","passed":14,"failed":0},{"name":"load.wast","passed":118,"failed":0},{"name":"load0.wast","passed":3,"failed":0},{"name":"load1.wast","passed":18,"failed":0},{"name":"load2.wast","passed":38,"failed":0},{"name":"memory-multi.wast","passed":6,"failed":0},{"name":"memory.wast","passed":86,"failed":0},{"name":"memory_copy0.wast","passed":29,"failed":0},{"name":"memory_copy1.wast","passed":14,"failed":0},{"name":"memory_fill0.wast","passed":16,"failed":0},{"name":"memory_grow.wast","passed":157,"failed":0},{"name":"memory_init0.wast","passed":13,"failed":0},{"name":"memory_size.wast","passed":49,"failed":0},{"name":"memory_size0.wast","passed":8,"failed":0},{"name":"memory_size1.wast","passed":15,"failed":0},{"name":"memory_size2.wast","passed":21,"failed":0},{"name":"memory_size3.wast","passed":2,"failed":0},{"name":"memory_trap0.wast","passed":14,"failed":0},{"name":"memory_trap1.wast","passed":168,"failed":0},{"name":"multi-memory/simd_memory-multi.wast (skipped)","passed":0,"failed":0},{"name":"start0.wast","passed":9,"failed":0},{"name":"store.wast","passed":111,"failed":0},{"name":"store0.wast","passed":5,"failed":0},{"name":"store1.wast","passed":13,"failed":0},{"name":"traps0.wast","passed":15,"failed":0}]
-0.9.0-alpha.0,1872,0,[{"name":"address0.wast","passed":92,"failed":0},{"name":"address1.wast","passed":127,"failed":0},{"name":"align.wast","passed":160,"failed":0},{"name":"align0.wast","passed":5,"failed":0},{"name":"binary.wast","passed":126,"failed":0},{"name":"binary0.wast","passed":7,"failed":0},{"name":"data.wast","passed":61,"failed":0},{"name":"data0.wast","passed":7,"failed":0},{"name":"data1.wast","passed":14,"failed":0},{"name":"data_drop0.wast","passed":11,"failed":0},{"name":"exports0.wast","passed":8,"failed":0},{"name":"float_exprs0.wast","passed":14,"failed":0},{"name":"float_exprs1.wast","passed":3,"failed":0},{"name":"float_memory0.wast","passed":30,"failed":0},{"name":"imports.wast","passed":175,"failed":0},{"name":"imports0.wast","passed":8,"failed":0},{"name":"imports1.wast","passed":5,"failed":0},{"name":"imports2.wast","passed":20,"failed":0},{"name":"imports3.wast","passed":10,"failed":0},{"name":"imports4.wast","passed":16,"failed":0},{"name":"linking0.wast","passed":6,"failed":0},{"name":"linking1.wast","passed":14,"failed":0},{"name":"linking2.wast","passed":11,"failed":0},{"name":"linking3.wast","passed":14,"failed":0},{"name":"load.wast","passed":118,"failed":0},{"name":"load0.wast","passed":3,"failed":0},{"name":"load1.wast","passed":18,"failed":0},{"name":"load2.wast","passed":38,"failed":0},{"name":"memory-multi.wast","passed":6,"failed":0},{"name":"memory.wast","passed":86,"failed":0},{"name":"memory_copy0.wast","passed":29,"failed":0},{"name":"memory_copy1.wast","passed":14,"failed":0},{"name":"memory_fill0.wast","passed":16,"failed":0},{"name":"memory_grow.wast","passed":157,"failed":0},{"name":"memory_init0.wast","passed":13,"failed":0},{"name":"memory_size.wast","passed":49,"failed":0},{"name":"memory_size0.wast","passed":8,"failed":0},{"name":"memory_size1.wast","passed":15,"failed":0},{"name":"memory_size2.wast","passed":21,"failed":0},{"name":"memory_size3.wast","passed":2,"failed":0},{"name":"memory_trap0.wast","passed":14,"failed":0},{"name":"memory_trap1.wast","passed":168,"failed":0},{"name":"multi-memory/simd_memory-multi.wast (skipped)","passed":0,"failed":0},{"name":"start0.wast","passed":9,"failed":0},{"name":"store.wast","passed":111,"failed":0},{"name":"store0.wast","passed":5,"failed":0},{"name":"store1.wast","passed":13,"failed":0},{"name":"traps0.wast","passed":15,"failed":0}]
+0.9.0-alpha.0,1872,0,[{"name":"address0.wast","passed":92,"failed":0},{"name":"address1.wast","passed":127,"failed":0},{"name":"align.wast","passed":160,"failed":0},{"name":"align0.wast","passed":5,"failed":0},{"name":"binary.wast","passed":126,"failed":0},{"name":"binary0.wast","passed":7,"failed":0},{"name":"data.wast","passed":61,"failed":0},{"name":"data0.wast","passed":7,"failed":0},{"name":"data1.wast","passed":14,"failed":0},{"name":"data_drop0.wast","passed":11,"failed":0},{"name":"exports0.wast","passed":8,"failed":0},{"name":"float_exprs0.wast","passed":14,"failed":0},{"name":"float_exprs1.wast","passed":3,"failed":0},{"name":"float_memory0.wast","passed":30,"failed":0},{"name":"imports.wast","passed":175,"failed":0},{"name":"imports0.wast","passed":8,"failed":0},{"name":"imports1.wast","passed":5,"failed":0},{"name":"imports2.wast","passed":20,"failed":0},{"name":"imports3.wast","passed":10,"failed":0},{"name":"imports4.wast","passed":16,"failed":0},{"name":"linking0.wast","passed":6,"failed":0},{"name":"linking1.wast","passed":14,"failed":0},{"name":"linking2.wast","passed":11,"failed":0},{"name":"linking3.wast","passed":14,"failed":0},{"name":"load.wast","passed":118,"failed":0},{"name":"load0.wast","passed":3,"failed":0},{"name":"load1.wast","passed":18,"failed":0},{"name":"load2.wast","passed":38,"failed":0},{"name":"memory-multi.wast","passed":6,"failed":0},{"name":"memory.wast","passed":86,"failed":0},{"name":"memory_copy0.wast","passed":29,"failed":0},{"name":"memory_copy1.wast","passed":14,"failed":0},{"name":"memory_fill0.wast","passed":16,"failed":0},{"name":"memory_grow.wast","passed":157,"failed":0},{"name":"memory_init0.wast","passed":13,"failed":0},{"name":"memory_size.wast","passed":49,"failed":0},{"name":"memory_size0.wast","passed":8,"failed":0},{"name":"memory_size1.wast","passed":15,"failed":0},{"name":"memory_size2.wast","passed":21,"failed":0},{"name":"memory_size3.wast","passed":2,"failed":0},{"name":"memory_trap0.wast","passed":14,"failed":0},{"name":"memory_trap1.wast","passed":168,"failed":0},{"name":"simd_memory-multi.wast (skipped)","passed":0,"failed":0},{"name":"start0.wast","passed":9,"failed":0},{"name":"store.wast","passed":111,"failed":0},{"name":"store0.wast","passed":5,"failed":0},{"name":"store1.wast","passed":13,"failed":0},{"name":"traps0.wast","passed":15,"failed":0}]

crates/tinywasm/tests/generated/wasm-annotations.csv

@@ -1,20 +1,13 @@
 mod testsuite;
-use eyre::{eyre, Result};
-use owo_colors::OwoColorize;
-use testsuite::TestSuite;
+use eyre::Result;
 
 fn main() -> Result<()> {
-    let mut test_suite = TestSuite::new();
+    println!("Skipping 1.0 tests (False positives due to relaxed validation in later versions)");
+    Ok(())
+    // TestSuite::set_log_level(log::LevelFilter::Off);
 
-    TestSuite::set_log_level(log::LevelFilter::Off);
-    test_suite.run_spec_group(wasm_testsuite::MVP_TESTS)?;
-    test_suite.save_csv("./tests/generated/wasm-1.csv", env!("CARGO_PKG_VERSION"))?;
-
-    if test_suite.failed() {
-        println!();
-        Err(eyre!(format!("{}:\n{:#?}", "failed one or more tests".red().bold(), test_suite,)))
-    } else {
-        println!("\n\npassed all tests:\n{test_suite:#?}");
-        Ok(())
-    }
+    // let mut test_suite = TestSuite::new();
+    // test_suite.run_files(spec(&SpecVersion::V1))?;
+    // test_suite.save_csv("./tests/generated/wasm-1.csv", env!("CARGO_PKG_VERSION"))?;
+    // test_suite.report_status()
 }

crates/tinywasm/tests/generated/wasm-multi-memory.csv

@@ -1,20 +1,13 @@
 mod testsuite;
-use eyre::{eyre, Result};
-use owo_colors::OwoColorize;
+use eyre::Result;
 use testsuite::TestSuite;
+use wasm_testsuite::data::{spec, SpecVersion};
 
 fn main() -> Result<()> {
-    let mut test_suite = TestSuite::new();
-
     TestSuite::set_log_level(log::LevelFilter::Off);
-    test_suite.run_spec_group(wasm_testsuite::V2_TESTS)?;
-    test_suite.save_csv("./tests/generated/wasm-2.csv", env!("CARGO_PKG_VERSION"))?;
 
-    if test_suite.failed() {
-        println!();
-        Err(eyre!(format!("{}:\n{:#?}", "failed one or more tests".red().bold(), test_suite,)))
-    } else {
-        println!("\n\npassed all tests:\n{test_suite:#?}");
-        Ok(())
-    }
+    let mut test_suite = TestSuite::new();
+    test_suite.run_files(spec(&SpecVersion::V2))?;
+    test_suite.save_csv("./tests/generated/wasm-2.csv", env!("CARGO_PKG_VERSION"))?;
+    test_suite.report_status()
 }

crates/tinywasm/tests/test-wasm-1.rs

@@ -1,21 +1,14 @@
 mod testsuite;
-use eyre::{eyre, Result};
-use owo_colors::OwoColorize;
+use eyre::Result;
 use testsuite::TestSuite;
+use wasm_testsuite::data::{proposal, Proposal};
 
 fn main() -> Result<()> {
-    let mut test_suite = TestSuite::new();
-
     TestSuite::set_log_level(log::LevelFilter::Off);
-    test_suite.skip("annotations/simd_lane.wast");
-    test_suite.run_spec_group(wasm_testsuite::get_proposal_tests("annotations"))?;
-    test_suite.save_csv("./tests/generated/wasm-annotations.csv", env!("CARGO_PKG_VERSION"))?;
 
-    if test_suite.failed() {
-        println!();
-        Err(eyre!(format!("{}:\n{:#?}", "failed one or more tests".red().bold(), test_suite,)))
-    } else {
-        println!("\n\npassed all tests:\n{test_suite:#?}");
-        Ok(())
-    }
+    let mut test_suite = TestSuite::new();
+    test_suite.skip("simd_lane.wast");
+    test_suite.run_files(proposal(&Proposal::Annotations))?;
+    test_suite.save_csv("./tests/generated/wasm-annotations.csv", env!("CARGO_PKG_VERSION"))?;
+    test_suite.report_status()
 }