Add internal tests for WASI threads (bytecodealliance#1963)

Add internal tests for WASI threads. These tests are run in addition to
the ones in the proposal:
https://github.com/WebAssembly/wasi-threads/tree/main/test/testsuite.

The purpose is to test additional and more complex scenarios.

.github/workflows/compilation_on_android_ubuntu.yml

@@ -457,6 +457,10 @@ jobs:
             $THREADS_TEST_OPTIONS,
             $WASI_TEST_OPTIONS,
           ]
+        wasi_sdk_release:
+          [
+            "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz",
+          ]
         llvm_cache_key:
           ["${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }}"]
         exclude:
@@ -493,6 +497,31 @@ jobs:
       - name: checkout
         uses: actions/checkout@v3
 
+      - name: download and install wasi-sdk
+        if: matrix.test_option == '$WASI_TEST_OPTIONS'
+        run: |
+          cd /opt
+          sudo wget ${{ matrix.wasi_sdk_release }}
+          sudo tar -xzf wasi-sdk-*.tar.gz
+          sudo mv wasi-sdk-19.0 wasi-sdk
+
+      - name: build wasi-libc (needed for wasi-threads)
+        if: matrix.test_option == '$WASI_TEST_OPTIONS'
+        run: |
+          mkdir wasi-libc
+          cd wasi-libc
+          git init
+          # "Rename thread_spawn import" commit on main branch
+          git fetch https://github.com/WebAssembly/wasi-libc \
+            8f5275796a82f8ecfd0833a4f3f444fa37ed4546
+          git checkout FETCH_HEAD
+          make \
+            AR=/opt/wasi-sdk/bin/llvm-ar \
+            NM=/opt/wasi-sdk/bin/llvm-nm \
+            CC=/opt/wasi-sdk/bin/clang \
+            THREAD_MODEL=posix
+        working-directory: core/deps
+
       - name: set env variable(if llvm are used)
         if: matrix.running_mode == 'aot' || matrix.running_mode == 'jit' || matrix.running_mode == 'multi-tier-jit'
         run: echo "USE_LLVM=true" >> $GITHUB_ENV
@@ -526,6 +555,11 @@ jobs:
         if: matrix.running_mode == 'aot' && matrix.test_option == '$WASI_TEST_OPTIONS'
         run: sudo apt-get update && sudo apt install -y jq
 
+      - name: Build WASI thread tests
+        if: matrix.test_option == '$WASI_TEST_OPTIONS'
+        run: WASI_SYSROOT=../../../../../core/deps/wasi-libc/sysroot bash build.sh
+        working-directory: ./core/iwasm/libraries/lib-wasi-threads/test/
+
       - name: run tests
         timeout-minutes: 10
         run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }}

.gitignore

@@ -13,6 +13,7 @@
 core/deps/**
 core/shared/mem-alloc/tlsf
 core/app-framework/wgl
+core/iwasm/libraries/lib-wasi-threads/test/*.wasm
 
 wamr-sdk/out/
 wamr-sdk/runtime/build_runtime_sdk/
@@ -35,3 +36,5 @@ tests/benchmarks/coremark/coremark*
 
 samples/workload/include/**
 !samples/workload/include/.gitkeep
+
+# core/iwasm/libraries/wasi-threads

core/iwasm/libraries/lib-wasi-threads/test/build.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#
+# Copyright (C) 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+
+CC=${CC:=/opt/wasi-sdk/bin/clang}
+WASI_SYSROOT=${WASI_SYSROOT:=~/dev/wasi-libc/sysroot}
+WAMR_DIR=../../../../..
+
+for test_c in *.c; do
+    test_wasm="$(basename $test_c .c).wasm"
+
+    echo "Compiling $test_c to $test_wasm"
+    $CC \
+        --sysroot $WASI_SYSROOT \
+        -target wasm32-wasi-threads \
+        -pthread -ftls-model=local-exec \
+        -z stack-size=32768 \
+        -Wl,--export=__heap_base \
+        -Wl,--export=__data_end \
+        -Wl,--shared-memory,--max-memory=1966080 \
+        -Wl,--export=wasi_thread_start \
+        -Wl,--export=malloc \
+        -Wl,--export=free \
+        -I $WAMR_DIR/samples/wasi-threads/wasm-apps \
+        $WAMR_DIR/samples/wasi-threads/wasm-apps/wasi_thread_start.S \
+        $test_c -o $test_wasm
+done

core/iwasm/libraries/lib-wasi-threads/test/common.h

@@ -0,0 +1,122 @@
+/*
+ * Copyright (C) 2022 Amazon.com Inc. or its affiliates. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <unistd.h>
+
+#include "wasi_thread_start.h"
+
+typedef enum {
+    BLOCKING_TASK_BUSY_WAIT,
+    BLOCKING_TASK_ATOMIC_WAIT,
+    BLOCKING_TASK_POLL_ONEOFF
+} blocking_task_type_t;
+
+/* Parameter to change test behavior */
+static bool termination_by_trap;
+static bool termination_in_main_thread;
+static blocking_task_type_t blocking_task_type;
+
+#define TIMEOUT_SECONDS 10ll
+#define NUM_THREADS 3
+static pthread_barrier_t barrier;
+
+typedef struct {
+    start_args_t base;
+    bool throw_exception;
+} shared_t;
+
+void
+run_long_task()
+{
+    if (blocking_task_type == BLOCKING_TASK_BUSY_WAIT) {
+        for (int i = 0; i < TIMEOUT_SECONDS; i++)
+            sleep(1);
+    }
+    else if (blocking_task_type == BLOCKING_TASK_ATOMIC_WAIT) {
+        __builtin_wasm_memory_atomic_wait32(
+            0, 0, TIMEOUT_SECONDS * 1000 * 1000 * 1000);
+    }
+    else {
+        sleep(TIMEOUT_SECONDS);
+    }
+}
+
+void
+start_job()
+{
+    /* Wait for all threads (including the main thread) to be ready */
+    pthread_barrier_wait(&barrier);
+    run_long_task(); /* Task to be interrupted */
+    assert(false && "Thread termination test failed");
+}
+
+void
+terminate_process()
+{
+    /* Wait for all threads (including the main thread) to be ready */
+    pthread_barrier_wait(&barrier);
+
+    if (termination_by_trap)
+        __builtin_trap();
+    else
+        __wasi_proc_exit(33);
+}
+
+void
+__wasi_thread_start_C(int thread_id, int *start_arg)
+{
+    shared_t *data = (shared_t *)start_arg;
+
+    if (data->throw_exception) {
+        terminate_process();
+    }
+    else {
+        start_job();
+    }
+}
+
+void
+test_termination(bool trap, bool main, blocking_task_type_t task_type)
+{
+    termination_by_trap = trap;
+    termination_in_main_thread = main;
+    blocking_task_type = task_type;
+
+    int thread_id = -1, i;
+    shared_t data[NUM_THREADS] = { 0 };
+    assert(pthread_barrier_init(&barrier, NULL, NUM_THREADS + 1) == 0
+           && "Failed to init barrier");
+
+    for (i = 0; i < NUM_THREADS; i++) {
+        /* No graceful memory free to simplify the test */
+        assert(start_args_init(&data[i].base)
+               && "Failed to allocate thread's stack");
+    }
+
+    /* Create a thread that forces termination through trap or `proc_exit` */
+    data[0].throw_exception = !termination_in_main_thread;
+    thread_id = __wasi_thread_spawn(&data[0]);
+    assert(thread_id > 0 && "Failed to create thread");
+
+    /* Create two additional threads to test exception propagation */
+    data[1].throw_exception = false;
+    thread_id = __wasi_thread_spawn(&data[1]);
+    assert(thread_id > 0 && "Failed to create thread");
+    data[2].throw_exception = false;
+    thread_id = __wasi_thread_spawn(&data[2]);
+    assert(thread_id > 0 && "Failed to create thread");
+
+    if (termination_in_main_thread) {
+        terminate_process();
+    }
+    else {
+        start_job();
+    }
+}

core/iwasm/libraries/lib-wasi-threads/test/create_threads_until_limit.c

@@ -0,0 +1,128 @@
+/*
+ * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#ifndef __wasi__
+#error This example only compiles to WASM/WASI target
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <stdbool.h>
+
+#include "wasi_thread_start.h"
+
+enum CONSTANTS {
+    MAX_NUM_THREADS = 4, /* Should be the same as "--max-threads" */
+    NUM_RETRY = 5,
+    SECOND = 1000 * 1000 * 1000, /* 1 second */
+    TIMEOUT = 10LL * SECOND
+};
+
+int g_count = 0;
+
+typedef struct {
+    start_args_t base;
+    int th_ready;
+    int th_continue;
+    int th_done;
+    bool no_ops;
+} shared_t;
+
+void
+__wasi_thread_start_C(int thread_id, int *start_arg)
+{
+    shared_t *data = (shared_t *)start_arg;
+
+    if (data->no_ops) {
+        __builtin_wasm_memory_atomic_wait32(NULL, 0, 2 * SECOND);
+        return;
+    }
+
+    __atomic_store_n(&data->th_ready, 1, __ATOMIC_SEQ_CST);
+    __builtin_wasm_memory_atomic_notify(&data->th_ready, 1);
+
+    if (__builtin_wasm_memory_atomic_wait32(&data->th_continue, 0, TIMEOUT)
+        == 2) {
+        assert(false && "Wait should not time out");
+    }
+
+    __atomic_fetch_add(&g_count, 1, __ATOMIC_SEQ_CST);
+
+    __atomic_store_n(&data->th_done, 1, __ATOMIC_SEQ_CST);
+    __builtin_wasm_memory_atomic_notify(&data->th_done, 1);
+}
+
+int
+main(int argc, char **argv)
+{
+    shared_t data[MAX_NUM_THREADS] = { 0 };
+    int thread_ids[MAX_NUM_THREADS];
+
+    for (int i = 0; i < MAX_NUM_THREADS; i++) {
+        assert(start_args_init(&data[i].base));
+        thread_ids[i] = __wasi_thread_spawn(&data[i]);
+        printf("Thread created with id=%d\n", thread_ids[i]);
+        assert(thread_ids[i] > 0 && "Thread creation failed");
+
+        for (int j = 0; j < i; j++) {
+            assert(thread_ids[i] != thread_ids[j] && "Duplicated TIDs");
+        }
+
+        if (__builtin_wasm_memory_atomic_wait32(&data[i].th_ready, 0, TIMEOUT)
+            == 2) {
+            assert(false && "Wait should not time out");
+        }
+    }
+
+    printf("Attempt to create thread when not possible\n");
+    shared_t data_fail = { 0 };
+    assert(start_args_init(&data_fail.base));
+    int thread_id = __wasi_thread_spawn(&data_fail);
+    start_args_deinit(&data_fail.base);
+    assert(thread_id < 0 && "Thread creation should fail");
+
+    printf("Unlock created threads\n");
+    for (int i = 0; i < MAX_NUM_THREADS; i++) {
+        __atomic_store_n(&data[i].th_continue, 1, __ATOMIC_SEQ_CST);
+        __builtin_wasm_memory_atomic_notify(&data[i].th_continue, 1);
+    }
+
+    printf("Wait for threads to finish\n");
+    for (int i = 0; i < MAX_NUM_THREADS; i++) {
+        if (__builtin_wasm_memory_atomic_wait32(&data[i].th_done, 0, TIMEOUT)
+            == 2) {
+            assert(false && "Wait should not time out");
+        }
+
+        start_args_deinit(&data[i].base);
+    }
+
+    printf("Value of count after update: %d\n", g_count);
+    assert(g_count == (MAX_NUM_THREADS)
+           && "Global count not updated correctly");
+
+    /* --------------------------------------------------- */
+
+    printf("Create new threads without waiting from them to finish\n");
+    shared_t data_no_join[MAX_NUM_THREADS] = { 0 };
+    for (int i = 0; i < MAX_NUM_THREADS; i++) {
+        /* No graceful memory free to simplify the test */
+        assert(start_args_init(&data_no_join[i].base));
+        data_no_join[i].no_ops = true;
+
+        int thread_id = -1;
+        for (int j = 0; j < NUM_RETRY && thread_id < 0; j++) {
+            thread_id = __wasi_thread_spawn(&data_no_join[i]);
+            if (thread_id < 0)
+                __builtin_wasm_memory_atomic_wait32(NULL, 0, SECOND);
+        }
+
+        printf("Thread created with id=%d\n", thread_id);
+        assert(thread_id > 0 && "Thread creation should succeed");
+    }
+
+    return EXIT_SUCCESS;
+}