pythongh-95174: Add pthread stubs for WASI (pythonGH-95234)

Co-authored-by: Brett Cannon <brett@python.org>

Doc/library/sys.rst

@@ -1658,6 +1658,8 @@ always available.
    |                  |                                                         |
    |                  |  * ``'nt'``: Windows threads                            |
    |                  |  * ``'pthread'``: POSIX threads                         |
+   |                  |  * ``'pthread-stubs'``: stub POSIX threads              |
+   |                  |    (on WebAssembly platforms without threading support) |
    |                  |  * ``'solaris'``: Solaris threads                       |
    +------------------+---------------------------------------------------------+
    | :const:`lock`    | Name of the lock implementation:                        |

Include/cpython/pthread_stubs.h

@@ -0,0 +1,88 @@
+#ifndef Py_CPYTHON_PTRHEAD_STUBS_H
+#define Py_CPYTHON_PTRHEAD_STUBS_H
+
+#if !defined(HAVE_PTHREAD_STUBS)
+#  error "this header file requires stubbed pthreads."
+#endif
+
+#ifndef _POSIX_THREADS
+#  define _POSIX_THREADS 1
+#endif
+
+/* Minimal pthread stubs for CPython.
+ *
+ * The stubs implement the minimum pthread API for CPython.
+ * - pthread_create() fails.
+ * - pthread_exit() calls exit(0).
+ * - pthread_key_*() functions implement minimal TSS without destructor.
+ * - all other functions do nothing and return 0.
+ */
+
+#ifdef __wasi__
+// WASI's bits/alltypes.h provides type definitions when __NEED_ is set.
+// The header file can be included multiple times.
+#  define __NEED_pthread_cond_t 1
+#  define __NEED_pthread_condattr_t 1
+#  define __NEED_pthread_mutex_t 1
+#  define __NEED_pthread_mutexattr_t 1
+#  define __NEED_pthread_key_t 1
+#  define __NEED_pthread_t 1
+#  define __NEED_pthread_attr_t 1
+#  include <bits/alltypes.h>
+#else
+typedef struct { void *__x; } pthread_cond_t;
+typedef struct { unsigned __attr; } pthread_condattr_t;
+typedef struct { void *__x; } pthread_mutex_t;
+typedef struct { unsigned __attr; } pthread_mutexattr_t;
+typedef unsigned pthread_key_t;
+typedef unsigned pthread_t;
+typedef struct { unsigned __attr; } pthread_attr_t;
+#endif
+
+// mutex
+PyAPI_FUNC(int) pthread_mutex_init(pthread_mutex_t *restrict mutex,
+                                   const pthread_mutexattr_t *restrict attr);
+PyAPI_FUNC(int) pthread_mutex_destroy(pthread_mutex_t *mutex);
+PyAPI_FUNC(int) pthread_mutex_trylock(pthread_mutex_t *mutex);
+PyAPI_FUNC(int) pthread_mutex_lock(pthread_mutex_t *mutex);
+PyAPI_FUNC(int) pthread_mutex_unlock(pthread_mutex_t *mutex);
+
+// condition
+PyAPI_FUNC(int) pthread_cond_init(pthread_cond_t *restrict cond,
+                                  const pthread_condattr_t *restrict attr);
+PyAPI_FUNC(int) pthread_cond_destroy(pthread_cond_t *cond);
+PyAPI_FUNC(int) pthread_cond_wait(pthread_cond_t *restrict cond,
+                                  pthread_mutex_t *restrict mutex);
+PyAPI_FUNC(int) pthread_cond_timedwait(pthread_cond_t *restrict cond,
+                                       pthread_mutex_t *restrict mutex,
+                                       const struct timespec *restrict abstime);
+PyAPI_FUNC(int) pthread_cond_signal(pthread_cond_t *cond);
+PyAPI_FUNC(int) pthread_condattr_init(pthread_condattr_t *attr);
+PyAPI_FUNC(int) pthread_condattr_setclock(
+    pthread_condattr_t *attr, clockid_t clock_id);
+
+// pthread
+PyAPI_FUNC(int) pthread_create(pthread_t *restrict thread,
+                               const pthread_attr_t *restrict attr,
+                               void *(*start_routine)(void *),
+                               void *restrict arg);
+PyAPI_FUNC(int) pthread_detach(pthread_t thread);
+PyAPI_FUNC(pthread_t) pthread_self(void);
+PyAPI_FUNC(int) pthread_exit(void *retval) __attribute__ ((__noreturn__));
+PyAPI_FUNC(int) pthread_attr_init(pthread_attr_t *attr);
+PyAPI_FUNC(int) pthread_attr_setstacksize(pthread_attr_t *attr, size_t stacksize);
+PyAPI_FUNC(int) pthread_attr_destroy(pthread_attr_t *attr);
+
+
+// pthread_key
+#ifndef PTHREAD_KEYS_MAX
+#  define PTHREAD_KEYS_MAX 128
+#endif
+
+PyAPI_FUNC(int) pthread_key_create(pthread_key_t *key,
+                                   void (*destr_function)(void *));
+PyAPI_FUNC(int) pthread_key_delete(pthread_key_t key);
+PyAPI_FUNC(void *) pthread_getspecific(pthread_key_t key);
+PyAPI_FUNC(int) pthread_setspecific(pthread_key_t key, const void *value);
+
+#endif // Py_CPYTHON_PTRHEAD_STUBS_H

Include/cpython/pythread.h

@@ -20,6 +20,9 @@ PyAPI_FUNC(int) _PyThread_at_fork_reinit(PyThread_type_lock *lock);
        but hardcode the unsigned long to avoid errors for include directive.
     */
 #   define NATIVE_TSS_KEY_T     unsigned long
+#elif defined(HAVE_PTHREAD_STUBS)
+#   include "cpython/pthread_stubs.h"
+#   define NATIVE_TSS_KEY_T     pthread_key_t
 #else
 #   error "Require native threads. See https://bugs.python.org/issue31370"
 #endif

Lib/test/pythoninfo.py

@@ -588,8 +588,8 @@ def collect_socket(info_add):
 
     try:
         hostname = socket.gethostname()
-    except OSError:
-        # WASI SDK 15.0 does not have gethostname(2).
+    except (OSError, AttributeError):
+        # WASI SDK 16.0 does not have gethostname(2).
         if sys.platform != "wasi":
             raise
     else:

Lib/test/test_sys.py

@@ -626,7 +626,7 @@ def test_attributes(self):
     def test_thread_info(self):
         info = sys.thread_info
         self.assertEqual(len(info), 3)
-        self.assertIn(info.name, ('nt', 'pthread', 'solaris', None))
+        self.assertIn(info.name, ('nt', 'pthread', 'pthread-stubs', 'solaris', None))
         self.assertIn(info.lock, ('semaphore', 'mutex+cond', None))
 
     @unittest.skipUnless(support.is_emscripten, "only available on Emscripten")

Lib/test/test_threadsignals.py

@@ -36,7 +36,9 @@ def send_signals():
     os.kill(process_pid, signal.SIGUSR2)
     signalled_all.release()
 
+
 @threading_helper.requires_working_threading()
+@unittest.skipUnless(hasattr(signal, "alarm"), "test requires signal.alarm")
 class ThreadSignals(unittest.TestCase):
 
     def test_signals(self):

Makefile.pre.in

@@ -1544,6 +1544,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/cpython/objimpl.h \
 		$(srcdir)/Include/cpython/odictobject.h \
 		$(srcdir)/Include/cpython/picklebufobject.h \
+		$(srcdir)/Include/cpython/pthread_stubs.h \
 		$(srcdir)/Include/cpython/pyctype.h \
 		$(srcdir)/Include/cpython/pydebug.h \
 		$(srcdir)/Include/cpython/pyerrors.h \

Misc/NEWS.d/next/Build/2022-07-25-09-48-43.gh-issue-95145.ZNS3dj.rst

@@ -0,0 +1,2 @@
+wasm32-wasi builds no longer depend on WASIX's pthread stubs. Python now has
+its own stubbed pthread API.

Python/thread.c

@@ -55,8 +55,15 @@ PyThread_init_thread(void)
     PyThread__init_thread();
 }
 
-#if defined(_POSIX_THREADS)
-#   define PYTHREAD_NAME "pthread"
+#if defined(HAVE_PTHREAD_STUBS)
+#   define PYTHREAD_NAME "pthread-stubs"
+#   include "thread_pthread_stubs.h"
+#elif defined(_POSIX_THREADS)
+#   if defined(__EMSCRIPTEN__) || !defined(__EMSCRIPTEN_PTHREADS__)
+#     define PYTHREAD_NAME "pthread-stubs"
+#   else
+#     define PYTHREAD_NAME "pthread"
+#   endif
 #   include "thread_pthread.h"
 #elif defined(NT_THREADS)
 #   define PYTHREAD_NAME "nt"
@@ -171,7 +178,9 @@ PyThread_GetInfo(void)
     }
     PyStructSequence_SET_ITEM(threadinfo, pos++, value);
 
-#ifdef _POSIX_THREADS
+#ifdef HAVE_PTHREAD_STUBS
+    value = Py_NewRef(Py_None);
+#elif defined(_POSIX_THREADS)
 #ifdef USE_SEMAPHORES
     value = PyUnicode_FromString("semaphore");
 #else
@@ -182,8 +191,7 @@ PyThread_GetInfo(void)
         return NULL;
     }
 #else
-    Py_INCREF(Py_None);
-    value = Py_None;
+    value = Py_NewRef(Py_None);
 #endif
     PyStructSequence_SET_ITEM(threadinfo, pos++, value);
 

Python/thread_pthread.h

@@ -7,7 +7,9 @@
 #if defined(__APPLE__) || defined(HAVE_PTHREAD_DESTRUCTOR)
 #define destructor xxdestructor
 #endif
-#include <pthread.h>
+#ifndef HAVE_PTHREAD_STUBS
+#  include <pthread.h>
+#endif
 #if defined(__APPLE__) || defined(HAVE_PTHREAD_DESTRUCTOR)
 #undef destructor
 #endif

Python/thread_pthread_stubs.h

@@ -0,0 +1,185 @@
+#include "cpython/pthread_stubs.h"
+
+// mutex
+int
+pthread_mutex_init(pthread_mutex_t *restrict mutex,
+                   const pthread_mutexattr_t *restrict attr)
+{
+    return 0;
+}
+
+int
+pthread_mutex_destroy(pthread_mutex_t *mutex)
+{
+    return 0;
+}
+
+int
+pthread_mutex_trylock(pthread_mutex_t *mutex)
+{
+    return 0;
+}
+
+int
+pthread_mutex_lock(pthread_mutex_t *mutex)
+{
+    return 0;
+}
+
+int
+pthread_mutex_unlock(pthread_mutex_t *mutex)
+{
+    return 0;
+}
+
+// condition
+int
+pthread_cond_init(pthread_cond_t *restrict cond,
+                  const pthread_condattr_t *restrict attr)
+{
+    return 0;
+}
+
+PyAPI_FUNC(int)pthread_cond_destroy(pthread_cond_t *cond)
+{
+    return 0;
+}
+
+int
+pthread_cond_wait(pthread_cond_t *restrict cond,
+                  pthread_mutex_t *restrict mutex)
+{
+    return 0;
+}
+
+int
+pthread_cond_timedwait(pthread_cond_t *restrict cond,
+                       pthread_mutex_t *restrict mutex,
+                       const struct timespec *restrict abstime)
+{
+    return 0;
+}
+
+int
+pthread_cond_signal(pthread_cond_t *cond)
+{
+    return 0;
+}
+
+int
+pthread_condattr_init(pthread_condattr_t *attr)
+{
+    return 0;
+}
+
+int
+pthread_condattr_setclock(pthread_condattr_t *attr, clockid_t clock_id)
+{
+    return 0;
+}
+
+// pthread
+int
+pthread_create(pthread_t *restrict thread,
+               const pthread_attr_t *restrict attr,
+               void *(*start_routine)(void *),
+               void *restrict arg)
+{
+    return EAGAIN;
+}
+
+int
+pthread_detach(pthread_t thread)
+{
+    return 0;
+}
+
+PyAPI_FUNC(pthread_t) pthread_self(void)
+{
+    return 0;
+}
+
+int
+pthread_exit(void *retval)
+{
+    exit(0);
+}
+
+int
+pthread_attr_init(pthread_attr_t *attr)
+{
+    return 0;
+}
+
+int
+pthread_attr_setstacksize(
+    pthread_attr_t *attr, size_t stacksize)
+{
+    return 0;
+}
+
+int
+pthread_attr_destroy(pthread_attr_t *attr)
+{
+    return 0;
+}
+
+// pthread_key
+typedef struct {
+    bool in_use;
+    void *value;
+} py_tls_entry;
+
+static py_tls_entry py_tls_entries[PTHREAD_KEYS_MAX] = {0};
+
+int
+pthread_key_create(pthread_key_t *key, void (*destr_function)(void *))
+{
+    if (!key) {
+        return EINVAL;
+    }
+    if (destr_function != NULL) {
+        Py_FatalError("pthread_key_create destructor is not supported");
+    }
+    for (pthread_key_t idx = 0; idx < PTHREAD_KEYS_MAX; idx++) {
+        if (!py_tls_entries[idx].in_use) {
+            py_tls_entries[idx].in_use = true;
+            *key = idx;
+            return 0;
+        }
+    }
+    return EAGAIN;
+}
+
+int
+pthread_key_delete(pthread_key_t key)
+{
+    if (key < 0 || key >= PTHREAD_KEYS_MAX || !py_tls_entries[key].in_use) {
+        return EINVAL;
+    }
+    py_tls_entries[key].in_use = false;
+    py_tls_entries[key].value = NULL;
+    return 0;
+}
+
+
+void *
+pthread_getspecific(pthread_key_t key) {
+    if (key < 0 || key >= PTHREAD_KEYS_MAX || !py_tls_entries[key].in_use) {
+        return NULL;
+    }
+    return py_tls_entries[key].value;
+}
+
+int
+pthread_setspecific(pthread_key_t key, const void *value)
+{
+    if (key < 0 || key >= PTHREAD_KEYS_MAX || !py_tls_entries[key].in_use) {
+        return EINVAL;
+    }
+    py_tls_entries[key].value = (void *)value;
+    return 0;
+}
+
+// let thread_pthread define the Python API
+#include "thread_pthread.h"