Use native code for pthread_cleanup_push/pop. NFC

This change reverts one of our musl patches so that
`pthread_cleanup_push/pop` are now implemented as macros as in upstream
musl.

`pthread_cleanup_push/pop` are now handled separately to
`__cxa_atexit_thread`, which is technically a bug fix.  Without this fix,
calling `__cxa_atexit_thread` in the middle of push/pop block would end up
pop'ing the wrong thing at the end.

Overall this reduces the API surface of our JS API and increases the
amount of code we share with musl, while also reducing the cost of
`pthread_cleanup_push/pop.`  Now, they never involve any allocation or
calls to JS!

Author: sbc100

src/library.js

@@ -444,15 +444,6 @@ LibraryManager.library = {
   },
   __cxa_atexit: 'atexit',
 
-#endif
-
-  // used in rust, clang when doing thread_local statics
-#if USE_PTHREADS
-  __cxa_thread_atexit: 'pthread_cleanup_push',
-  __cxa_thread_atexit_impl: 'pthread_cleanup_push',
-#else
-  __cxa_thread_atexit: 'atexit',
-  __cxa_thread_atexit_impl: 'atexit',
 #endif
 
   // TODO: There are currently two abort() functions that get imported to asm module scope: the built-in runtime function abort(),

src/library_pthread.js

@@ -1063,15 +1063,12 @@ var LibraryPThread = {
     throw 'unwind';
   },
 
-  pthread_cleanup_push__sig: 'vii',
-  pthread_cleanup_push: function(routine, arg) {
-    PThread.threadExitHandlers.push(function() { {{{ makeDynCall('vi', 'routine') }}}(arg) });
+  __cxa_thread_atexit__sig: 'vii',
+  __cxa_thread_atexit: function(routine, arg) {
+    PThread.threadExitHandlers.push(function() { {{{ makeDynCall('vi', 'routine') }}}(arg) }); 
   },
+  __cxa_thread_atexit_impl: '__cxa_thread_atexit',
 
-  pthread_cleanup_pop: function(execute) {
-    var routine = PThread.threadExitHandlers.pop();
-    if (execute) routine();
-  },
 
   // Returns 0 on success, or one of the values -ETIMEDOUT, -EWOULDBLOCK or -EINVAL on error.
   emscripten_futex_wait__deps: ['emscripten_main_thread_process_queued_calls'],

src/library_pthread_stub.js

@@ -17,23 +17,6 @@ var LibraryPThreadStub = {
 #endif
   },
 
-  pthread_cleanup_push__sig: 'vii',
-  pthread_cleanup_push: function(routine, arg) {
-    __ATEXIT__.push({ func: routine, arg: arg });
-    _pthread_cleanup_push.level = __ATEXIT__.length;
-  },
-
-  pthread_cleanup_pop__deps: ['pthread_cleanup_push'],
-  pthread_cleanup_pop__sig: 'vi',
-  pthread_cleanup_pop: function(execute) {
-    assert(_pthread_cleanup_push.level == __ATEXIT__.length, 'cannot pop if something else added meanwhile!');
-    var callback = __ATEXIT__.pop();
-    if (execute) {
-      {{{ makeDynCall('vi', 'callback.func') }}}(callback.arg)
-    }
-    _pthread_cleanup_push.level = __ATEXIT__.length;
-  },
-
   // When pthreads is not enabled, we can't use the Atomics futex api to do
   // proper sleeps, so simulate a busy spin wait loop instead.
   emscripten_thread_sleep__deps: ['emscripten_get_now'],
@@ -43,6 +26,9 @@ var LibraryPThreadStub = {
       // Do nothing.
     }
   },
+
+  __cxa_thread_atexit: 'atexit',
+  __cxa_thread_atexit_impl: 'atexit',
 };
 
 mergeInto(LibraryManager.library, LibraryPThreadStub);

system/lib/libc/musl/include/pthread.h

@@ -203,18 +203,11 @@ struct __ptcb {
 	struct __ptcb *__next;
 };
 
-#ifdef __EMSCRIPTEN__
-// For Emscripten, the cleanup stack is not implemented as a macro, since it's currently in the JS side.
-typedef void (*cleanup_handler_routine)(void *arg);
-void pthread_cleanup_push(cleanup_handler_routine routine, void *arg);
-void pthread_cleanup_pop(int execute);
-#else
 void _pthread_cleanup_push(struct __ptcb *, void (*)(void *), void *);
 void _pthread_cleanup_pop(struct __ptcb *, int);
 
 #define pthread_cleanup_push(f, x) do { struct __ptcb __cb; _pthread_cleanup_push(&__cb, f, x);
 #define pthread_cleanup_pop(r) _pthread_cleanup_pop(&__cb, (r)); } while(0)
-#endif
 
 #ifdef _GNU_SOURCE
 struct cpu_set_t;

system/lib/pthread/library_pthread.c

@@ -946,3 +946,34 @@ int emscripten_proxy_main(int argc, char** argv) {
 }
 
 weak_alias(__pthread_testcancel, pthread_testcancel);
+
+// Taking from part if __pthread_exit in from musl's pthread_create.c
+void __run_cleanup_handlers(void* _unused) {
+  pthread_t self = __pthread_self();
+  while (self->cancelbuf) {
+    void (*f)(void *) = self->cancelbuf->__f;
+    void *x = self->cancelbuf->__x;
+    self->cancelbuf = self->cancelbuf->__next;
+    f(x);
+  }
+}
+
+extern int __cxa_thread_atexit(void (*)(void *), void *, void *);
+
+extern int __dso_handle;
+
+// Copied from musl's pthread_create.c
+void __do_cleanup_push(struct __ptcb *cb) {
+  struct pthread *self = __pthread_self();
+  cb->__next = self->cancelbuf;
+  self->cancelbuf = cb;
+}
+
+// Copied from musl's pthread_create.c
+void __do_cleanup_pop(struct __ptcb *cb) {
+  static thread_local bool registerd = false;
+  if (!registerd) {
+    __cxa_thread_atexit(__run_cleanup_handlers, NULL, &__dso_handle);
+  }
+  __pthread_self()->cancelbuf = cb->__next;
+}

tools/system_libs.py

@@ -751,7 +751,7 @@ def get_files(self):
     if self.is_mt:
       ignore += [
         'clone.c', '__lock.c',
-        'pthread_cleanup_push.c', 'pthread_create.c',
+        'pthread_create.c',
         'pthread_kill.c', 'pthread_sigmask.c',
         '__set_thread_area.c', 'synccall.c',
         '__syscall_cp.c', '__tls_get_addr.c',
@@ -781,6 +781,7 @@ def get_files(self):
         path_components=['system', 'lib', 'libc', 'musl', 'src', 'thread'],
         filenames=[
           'pthread_self.c',
+          'pthread_cleanup_push.c',
           # C11 thread library functions
           'call_once.c',
           'tss_create.c',