Move __cxa_thread_atexit to native code

Rather than using a separate JS array use pthread TLS key which get
cleaned during `__pthread_tsd_run_dtors`.

Followup to #14484 and #14464 which both move more of the cleanup
handling for threads and processes onto the native side.

Author: sbc100

src/library_pthread.js

@@ -96,7 +96,6 @@ var LibraryPThread = {
     },
     // Maps pthread_t to pthread info objects
     pthreads: {},
-    threadExitHandlers: [], // An array of C functions to run when this thread exits.
 
 #if PTHREADS_PROFILING
     createProfilerBlock: function(pthreadPtr) {
@@ -165,10 +164,6 @@ var LibraryPThread = {
 #endif
 
     runExitHandlers: function() {
-      while (PThread.threadExitHandlers.length > 0) {
-        PThread.threadExitHandlers.pop()();
-      }
-
       // Call into the musl function that runs destructors of all thread-specific data.
 #if ASSERTIONS
       assert(_pthread_self())
@@ -1062,13 +1057,6 @@ var LibraryPThread = {
     throw 'unwind';
   },
 
-  __cxa_thread_atexit__sig: 'vii',
-  __cxa_thread_atexit: function(routine, arg) {
-    PThread.threadExitHandlers.push(function() { {{{ makeDynCall('vi', 'routine') }}}(arg) }); 
-  },
-  __cxa_thread_atexit_impl: '__cxa_thread_atexit',
-
-
   // Returns 0 on success, or one of the values -ETIMEDOUT, -EWOULDBLOCK or -EINVAL on error.
   emscripten_futex_wait__deps: ['emscripten_main_thread_process_queued_calls'],
   emscripten_futex_wait: function(addr, val, timeout) {

system/lib/libc/musl/src/thread/pthread_key_create.c

@@ -40,10 +40,9 @@ int __pthread_key_delete(pthread_key_t k)
 }
 
 #ifdef __EMSCRIPTEN__
-void EMSCRIPTEN_KEEPALIVE __pthread_tsd_run_dtors()
-#else
-void __pthread_tsd_run_dtors()
+EMSCRIPTEN_KEEPALIVE
 #endif
+void __pthread_tsd_run_dtors()
 {
 	pthread_t self = __pthread_self();
 	int i, j, not_finished = self->tsd_used;

system/lib/pthread/cxa_thread_atexit.c

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2019 The Emscripten Authors.  All rights reserved.
+ * Emscripten is available under two separate licenses, the MIT license and the
+ * University of Illinois/NCSA Open Source License.  Both these licenses can be
+ * found in the LICENSE file.
+ * Inspired by libcxxabi/src/cxa_thread_atexit.cpp.
+ * The main reasons we don't use that version direclty are:
+ * 1. We want to be able to use __cxa_thread_atexit in pure C programs
+ *    where libcxxabi is not linked in at all.
+ * 2. The libcxxabi relies on TLS variables and we use to free our TLS data
+ *    block on thread exit.  This would cause a chicken and egg issue
+ *    where the TLS variables in libcxxabi/src/cxa_thread_atexit.cpp would
+ *    be freed while processing the list of __cxa_thread_atexit handlers
+ */
+#include <assert.h>
+#include <stdbool.h>
+#include <stdatomic.h>
+#include <pthread.h>
+#include <stdlib.h>
+
+#include "libc.h"
+
+typedef void (*Dtor)(void*);
+
+typedef struct DtorList {
+  Dtor dtor;
+  void* obj;
+  struct DtorList* next;
+} DtorList;
+
+void run_dtors(void* arg) {
+  DtorList* dtors = (DtorList*)arg;
+  DtorList* head;
+  while ((head = dtors)) {
+    dtors = head->next;
+    head->dtor(head->obj);
+    free(head);
+  }
+}
+
+static pthread_key_t key;
+atomic_bool key_created = false;
+static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+
+static void ensure_key() {
+  // can't use pthread_once here because it depends __cxa_thread_atexit
+  // the first rime its called to register the cleanup handlers
+  if (!key_created) {
+    pthread_mutex_lock(&mutex);
+    if (!key_created) {
+      pthread_key_create(&key, run_dtors);
+      key_created = true;
+    }
+    pthread_mutex_unlock(&mutex);
+  }
+}
+
+int __cxa_thread_atexit_impl(Dtor dtor, void* obj, void* dso_symbol) {
+  ensure_key();
+  DtorList* old_head = pthread_getspecific(key);
+  DtorList* head = (DtorList*)(malloc(sizeof(DtorList)));
+  assert(head);
+  head->dtor = dtor;
+  head->obj = obj;
+  head->next = old_head;
+  pthread_setspecific(key, head);
+  return 0;
+}
+
+weak_alias(__cxa_thread_atexit_impl, __cxa_thread_atexit);

system/lib/pthread/library_pthread.c

@@ -6,8 +6,7 @@
  */
 
 #define _GNU_SOURCE
-#include "../internal/libc.h"
-#include "../internal/pthread_impl.h"
+
 #include <assert.h>
 #include <dirent.h>
 #include <errno.h>
@@ -35,6 +34,9 @@
 #include <emscripten/threading.h>
 #include <emscripten/stack.h>
 
+#include "libc.h"
+#include "pthread_impl.h"
+
 // With LLVM 3.6, C11 is the default compilation mode.
 // gets() is deprecated under that standard, but emcc
 // still provides it, so always include it in the build.

tests/other/metadce/minimal_main_Oz_USE_PTHREADS_PROXY_TO_PTHREAD.exports

@@ -13,7 +13,7 @@ L
 M
 N
 O
-P
+r
 s
 t
 u

tests/other/metadce/minimal_main_Oz_USE_PTHREADS_PROXY_TO_PTHREAD.funcs

@@ -36,6 +36,8 @@ $emscripten_tls_init
 $free_tls
 $init_mparams
 $main
+$nodtor
+$run_dtors
 $sbrk
 $stackAlloc
 $stackRestore

tests/other/metadce/minimal_main_Oz_USE_PTHREADS_PROXY_TO_PTHREAD.imports

@@ -15,4 +15,3 @@ a.n
 a.o
 a.p
 a.q
-a.r

tests/other/metadce/minimal_main_Oz_USE_PTHREADS_PROXY_TO_PTHREAD.sent

@@ -15,4 +15,3 @@ n
 o
 p
 q
-r

tests/other/metadce/minimal_main_Oz_USE_PTHREADS_PROXY_TO_PTHREAD.size

@@ -1 +1 @@
-16012
+16455

tools/system_libs.py

@@ -773,6 +773,7 @@ def get_files(self):
         filenames=[
           'library_pthread.c',
           'emscripten_thread_state.s',
+          'cxa_thread_atexit.c',
         ])
     else:
       ignore += ['thread']