Move __cxa_thread_atexit to native code

Rather than using a separate JS array use pthread TLS key which get
cleaned during `__pthread_tsd_run_dtors`.

Followup to #14484 and #14464 which both move more of the cleanup
handling for threads and processes onto the native side.

Author: sbc100

src/library_pthread.js

@@ -96,7 +96,6 @@ var LibraryPThread = {
     },
     // Maps pthread_t to pthread info objects
     pthreads: {},
-    threadExitHandlers: [], // An array of C functions to run when this thread exits.
 
 #if PTHREADS_PROFILING
     createProfilerBlock: function(pthreadPtr) {
@@ -165,12 +164,8 @@ var LibraryPThread = {
 #endif
 
     runExitHandlers: function() {
-      while (PThread.threadExitHandlers.length > 0) {
-        PThread.threadExitHandlers.pop()();
-      }
-
       // Call into the musl function that runs destructors of all thread-specific data.
-      if (ENVIRONMENT_IS_PTHREAD && _pthread_self()) ___pthread_tsd_run_dtors();
+      if (_pthread_self()) ___pthread_tsd_run_dtors();
     },
 
     runExitHandlersAndDeinitThread: function(tb, exitCode) {
@@ -1058,13 +1053,6 @@ var LibraryPThread = {
     throw 'unwind';
   },
 
-  __cxa_thread_atexit__sig: 'vii',
-  __cxa_thread_atexit: function(routine, arg) {
-    PThread.threadExitHandlers.push(function() { {{{ makeDynCall('vi', 'routine') }}}(arg) }); 
-  },
-  __cxa_thread_atexit_impl: '__cxa_thread_atexit',
-
-
   // Returns 0 on success, or one of the values -ETIMEDOUT, -EWOULDBLOCK or -EINVAL on error.
   emscripten_futex_wait__deps: ['emscripten_main_thread_process_queued_calls'],
   emscripten_futex_wait: function(addr, val, timeout) {

src/postamble_minimal.js

@@ -23,12 +23,12 @@ function run() {
 #else
   var ret = _main();
 
-#if EXIT_RUNTIME
-  callRuntimeCallbacks(__ATEXIT__);
-  <<< ATEXITS >>>
 #if USE_PTHREADS
   PThread.runExitHandlers();
 #endif
+#if EXIT_RUNTIME
+  callRuntimeCallbacks(__ATEXIT__);
+  <<< ATEXITS >>>
 #endif
 
 #if IN_TEST_HARNESS

src/preamble.js

@@ -426,12 +426,12 @@ function exitRuntime() {
 #if USE_PTHREADS
   if (ENVIRONMENT_IS_PTHREAD) return; // PThreads reuse the runtime from the main thread.
 #endif
-#if EXIT_RUNTIME
-  callRuntimeCallbacks(__ATEXIT__);
-  <<< ATEXITS >>>
 #if USE_PTHREADS
   PThread.runExitHandlers();
 #endif
+#if EXIT_RUNTIME
+  callRuntimeCallbacks(__ATEXIT__);
+  <<< ATEXITS >>>
 #endif
   runtimeExited = true;
 }

system/lib/libc/musl/src/thread/pthread_key_create.c

@@ -40,10 +40,9 @@ int __pthread_key_delete(pthread_key_t k)
 }
 
 #ifdef __EMSCRIPTEN__
-void EMSCRIPTEN_KEEPALIVE __pthread_tsd_run_dtors()
-#else
-void __pthread_tsd_run_dtors()
+EMSCRIPTEN_KEEPALIVE
 #endif
+void __pthread_tsd_run_dtors()
 {
 	pthread_t self = __pthread_self();
 	int i, j, not_finished = self->tsd_used;

system/lib/pthread/cxa_thread_atexit.c

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2019 The Emscripten Authors.  All rights reserved.
+ * Emscripten is available under two separate licenses, the MIT license and the
+ * University of Illinois/NCSA Open Source License.  Both these licenses can be
+ * found in the LICENSE file.
+ * Inspired by libcxxabi/src/cxa_thread_atexit.cpp.
+ * The main reasons we don't use that version direclty are:
+ * 1. We want to be able to use __cxa_thread_atexit in pure C programs
+ *    where libcxxabi is not linked in at all.
+ * 2. The libcxxabi relies on TLS variables and we use to free our TLS data
+ *    block on thread exit.  This would cause a chicken and egg issue
+ *    where the TLS variables in libcxxabi/src/cxa_thread_atexit.cpp would
+ *    be freed while porcessing the list of __cxa_thread_atexit handlers
+ */
+#include <assert.h>
+#include <stdatomic.h>
+#include <stdbool.h>
+#include <pthread.h>
+#include <stdlib.h>
+
+#include "libc.h"
+
+typedef void(*Dtor)(void*);
+
+typedef struct DtorList {
+  Dtor dtor;
+  void* obj;
+  struct DtorList* next;
+} DtorList;
+
+void run_dtors(void* arg) {
+  DtorList* dtors = (DtorList*)arg;
+  DtorList* head;
+  while ((head = dtors)) {
+    dtors = head->next;
+    head->dtor(head->obj);
+    free(head);
+  }
+}
+
+static pthread_key_t key;
+atomic_bool key_created = false;
+
+// Note: emscripten_tls_init is 49, and must come before that since
+// it calls __cxa_thread_atexit
+__attribute__((constructor(48)))
+static void make_key() {
+  pthread_key_create(&key, run_dtors);
+  key_created = true;
+}
+
+int __cxa_thread_atexit_impl(Dtor dtor, void* obj, void* dso_symbol) {
+  assert(key_created);
+  DtorList* old_head = pthread_getspecific(key);
+  DtorList* head = (DtorList*)(malloc(sizeof(DtorList)));
+  assert(head);
+  head->dtor = dtor;
+  head->obj = obj;
+  head->next = old_head;
+  pthread_setspecific(key, head);
+  return 0;
+}
+
+weak_alias(__cxa_thread_atexit_impl, __cxa_thread_atexit);

system/lib/pthread/library_pthread.c

@@ -6,8 +6,7 @@
  */
 
 #define _GNU_SOURCE
-#include "../internal/libc.h"
-#include "../internal/pthread_impl.h"
+
 #include <assert.h>
 #include <dirent.h>
 #include <errno.h>
@@ -35,6 +34,9 @@
 #include <emscripten/threading.h>
 #include <emscripten/stack.h>
 
+#include "libc.h"
+#include "pthread_impl.h"
+
 // With LLVM 3.6, C11 is the default compilation mode.
 // gets() is deprecated under that standard, but emcc
 // still provides it, so always include it in the build.

tests/other/metadce/minimal_Oz_USE_PTHREADS_PROXY_TO_PTHREAD.exports

@@ -11,7 +11,7 @@ J
 K
 L
 M
-N
+r
 s
 t
 u

tests/other/metadce/minimal_Oz_USE_PTHREADS_PROXY_TO_PTHREAD.funcs

@@ -34,6 +34,8 @@ $emscripten_sync_run_in_main_thread
 $emscripten_tls_init
 $free_tls
 $init_mparams
+$nodtor
+$run_dtors
 $sbrk
 $stackAlloc
 $stackRestore

tests/other/metadce/minimal_Oz_USE_PTHREADS_PROXY_TO_PTHREAD.imports

@@ -15,4 +15,3 @@ a.n
 a.o
 a.p
 a.q
-a.r

tests/other/metadce/minimal_Oz_USE_PTHREADS_PROXY_TO_PTHREAD.sent

@@ -15,4 +15,3 @@ n
 o
 p
 q
-r

tests/other/metadce/minimal_Oz_USE_PTHREADS_PROXY_TO_PTHREAD.size

@@ -1 +1 @@
-15872
+16310

tools/system_libs.py

@@ -774,6 +774,7 @@ def get_files(self):
         filenames=[
           'library_pthread.c',
           'emscripten_thread_state.s',
+          'cxa_thread_atexit.c',
         ])
     else:
       ignore += ['thread']