backport of: macOS: workaround a keymgr/libunwind deadlock issue on x86_64 since 10.9

Author: vtjnash

src/signals-mach.c

@@ -17,6 +17,18 @@
 
 #include "julia_assert.h"
 
+// private keymgr stuff
+#define KEYMGR_GCC3_DW2_OBJ_LIST 302
+enum {
+  NM_ALLOW_RECURSION = 1,
+  NM_RECURSION_ILLEGAL = 2
+};
+extern void _keymgr_set_and_unlock_processwide_ptr(unsigned int key, void *ptr);
+extern int _keymgr_unlock_processwide_ptr(unsigned int key);
+extern void *_keymgr_get_and_lock_processwide_ptr(unsigned int key);
+extern int _keymgr_get_and_lock_processwide_ptr_2(unsigned int key, void **result);
+extern int _keymgr_set_lockmode_processwide_ptr(unsigned int key, unsigned int mode);
+
 static void attach_exception_port(thread_port_t thread, int segv_only);
 
 // low 16 bits are the thread id, the next 8 bits are the original gc_state
@@ -78,6 +90,17 @@ void *mach_segv_listener(void *arg)
 
 static void allocate_segv_handler()
 {
+    // ensure KEYMGR_GCC3_DW2_OBJ_LIST is initialized, as this requires malloc
+    // and thus can deadlock when used without first initializing it.
+    // Apple caused this problem in their libunwind in 10.9 (circa keymgr-28)
+    // when they removed this part of the code from keymgr.
+    // Much thanks to Apple for providing source code, or this would probably
+    // have simply remained unsolved forever on their platform.
+    // This is similar to just calling checkKeyMgrRegisteredFDEs
+    // (this is quite thread-unsafe)
+    if (_keymgr_set_lockmode_processwide_ptr(KEYMGR_GCC3_DW2_OBJ_LIST, NM_ALLOW_RECURSION))
+        jl_error("_keymgr_set_lockmode_processwide_ptr failed");
+
     arraylist_new(&suspended_threads, jl_n_threads);
     pthread_t thread;
     pthread_attr_t attr;
@@ -421,6 +444,8 @@ void *mach_profile_listener(void *arg)
         // sample each thread, round-robin style in reverse order
         // (so that thread zero gets notified last)
         jl_lock_profile();
+        void *unused = NULL;
+        int keymgr_locked = _keymgr_get_and_lock_processwide_ptr_2(KEYMGR_GCC3_DW2_OBJ_LIST, &unused) == 0;
         for (i = jl_n_threads; i-- > 0; ) {
             // if there is no space left, break early
             if (bt_size_cur >= bt_size_max - 1)
@@ -469,6 +494,8 @@ void *mach_profile_listener(void *arg)
             // We're done! Resume the thread.
             jl_thread_resume(i, 0);
         }
+        if (keymgr_locked)
+            _keymgr_unlock_processwide_ptr(KEYMGR_GCC3_DW2_OBJ_LIST);
         jl_unlock_profile();
         if (running) {
             // Reset the alarm