Make jl_codegen_lock a sleeping mutex instead of a spinning one

Author: pchintalapudi

src/aotcompile.cpp

@@ -299,7 +299,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm
 
     // compile all methods for the current world and type-inference world
 
-    JL_SPIN_LOCK(&jl_codegen_lock);
+    JL_SLEEP_LOCK(&jl_codegen_lock);
     auto target_info = clone.withModuleDo([&](Module &M) {
         return std::make_pair(M.getDataLayout(), Triple(M.getTargetTriple()));
     });
@@ -351,7 +351,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm
         // finally, make sure all referenced methods also get compiled or fixed up
         jl_compile_workqueue(emitted, *clone.getModuleUnlocked(), params, policy);
     }
-    JL_SPIN_UNLOCK(&jl_codegen_lock); // Might GC
+    JL_SLEEP_UNLOCK(&jl_codegen_lock); // Might GC
     JL_GC_POP();
 
     // process the globals array, before jl_merge_module destroys them
@@ -2034,15 +2034,15 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t* dump, jl_method_instance_t *mi, siz
         uint8_t measure_compile_time_enabled = jl_atomic_load_relaxed(&jl_measure_compile_time_enabled);
         if (measure_compile_time_enabled)
             compiler_start_time = jl_hrtime();
-        JL_SPIN_LOCK(&jl_codegen_lock);
+        JL_SLEEP_LOCK(&jl_codegen_lock);
         auto target_info = m.withModuleDo([&](Module &M) {
             return std::make_pair(M.getDataLayout(), Triple(M.getTargetTriple()));
         });
         jl_codegen_params_t output(*ctx, std::move(target_info.first), std::move(target_info.second));
         output.world = world;
         output.params = &params;
         auto decls = jl_emit_code(m, mi, src, jlrettype, output);
-        JL_SPIN_UNLOCK(&jl_codegen_lock); // Might GC
+        JL_SLEEP_UNLOCK(&jl_codegen_lock); // Might GC
 
         Function *F = NULL;
         if (m) {

src/gf.c

@@ -3843,12 +3843,12 @@ JL_DLLEXPORT void jl_typeinf_timing_end(uint64_t start)
 
 JL_DLLEXPORT void jl_typeinf_lock_begin(void)
 {
-    JL_SPIN_LOCK(&jl_codegen_lock);
+    JL_SLEEP_LOCK(&jl_codegen_lock);
 }
 
 JL_DLLEXPORT void jl_typeinf_lock_end(void)
 {
-    JL_SPIN_UNLOCK(&jl_codegen_lock);
+    JL_SLEEP_UNLOCK(&jl_codegen_lock);
 }
 
 #ifdef __cplusplus

src/init.c

@@ -725,7 +725,7 @@ static void init_global_mutexes(void) {
     JL_SPIN_MUTEX_INIT(&precomp_statement_out_lock, "precomp_statement_out_lock");
     JL_SPIN_MUTEX_INIT(&newly_inferred_mutex, "newly_inferred_mutex");
     JL_SPIN_MUTEX_INIT(&global_roots_lock, "global_roots_lock");
-    JL_SPIN_MUTEX_INIT(&jl_codegen_lock, "jl_codegen_lock");
+    JL_SLEEP_MUTEX_INIT(&jl_codegen_lock, "jl_codegen_lock");
     JL_SPIN_MUTEX_INIT(&typecache_lock, "typecache_lock");
 }
 

src/jitlayers.cpp

@@ -343,7 +343,7 @@ int jl_compile_extern_c_impl(LLVMOrcThreadSafeModuleRef llvmmod, void *p, void *
         backing = jl_create_ts_module("cextern", pparams ? pparams->tsctx : ctx, pparams ? pparams->imaging : imaging_default());
         into = &backing;
     }
-    JL_SPIN_LOCK(&jl_codegen_lock);
+    JL_SLEEP_LOCK(&jl_codegen_lock);
     auto target_info = into->withModuleDo([&](Module &M) {
         return std::make_pair(M.getDataLayout(), Triple(M.getTargetTriple()));
     });
@@ -366,7 +366,7 @@ int jl_compile_extern_c_impl(LLVMOrcThreadSafeModuleRef llvmmod, void *p, void *
         if (success && llvmmod == NULL)
             jl_ExecutionEngine->addModule(std::move(*into));
     }
-    JL_SPIN_UNLOCK(&jl_codegen_lock);
+    JL_SLEEP_UNLOCK(&jl_codegen_lock);
     if (timed) {
         if (measure_compile_time_enabled) {
             auto end = jl_hrtime();
@@ -439,7 +439,7 @@ jl_code_instance_t *jl_generate_fptr_impl(jl_method_instance_t *mi JL_PROPAGATES
     // if we don't have any decls already, try to generate it now
     jl_code_info_t *src = NULL;
     JL_GC_PUSH1(&src);
-    JL_SPIN_LOCK(&jl_codegen_lock); // also disables finalizers, to prevent any unexpected recursion
+    JL_SLEEP_LOCK(&jl_codegen_lock); // also disables finalizers, to prevent any unexpected recursion
     jl_value_t *ci = jl_rettype_inferred(mi, world, world);
     jl_code_instance_t *codeinst = (ci == jl_nothing ? NULL : (jl_code_instance_t*)ci);
     if (codeinst) {
@@ -482,7 +482,7 @@ jl_code_instance_t *jl_generate_fptr_impl(jl_method_instance_t *mi JL_PROPAGATES
     else {
         codeinst = NULL;
     }
-    JL_SPIN_UNLOCK(&jl_codegen_lock);
+    JL_SLEEP_UNLOCK(&jl_codegen_lock);
     if (timed) {
         if (measure_compile_time_enabled) {
             uint64_t t_comp = jl_hrtime() - compiler_start_time;
@@ -511,7 +511,7 @@ void jl_generate_fptr_for_unspecialized_impl(jl_code_instance_t *unspec)
     uint8_t measure_compile_time_enabled = jl_atomic_load_relaxed(&jl_measure_compile_time_enabled);
     if (measure_compile_time_enabled)
         compiler_start_time = jl_hrtime();
-    JL_SPIN_LOCK(&jl_codegen_lock);
+    JL_SLEEP_LOCK(&jl_codegen_lock);
     if (jl_atomic_load_relaxed(&unspec->invoke) == NULL) {
         jl_code_info_t *src = NULL;
         JL_GC_PUSH1(&src);
@@ -538,7 +538,7 @@ void jl_generate_fptr_for_unspecialized_impl(jl_code_instance_t *unspec)
         jl_atomic_cmpswap(&unspec->invoke, &null, jl_fptr_interpret_call_addr);
         JL_GC_POP();
     }
-    JL_SPIN_UNLOCK(&jl_codegen_lock); // Might GC
+    JL_SLEEP_UNLOCK(&jl_codegen_lock); // Might GC
     if (timed) {
         if (measure_compile_time_enabled) {
             auto end = jl_hrtime();
@@ -573,7 +573,7 @@ jl_value_t *jl_dump_method_asm_impl(jl_method_instance_t *mi, size_t world,
             uint8_t measure_compile_time_enabled = jl_atomic_load_relaxed(&jl_measure_compile_time_enabled);
             if (measure_compile_time_enabled)
                 compiler_start_time = jl_hrtime();
-            JL_SPIN_LOCK(&jl_codegen_lock); // also disables finalizers, to prevent any unexpected recursion
+            JL_SLEEP_LOCK(&jl_codegen_lock); // also disables finalizers, to prevent any unexpected recursion
             specfptr = (uintptr_t)jl_atomic_load_relaxed(&codeinst->specptr.fptr);
             if (specfptr == 0) {
                 jl_code_info_t *src = jl_type_infer(mi, world, 0);
@@ -597,7 +597,7 @@ jl_value_t *jl_dump_method_asm_impl(jl_method_instance_t *mi, size_t world,
                 }
                 JL_GC_POP();
             }
-            JL_SPIN_UNLOCK(&jl_codegen_lock);
+            JL_SLEEP_UNLOCK(&jl_codegen_lock);
             if (timed) {
                 if (measure_compile_time_enabled) {
                     auto end = jl_hrtime();

src/julia_internal.h

@@ -1399,7 +1399,7 @@ JL_DLLEXPORT void jl_set_next_task(jl_task_t *task) JL_NOTSAFEPOINT;
 // -- synchronization utilities -- //
 
 extern jl_spin_mutex_t typecache_lock;
-extern JL_DLLEXPORT jl_spin_mutex_t jl_codegen_lock;
+extern JL_DLLEXPORT jl_sleep_mutex_t jl_codegen_lock;
 
 #if defined(__APPLE__)
 void jl_mach_gc_end(void);

src/julia_locks.h

@@ -164,6 +164,12 @@ static inline void jl_dyn_mutex_unlock_nogc(void *lock)
 #define JL_SPIN_LOCK_NOGC(m) jl_spin_mutex_lock_nogc(m)
 #define JL_SPIN_UNLOCK_NOGC(m) jl_spin_mutex_unlock_nogc(m)
 
+#define JL_SLEEP_MUTEX_INIT(m, name) jl_sleep_mutex_init(m, name)
+#define JL_SLEEP_LOCK(m) jl_sleep_mutex_lock(m)
+#define JL_SLEEP_UNLOCK(m) jl_sleep_mutex_unlock(m)
+#define JL_SLEEP_LOCK_NOGC(m) jl_sleep_mutex_lock_nogc(m)
+#define JL_SLEEP_UNLOCK_NOGC(m) jl_sleep_mutex_unlock_nogc(m)
+
 #ifdef __cplusplus
 }
 #endif

src/threading.c

@@ -483,7 +483,7 @@ static void jl_delete_thread(void *value) JL_NOTSAFEPOINT_ENTER
 //// the other threads time to fail and emit their failure message
 //__attribute__((destructor)) static void _waitthreaddeath(void) { sleep(1); }
 
-JL_DLLEXPORT jl_spin_mutex_t jl_codegen_lock;
+JL_DLLEXPORT jl_sleep_mutex_t jl_codegen_lock;
 jl_spin_mutex_t typecache_lock;
 
 JL_DLLEXPORT ssize_t jl_tls_offset = -1;
@@ -893,6 +893,11 @@ JL_DLLEXPORT void _jl_sleep_mutex_wait(jl_task_t *self, jl_sleep_mutex_t *lock,
         lock->count = 1;
         return;
     }
+    if (jl_atomic_load_relaxed(&lock->owner) == self) {
+        // we already own the lock, just increment the count
+        lock->count++;
+        return;
+    }
     jl_profile_lock_start_wait(lock);
     //Do some retries?
     jl_atomic_fetch_add(&lock->waiters, 0b10);