Also apply opt to OS-specific TLS impls

Author: orlp

library/std/src/sys/thread_local/key/racy.rs

@@ -34,14 +34,16 @@ impl LazyKey {
         LazyKey { key: AtomicUsize::new(KEY_SENTVAL), dtor }
     }
 
-    #[inline]
+    #[inline(always)]
     pub fn force(&self) -> super::Key {
         match self.key.load(Ordering::Acquire) {
             KEY_SENTVAL => self.lazy_init() as super::Key,
             n => n as super::Key,
         }
     }
 
+    #[cold]
+    #[inline(never)]
     fn lazy_init(&self) -> usize {
         // POSIX allows the key created here to be KEY_SENTVAL, but the compare_exchange
         // below relies on using KEY_SENTVAL as a sentinel value to check who won the

library/std/src/sys/thread_local/key/windows.rs

@@ -58,7 +58,7 @@ impl LazyKey {
         }
     }
 
-    #[inline]
+    #[inline(always)]
     pub fn force(&'static self) -> Key {
         match self.key.load(Acquire) {
             0 => unsafe { self.init() },
@@ -67,6 +67,7 @@ impl LazyKey {
     }
 
     #[cold]
+    #[inline(never)]
     unsafe fn init(&'static self) -> Key {
         if self.dtor.is_some() {
             let mut pending = c::FALSE;

library/std/src/sys/thread_local/key/xous.rs

@@ -67,6 +67,7 @@ unsafe extern "Rust" {
     static DTORS: Atomic<*mut Node>;
 }
 
+#[inline]
 fn tls_ptr_addr() -> *mut *mut u8 {
     let mut tp: usize;
     unsafe {
@@ -80,14 +81,20 @@ fn tls_ptr_addr() -> *mut *mut u8 {
 
 /// Creates an area of memory that's unique per thread. This area will
 /// contain all thread local pointers.
+#[inline]
 fn tls_table() -> &'static mut [*mut u8] {
     let tp = tls_ptr_addr();
 
     if !tp.is_null() {
-        return unsafe {
-            core::slice::from_raw_parts_mut(tp, TLS_MEMORY_SIZE / size_of::<*mut u8>())
-        };
+        unsafe { core::slice::from_raw_parts_mut(tp, TLS_MEMORY_SIZE / size_of::<*mut u8>()) }
+    } else {
+        tls_table_slow()
     }
+}
+
+#[cold]
+#[inline(never)]
+fn tls_table_slow() -> &'static mut [*mut u8] {
     // If the TP register is `0`, then this thread hasn't initialized
     // its TLS yet. Allocate a new page to store this memory.
     let tp = unsafe {

library/std/src/sys/thread_local/os.rs

@@ -68,6 +68,7 @@ impl<T: 'static> Storage<T> {
     ///
     /// The resulting pointer may not be used after reentrant inialialization
     /// or thread destruction has occurred.
+    #[inline]
     pub fn get(&'static self, i: Option<&mut Option<T>>, f: impl FnOnce() -> T) -> *const T {
         let key = self.key.force();
         let ptr = unsafe { get(key) as *mut Value<T> };
@@ -84,6 +85,8 @@ impl<T: 'static> Storage<T> {
     /// # Safety
     /// * `key` must be the result of calling `self.key.force()`
     /// * `ptr` must be the current value associated with `key`.
+    #[cold]
+    #[inline(never)]
     unsafe fn try_initialize(
         key: Key,
         ptr: *mut Value<T>,