From 4fbd71c94324965f5a3bcf01845b6da0d51076b7 Mon Sep 17 00:00:00 2001
From: Mara Bos <m-ou.se@m-ou.se>
Date: Thu, 17 Mar 2022 12:27:20 +0100
Subject: [PATCH 01/14] Return timeout status in futex_wait.

---
 library/std/src/sys/unix/futex.rs | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/library/std/src/sys/unix/futex.rs b/library/std/src/sys/unix/futex.rs
index 42ddc1d514ec7..22ba1a57fef2d 100644
--- a/library/std/src/sys/unix/futex.rs
+++ b/library/std/src/sys/unix/futex.rs
@@ -12,7 +12,7 @@ use crate::sync::atomic::AtomicI32;
 use crate::time::Duration;
 
 #[cfg(any(target_os = "linux", target_os = "android"))]
-pub fn futex_wait(futex: &AtomicI32, expected: i32, timeout: Option<Duration>) {
+pub fn futex_wait(futex: &AtomicI32, expected: i32, timeout: Option<Duration>) -> bool {
     let timespec = timeout.and_then(|d| {
         Some(libc::timespec {
             // Sleep forever if the timeout is longer than fits in a timespec.
@@ -21,15 +21,16 @@ pub fn futex_wait(futex: &AtomicI32, expected: i32, timeout: Option<Duration>) {
             tv_nsec: d.subsec_nanos() as _,
         })
     });
-    unsafe {
+    let r = unsafe {
         libc::syscall(
             libc::SYS_futex,
             futex as *const AtomicI32,
             libc::FUTEX_WAIT | libc::FUTEX_PRIVATE_FLAG,
             expected,
             timespec.as_ref().map_or(null(), |d| d as *const libc::timespec),
-        );
-    }
+        )
+    };
+    !(r < 0 && super::os::errno() == libc::ETIMEDOUT)
 }
 
 #[cfg(target_os = "emscripten")]

From 73d63488e4a23de46a6312525ccb7b4d04a7c55f Mon Sep 17 00:00:00 2001
From: Mara Bos <m-ou.se@m-ou.se>
Date: Thu, 17 Mar 2022 12:27:30 +0100
Subject: [PATCH 02/14] Add futex_wake_all.

---
 library/std/src/sys/unix/futex.rs | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/library/std/src/sys/unix/futex.rs b/library/std/src/sys/unix/futex.rs
index 22ba1a57fef2d..cc6da97ec7306 100644
--- a/library/std/src/sys/unix/futex.rs
+++ b/library/std/src/sys/unix/futex.rs
@@ -66,6 +66,18 @@ pub fn futex_wake(futex: &AtomicI32) {
     }
 }
 
+#[cfg(any(target_os = "linux", target_os = "android"))]
+pub fn futex_wake_all(futex: &AtomicI32) {
+    unsafe {
+        libc::syscall(
+            libc::SYS_futex,
+            futex as *const AtomicI32,
+            libc::FUTEX_WAKE | libc::FUTEX_PRIVATE_FLAG,
+            i32::MAX,
+        );
+    }
+}
+
 #[cfg(target_os = "emscripten")]
 pub fn futex_wake(futex: &AtomicI32) {
     extern "C" {

From 7f26adeac19f78f0277637d1e798439d8f89b853 Mon Sep 17 00:00:00 2001
From: Mara Bos <m-ou.se@m-ou.se>
Date: Thu, 17 Mar 2022 12:29:07 +0100
Subject: [PATCH 03/14] Replace Linux Mutex and Condvar with futex based ones.

---
 library/std/src/sys/unix/locks/futex.rs | 125 ++++++++++++++++++++++++
 library/std/src/sys/unix/locks/mod.rs   |  32 ++++--
 2 files changed, 149 insertions(+), 8 deletions(-)
 create mode 100644 library/std/src/sys/unix/locks/futex.rs

diff --git a/library/std/src/sys/unix/locks/futex.rs b/library/std/src/sys/unix/locks/futex.rs
new file mode 100644
index 0000000000000..48d800341a378
--- /dev/null
+++ b/library/std/src/sys/unix/locks/futex.rs
@@ -0,0 +1,125 @@
+use crate::sync::atomic::{
+    AtomicI32,
+    Ordering::{Acquire, Relaxed, Release},
+};
+use crate::sys::futex::{futex_wait, futex_wake, futex_wake_all};
+use crate::time::Duration;
+
+pub type MovableMutex = Mutex;
+pub type MovableCondvar = Condvar;
+
+pub struct Mutex {
+    /// 0: unlocked
+    /// 1: locked, no other threads waiting
+    /// 2: locked, and other threads waiting (contended)
+    futex: AtomicI32,
+}
+
+impl Mutex {
+    pub const fn new() -> Self {
+        Self { futex: AtomicI32::new(0) }
+    }
+
+    #[inline]
+    pub unsafe fn init(&mut self) {}
+
+    #[inline]
+    pub unsafe fn destroy(&self) {}
+
+    #[inline]
+    pub unsafe fn try_lock(&self) -> bool {
+        self.futex.compare_exchange(0, 1, Acquire, Relaxed).is_ok()
+    }
+
+    #[inline]
+    pub unsafe fn lock(&self) {
+        if self.futex.compare_exchange(0, 1, Acquire, Relaxed).is_err() {
+            self.lock_contended();
+        }
+    }
+
+    fn lock_contended(&self) {
+        loop {
+            // Put the lock in contended state, if it wasn't already.
+            if self.futex.swap(2, Acquire) == 0 {
+                // It was unlocked, so we just locked it.
+                return;
+            }
+            // Wait for the futex to change state.
+            futex_wait(&self.futex, 2, None);
+        }
+    }
+
+    #[inline]
+    pub unsafe fn unlock(&self) {
+        if self.futex.swap(0, Release) == 2 {
+            // We only wake up one thread. When that thread locks the mutex, it
+            // will mark the mutex as contended (2) (see lock_contended above),
+            // which makes sure that any other waiting threads will also be
+            // woken up eventually.
+            futex_wake(&self.futex);
+        }
+    }
+}
+
+pub struct Condvar {
+    // The value of this atomic is simply incremented on every notification.
+    // This is used by `.wait()` to not miss any notifications after
+    // unlocking the mutex and before waiting for notifications.
+    futex: AtomicI32,
+}
+
+impl Condvar {
+    #[inline]
+    pub const fn new() -> Self {
+        Self { futex: AtomicI32::new(0) }
+    }
+
+    #[inline]
+    pub unsafe fn init(&mut self) {}
+
+    #[inline]
+    pub unsafe fn destroy(&self) {}
+
+    // All the memory orderings here are `Relaxed`,
+    // because synchronization is done by unlocking and locking the mutex.
+
+    #[inline]
+    pub unsafe fn notify_one(&self) {
+        self.futex.fetch_add(1, Relaxed);
+        futex_wake(&self.futex);
+    }
+
+    #[inline]
+    pub unsafe fn notify_all(&self) {
+        self.futex.fetch_add(1, Relaxed);
+        futex_wake_all(&self.futex);
+    }
+
+    #[inline]
+    pub unsafe fn wait(&self, mutex: &Mutex) {
+        self.wait_optional_timeout(mutex, None);
+    }
+
+    #[inline]
+    pub unsafe fn wait_timeout(&self, mutex: &Mutex, timeout: Duration) -> bool {
+        self.wait_optional_timeout(mutex, Some(timeout))
+    }
+
+    unsafe fn wait_optional_timeout(&self, mutex: &Mutex, timeout: Option<Duration>) -> bool {
+        // Check the notification counter before we unlock the mutex.
+        let futex_value = self.futex.load(Relaxed);
+
+        // Unlock the mutex before going to sleep.
+        mutex.unlock();
+
+        // Wait, but only if there hasn't been any
+        // notification since we unlocked the mutex.
+        let r = futex_wait(&self.futex, futex_value, timeout);
+
+        // Lock the mutex again.
+        mutex.lock();
+
+        r
+    }
+}
diff --git a/library/std/src/sys/unix/locks/mod.rs b/library/std/src/sys/unix/locks/mod.rs
index f07a9f93b79a5..30e9f407eec4c 100644
--- a/library/std/src/sys/unix/locks/mod.rs
+++ b/library/std/src/sys/unix/locks/mod.rs
@@ -1,8 +1,24 @@
-mod pthread_condvar;
-mod pthread_mutex;
-mod pthread_remutex;
-mod pthread_rwlock;
-pub use pthread_condvar::{Condvar, MovableCondvar};
-pub use pthread_mutex::{MovableMutex, Mutex};
-pub use pthread_remutex::ReentrantMutex;
-pub use pthread_rwlock::{MovableRWLock, RWLock};
+cfg_if::cfg_if! {
+    if #[cfg(any(
+        target_os = "linux",
+        target_os = "android",
+    ))] {
+        mod futex;
+        #[allow(dead_code)]
+        mod pthread_mutex; // Only used for PthreadMutexAttr, needed by pthread_remutex.
+        mod pthread_remutex; // FIXME: Implement this using a futex
+        mod pthread_rwlock; // FIXME: Implement this using a futex
+        pub use futex::{Mutex, MovableMutex, Condvar, MovableCondvar};
+        pub use pthread_remutex::ReentrantMutex;
+        pub use pthread_rwlock::{RWLock, MovableRWLock};
+    } else {
+        mod pthread_mutex;
+        mod pthread_remutex;
+        mod pthread_rwlock;
+        mod pthread_condvar;
+        pub use pthread_mutex::{Mutex, MovableMutex};
+        pub use pthread_remutex::ReentrantMutex;
+        pub use pthread_rwlock::{RWLock, MovableRWLock};
+        pub use pthread_condvar::{Condvar, MovableCondvar};
+    }
+}

From 10b6f3350820f05ab7aed5c9e2cf30bc34663668 Mon Sep 17 00:00:00 2001
From: Mara Bos <m-ou.se@m-ou.se>
Date: Thu, 17 Mar 2022 12:29:30 +0100
Subject: [PATCH 04/14] Update tests.

---
 library/std/src/sync/condvar/tests.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/library/std/src/sync/condvar/tests.rs b/library/std/src/sync/condvar/tests.rs
index 6757707cd9513..f7a00676daaed 100644
--- a/library/std/src/sync/condvar/tests.rs
+++ b/library/std/src/sync/condvar/tests.rs
@@ -191,7 +191,7 @@ fn wait_timeout_wake() {
 
 #[test]
 #[should_panic]
-#[cfg_attr(not(unix), ignore)]
+#[cfg(all(unix, not(target_os = "linux"), not(target_os = "android")))]
 fn two_mutexes() {
     let m = Arc::new(Mutex::new(()));
     let m2 = m.clone();

From da4ef044c1d0e8e58f2ab18459208469110c04be Mon Sep 17 00:00:00 2001
From: Mara Bos <m-ou.se@m-ou.se>
Date: Mon, 21 Mar 2022 14:53:00 +0100
Subject: [PATCH 05/14] Spin before blocking in Mutex::lock.

---
 library/std/src/sys/unix/locks/futex.rs | 42 ++++++++++++++++++++++---
 1 file changed, 38 insertions(+), 4 deletions(-)

diff --git a/library/std/src/sys/unix/locks/futex.rs b/library/std/src/sys/unix/locks/futex.rs
index 48d800341a378..ba37d58cc31b7 100644
--- a/library/std/src/sys/unix/locks/futex.rs
+++ b/library/std/src/sys/unix/locks/futex.rs
@@ -39,14 +39,48 @@ impl Mutex {
     }
 
     fn lock_contended(&self) {
+        // Spin first to speed things up if the lock is released quickly.
+        let mut state = self.spin();
+
+        // If it's unlocked now, attempt to take the lock
+        // without marking it as contended.
+        if state == 0 {
+            match self.futex.compare_exchange(0, 1, Acquire, Relaxed) {
+                Ok(_) => return, // Locked!
+                Err(s) => state = s,
+            }
+        }
+
         loop {
-            // Put the lock in contended state, if it wasn't already.
-            if self.futex.swap(2, Acquire) == 0 {
-                // It was unlocked, so we just locked it.
+            // Put the lock in contended state.
+            // We avoid an unnecessary write if it as already set to 2,
+            // to be friendlier for the caches.
+            if state != 2 && self.futex.swap(2, Acquire) == 0 {
+                // We changed it from 0 to 2, so we just succesfully locked it.
                 return;
             }
-            // Wait for the futex to change state.
+
+            // Wait for the futex to change state, assuming it is still 2.
             futex_wait(&self.futex, 2, None);
+
+            // Spin again after waking up.
+            state = self.spin();
+        }
+    }
+
+    fn spin(&self) -> i32 {
+        let mut spin = 100;
+        loop {
+            // We only use `load` (and not `swap` or `compare_exchange`)
+            // while spinning, to be easier on the caches.
+            let state = self.futex.load(Relaxed);
+
+            if state == 0 || spin == 0 {
+                return state;
+            }
+
+            crate::hint::spin_loop();
+            spin -= 1;
         }
     }
 

From 87299298d925af0943c94d2cc5bb8a2711d9f6b4 Mon Sep 17 00:00:00 2001
From: Mara Bos <m-ou.se@m-ou.se>
Date: Thu, 24 Mar 2022 09:51:48 +0100
Subject: [PATCH 06/14] Use FUTEX_WAIT_BITSET rather than FUTEX_WAIT on Linux.

---
 library/std/src/sys/unix/futex.rs | 56 +++++++++++++++++++------------
 library/std/src/sys/unix/time.rs  |  4 +++
 2 files changed, 38 insertions(+), 22 deletions(-)

diff --git a/library/std/src/sys/unix/futex.rs b/library/std/src/sys/unix/futex.rs
index cc6da97ec7306..adb661d877bb3 100644
--- a/library/std/src/sys/unix/futex.rs
+++ b/library/std/src/sys/unix/futex.rs
@@ -4,33 +4,45 @@
     all(target_os = "emscripten", target_feature = "atomics")
 ))]
 
-#[cfg(any(target_os = "linux", target_os = "android"))]
-use crate::convert::TryInto;
-#[cfg(any(target_os = "linux", target_os = "android"))]
-use crate::ptr::null;
 use crate::sync::atomic::AtomicI32;
 use crate::time::Duration;
 
 #[cfg(any(target_os = "linux", target_os = "android"))]
 pub fn futex_wait(futex: &AtomicI32, expected: i32, timeout: Option<Duration>) -> bool {
-    let timespec = timeout.and_then(|d| {
-        Some(libc::timespec {
-            // Sleep forever if the timeout is longer than fits in a timespec.
-            tv_sec: d.as_secs().try_into().ok()?,
-            // This conversion never truncates, as subsec_nanos is always <1e9.
-            tv_nsec: d.subsec_nanos() as _,
-        })
-    });
-    let r = unsafe {
-        libc::syscall(
-            libc::SYS_futex,
-            futex as *const AtomicI32,
-            libc::FUTEX_WAIT | libc::FUTEX_PRIVATE_FLAG,
-            expected,
-            timespec.as_ref().map_or(null(), |d| d as *const libc::timespec),
-        )
-    };
-    !(r < 0 && super::os::errno() == libc::ETIMEDOUT)
+    use super::time::Instant;
+    use crate::ptr::null;
+    use crate::sync::atomic::Ordering::Relaxed;
+
+    // Calculate the timeout as an absolute timespec.
+    let timespec =
+        timeout.and_then(|d| Some(Instant::now().checked_add_duration(&d)?.as_timespec()));
+
+    loop {
+        // No need to wait if the value already changed.
+        if futex.load(Relaxed) != expected {
+            return true;
+        }
+
+        // Use FUTEX_WAIT_BITSET rather than FUTEX_WAIT to be able to give an
+        // absolute time rather than a relative time.
+        let r = unsafe {
+            libc::syscall(
+                libc::SYS_futex,
+                futex as *const AtomicI32,
+                libc::FUTEX_WAIT_BITSET | libc::FUTEX_PRIVATE_FLAG,
+                expected,
+                timespec.as_ref().map_or(null(), |d| d as *const libc::timespec),
+                null::<u32>(), // This argument is unused for FUTEX_WAIT_BITSET.
+                !0u32,         // A full bitmask, to make it behave like a regular FUTEX_WAIT.
+            )
+        };
+
+        match (r < 0).then(super::os::errno) {
+            Some(libc::ETIMEDOUT) => return false,
+            Some(libc::EINTR) => continue,
+            _ => return true,
+        }
+    }
 }
 
 #[cfg(target_os = "emscripten")]
diff --git a/library/std/src/sys/unix/time.rs b/library/std/src/sys/unix/time.rs
index 59ddd1aa92f81..64c249f33ebc5 100644
--- a/library/std/src/sys/unix/time.rs
+++ b/library/std/src/sys/unix/time.rs
@@ -299,6 +299,10 @@ mod inner {
         pub fn checked_sub_duration(&self, other: &Duration) -> Option<Instant> {
             Some(Instant { t: self.t.checked_sub_duration(other)? })
         }
+
+        pub(in crate::sys::unix) fn as_timespec(&self) -> libc::timespec {
+            self.t.t
+        }
     }
 
     impl fmt::Debug for Instant {

From 23badeb4cb22afd09e0264348800e44cda80af38 Mon Sep 17 00:00:00 2001
From: Mara Bos <m-ou.se@m-ou.se>
Date: Thu, 24 Mar 2022 11:11:03 +0100
Subject: [PATCH 07/14] Make Timespec available in sys::unix.

---
 library/std/src/sys/unix/time.rs | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/library/std/src/sys/unix/time.rs b/library/std/src/sys/unix/time.rs
index 64c249f33ebc5..8741e568c672a 100644
--- a/library/std/src/sys/unix/time.rs
+++ b/library/std/src/sys/unix/time.rs
@@ -9,8 +9,8 @@ use crate::convert::TryInto;
 const NSEC_PER_SEC: u64 = 1_000_000_000;
 
 #[derive(Copy, Clone)]
-struct Timespec {
-    t: libc::timespec,
+pub(in crate::sys::unix) struct Timespec {
+    pub t: libc::timespec,
 }
 
 impl Timespec {
@@ -18,7 +18,7 @@ impl Timespec {
         Timespec { t: libc::timespec { tv_sec: 0, tv_nsec: 0 } }
     }
 
-    fn sub_timespec(&self, other: &Timespec) -> Result<Duration, Duration> {
+    pub fn sub_timespec(&self, other: &Timespec) -> Result<Duration, Duration> {
         if self >= other {
             // NOTE(eddyb) two aspects of this `if`-`else` are required for LLVM
             // to optimize it into a branchless form (see also #75545):
@@ -51,7 +51,7 @@ impl Timespec {
         }
     }
 
-    fn checked_add_duration(&self, other: &Duration) -> Option<Timespec> {
+    pub fn checked_add_duration(&self, other: &Duration) -> Option<Timespec> {
         let mut secs = other
             .as_secs()
             .try_into() // <- target type would be `libc::time_t`
@@ -68,7 +68,7 @@ impl Timespec {
         Some(Timespec { t: libc::timespec { tv_sec: secs, tv_nsec: nsec as _ } })
     }
 
-    fn checked_sub_duration(&self, other: &Duration) -> Option<Timespec> {
+    pub fn checked_sub_duration(&self, other: &Duration) -> Option<Timespec> {
         let mut secs = other
             .as_secs()
             .try_into() // <- target type would be `libc::time_t`
@@ -285,7 +285,7 @@ mod inner {
 
     impl Instant {
         pub fn now() -> Instant {
-            Instant { t: now(libc::CLOCK_MONOTONIC) }
+            Instant { t: Timespec::now(libc::CLOCK_MONOTONIC) }
         }
 
         pub fn checked_sub_instant(&self, other: &Instant) -> Option<Duration> {
@@ -299,10 +299,6 @@ mod inner {
         pub fn checked_sub_duration(&self, other: &Duration) -> Option<Instant> {
             Some(Instant { t: self.t.checked_sub_duration(other)? })
         }
-
-        pub(in crate::sys::unix) fn as_timespec(&self) -> libc::timespec {
-            self.t.t
-        }
     }
 
     impl fmt::Debug for Instant {
@@ -316,7 +312,7 @@ mod inner {
 
     impl SystemTime {
         pub fn now() -> SystemTime {
-            SystemTime { t: now(libc::CLOCK_REALTIME) }
+            SystemTime { t: Timespec::now(libc::CLOCK_REALTIME) }
         }
 
         pub fn sub_time(&self, other: &SystemTime) -> Result<Duration, Duration> {
@@ -352,9 +348,11 @@ mod inner {
     #[cfg(any(target_os = "dragonfly", target_os = "espidf"))]
     pub type clock_t = libc::c_ulong;
 
-    fn now(clock: clock_t) -> Timespec {
-        let mut t = Timespec { t: libc::timespec { tv_sec: 0, tv_nsec: 0 } };
-        cvt(unsafe { libc::clock_gettime(clock, &mut t.t) }).unwrap();
-        t
+    impl Timespec {
+        pub fn now(clock: clock_t) -> Timespec {
+            let mut t = Timespec { t: libc::timespec { tv_sec: 0, tv_nsec: 0 } };
+            cvt(unsafe { libc::clock_gettime(clock, &mut t.t) }).unwrap();
+            t
+        }
     }
 }

From c9ae3fe68fc4477350476acb649b817a80bfd973 Mon Sep 17 00:00:00 2001
From: Mara Bos <m-ou.se@m-ou.se>
Date: Thu, 24 Mar 2022 11:11:31 +0100
Subject: [PATCH 08/14] Explicitly use CLOCK_MONOTONIC in futex_wait.

Instant might be changed to use CLOCK_BOOTTIME at some point.
---
 library/std/src/sys/unix/futex.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/library/std/src/sys/unix/futex.rs b/library/std/src/sys/unix/futex.rs
index adb661d877bb3..b82a4a734402b 100644
--- a/library/std/src/sys/unix/futex.rs
+++ b/library/std/src/sys/unix/futex.rs
@@ -9,13 +9,13 @@ use crate::time::Duration;
 
 #[cfg(any(target_os = "linux", target_os = "android"))]
 pub fn futex_wait(futex: &AtomicI32, expected: i32, timeout: Option<Duration>) -> bool {
-    use super::time::Instant;
+    use super::time::Timespec;
     use crate::ptr::null;
     use crate::sync::atomic::Ordering::Relaxed;
 
     // Calculate the timeout as an absolute timespec.
     let timespec =
-        timeout.and_then(|d| Some(Instant::now().checked_add_duration(&d)?.as_timespec()));
+        timeout.and_then(|d| Some(Timespec::now(libc::CLOCK_MONOTONIC).checked_add_duration(&d)?));
 
     loop {
         // No need to wait if the value already changed.
@@ -31,7 +31,7 @@ pub fn futex_wait(futex: &AtomicI32, expected: i32, timeout: Option<Duration>) -
                 futex as *const AtomicI32,
                 libc::FUTEX_WAIT_BITSET | libc::FUTEX_PRIVATE_FLAG,
                 expected,
-                timespec.as_ref().map_or(null(), |d| d as *const libc::timespec),
+                timespec.as_ref().map_or(null(), |t| &t.t as *const libc::timespec),
                 null::<u32>(), // This argument is unused for FUTEX_WAIT_BITSET.
                 !0u32,         // A full bitmask, to make it behave like a regular FUTEX_WAIT.
             )

From c49887da277334226d17f7633392425e808b46d7 Mon Sep 17 00:00:00 2001
From: Mara Bos <m-ou.se@m-ou.se>
Date: Fri, 1 Apr 2022 11:10:58 +0200
Subject: [PATCH 09/14] Add comment about futex_wait timeout.

---
 library/std/src/sys/unix/futex.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/library/std/src/sys/unix/futex.rs b/library/std/src/sys/unix/futex.rs
index b82a4a734402b..c61d948fb601d 100644
--- a/library/std/src/sys/unix/futex.rs
+++ b/library/std/src/sys/unix/futex.rs
@@ -14,6 +14,8 @@ pub fn futex_wait(futex: &AtomicI32, expected: i32, timeout: Option<Duration>) -
     use crate::sync::atomic::Ordering::Relaxed;
 
     // Calculate the timeout as an absolute timespec.
+    //
+    // Overflows are rounded up to an infinite timeout (None).
     let timespec =
         timeout.and_then(|d| Some(Timespec::now(libc::CLOCK_MONOTONIC).checked_add_duration(&d)?));
 

From 6392f1555e6515c59faa8393d1bf106b0c8872bd Mon Sep 17 00:00:00 2001
From: Mara Bos <m-ou.se@m-ou.se>
Date: Fri, 1 Apr 2022 11:11:28 +0200
Subject: [PATCH 10/14] Shuffle around #[inline] and #[cold] in mutex impl.

---
 library/std/src/sys/unix/locks/futex.rs | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/library/std/src/sys/unix/locks/futex.rs b/library/std/src/sys/unix/locks/futex.rs
index ba37d58cc31b7..08c042f279bb2 100644
--- a/library/std/src/sys/unix/locks/futex.rs
+++ b/library/std/src/sys/unix/locks/futex.rs
@@ -38,6 +38,7 @@ impl Mutex {
         }
     }
 
+    #[cold]
     fn lock_contended(&self) {
         // Spin first to speed things up if the lock is released quickly.
         let mut state = self.spin();
@@ -91,9 +92,14 @@ impl Mutex {
             // will mark the mutex as contended (2) (see lock_contended above),
             // which makes sure that any other waiting threads will also be
             // woken up eventually.
-            futex_wake(&self.futex);
+            self.wake();
         }
     }
+
+    #[cold]
+    fn wake(&self) {
+        futex_wake(&self.futex);
+    }
 }
 
 pub struct Condvar {
@@ -118,24 +124,20 @@ impl Condvar {
     // All the memory orderings here are `Relaxed`,
     // because synchronization is done by unlocking and locking the mutex.
 
-    #[inline]
     pub unsafe fn notify_one(&self) {
         self.futex.fetch_add(1, Relaxed);
         futex_wake(&self.futex);
     }
 
-    #[inline]
     pub unsafe fn notify_all(&self) {
         self.futex.fetch_add(1, Relaxed);
         futex_wake_all(&self.futex);
     }
 
-    #[inline]
     pub unsafe fn wait(&self, mutex: &Mutex) {
         self.wait_optional_timeout(mutex, None);
     }
 
-    #[inline]
     pub unsafe fn wait_timeout(&self, mutex: &Mutex, timeout: Duration) -> bool {
         self.wait_optional_timeout(mutex, Some(timeout))
     }

From 321690c8278a5cf038adc50d1642c8e158de38a7 Mon Sep 17 00:00:00 2001
From: Mara Bos <m-ou.se@m-ou.se>
Date: Fri, 1 Apr 2022 11:11:46 +0200
Subject: [PATCH 11/14] Don't spin on contended mutexes.

---
 library/std/src/sys/unix/locks/futex.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/library/std/src/sys/unix/locks/futex.rs b/library/std/src/sys/unix/locks/futex.rs
index 08c042f279bb2..d9b5bfe3ac62c 100644
--- a/library/std/src/sys/unix/locks/futex.rs
+++ b/library/std/src/sys/unix/locks/futex.rs
@@ -76,7 +76,9 @@ impl Mutex {
             // while spinning, to be easier on the caches.
             let state = self.futex.load(Relaxed);
 
-            if state == 0 || spin == 0 {
+            // We stop spinning when the mutex is unlocked (0),
+            // but also when it's contended (2).
+            if state != 1 || spin == 0 {
                 return state;
             }
 

From 4b1b305ccb7a981d6b6246b8dbcfdb36b83a00c6 Mon Sep 17 00:00:00 2001
From: Mara Bos <m-ou.se@m-ou.se>
Date: Fri, 1 Apr 2022 11:11:58 +0200
Subject: [PATCH 12/14] Use MaybeUninit for clock_gettime's timespec.

---
 library/std/src/sys/unix/time.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/library/std/src/sys/unix/time.rs b/library/std/src/sys/unix/time.rs
index 8741e568c672a..498c94d0cdcba 100644
--- a/library/std/src/sys/unix/time.rs
+++ b/library/std/src/sys/unix/time.rs
@@ -266,6 +266,7 @@ mod inner {
 #[cfg(not(any(target_os = "macos", target_os = "ios")))]
 mod inner {
     use crate::fmt;
+    use crate::mem::MaybeUninit;
     use crate::sys::cvt;
     use crate::time::Duration;
 
@@ -350,9 +351,9 @@ mod inner {
 
     impl Timespec {
         pub fn now(clock: clock_t) -> Timespec {
-            let mut t = Timespec { t: libc::timespec { tv_sec: 0, tv_nsec: 0 } };
-            cvt(unsafe { libc::clock_gettime(clock, &mut t.t) }).unwrap();
-            t
+            let mut t = MaybeUninit::uninit();
+            cvt(unsafe { libc::clock_gettime(clock, t.as_mut_ptr()) }).unwrap();
+            Timespec { t: unsafe { t.assume_init() } }
         }
     }
 }

From 104e95f8481b8edf17e399bddf5b210d40a7775a Mon Sep 17 00:00:00 2001
From: Mara Bos <m-ou.se@m-ou.se>
Date: Tue, 5 Apr 2022 13:58:10 +0200
Subject: [PATCH 13/14] Mark unix::locks::futex::Mutex::new as #[inline].

---
 library/std/src/sys/unix/locks/futex.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/library/std/src/sys/unix/locks/futex.rs b/library/std/src/sys/unix/locks/futex.rs
index d9b5bfe3ac62c..d45880730728f 100644
--- a/library/std/src/sys/unix/locks/futex.rs
+++ b/library/std/src/sys/unix/locks/futex.rs
@@ -16,6 +16,7 @@ pub struct Mutex {
 }
 
 impl Mutex {
+    #[inline]
     pub const fn new() -> Self {
         Self { futex: AtomicI32::new(0) }
     }

From 650315ee8801e650c049312a899de4202f54a701 Mon Sep 17 00:00:00 2001
From: Mara Bos <m-ou.se@m-ou.se>
Date: Tue, 5 Apr 2022 17:08:12 +0200
Subject: [PATCH 14/14] Reword comment in futex condvar implementation.

---
 library/std/src/sys/unix/locks/futex.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/library/std/src/sys/unix/locks/futex.rs b/library/std/src/sys/unix/locks/futex.rs
index d45880730728f..630351d0dc278 100644
--- a/library/std/src/sys/unix/locks/futex.rs
+++ b/library/std/src/sys/unix/locks/futex.rs
@@ -146,7 +146,7 @@ impl Condvar {
     }
 
     unsafe fn wait_optional_timeout(&self, mutex: &Mutex, timeout: Option<Duration>) -> bool {
-        // Check the notification counter before we unlock the mutex.
+        // Examine the notification counter _before_ we unlock the mutex.
         let futex_value = self.futex.load(Relaxed);
 
         // Unlock the mutex before going to sleep.