diff --git a/tokio/src/sync/notify.rs b/tokio/src/sync/notify.rs
index efe16f9f8ef..7f39bdc39f5 100644
--- a/tokio/src/sync/notify.rs
+++ b/tokio/src/sync/notify.rs
@@ -829,10 +829,17 @@ impl Notified<'_> {
                         }
                     }
 
+                    let mut old_waker = None;
                     if waker.is_some() {
                         // Safety: called while locked.
+                        //
+                        // This code makes the following assignment, but it
+                        // stores the old waker in `old_waker` so that we can
+                        // drop it after releasing the mutex.
+                        //
+                        // (*waiter.get()).waker = waker;
                         unsafe {
-                            (*waiter.get()).waker = waker;
+                            old_waker = std::mem::replace(&mut (*waiter.get()).waker, waker);
                         }
                     }
 
@@ -843,6 +850,9 @@ impl Notified<'_> {
 
                     *state = Waiting;
 
+                    drop(waiters);
+                    drop(old_waker);
+
                     return Poll::Pending;
                 }
                 Waiting => {
@@ -855,11 +865,12 @@ impl Notified<'_> {
 
                     // Safety: called while locked
                     let w = unsafe { &mut *waiter.get() };
+                    let mut old_waker = None;
 
                     if w.notified.is_some() {
                         // Our waker has been notified. Reset the fields and
                         // remove it from the list.
-                        w.waker = None;
+                        old_waker = std::mem::take(&mut w.waker);
                         w.notified = None;
 
                         *state = Done;
@@ -871,10 +882,14 @@ impl Notified<'_> {
                                 None => true,
                             };
                             if should_update {
-                                w.waker = Some(waker.clone());
+                                old_waker = std::mem::replace(&mut w.waker, Some(waker.clone()));
                             }
                         }
 
+                        // Drop the old waker after releasing the lock.
+                        drop(waiters);
+                        drop(old_waker);
+
                         return Poll::Pending;
                     }
 
@@ -884,6 +899,9 @@ impl Notified<'_> {
                     // is helpful to visualize the scope of the critical
                     // section.
                     drop(waiters);
+
+                    // Drop the old waker after releasing the lock.
+                    drop(old_waker);
                 }
                 Done => {
                     return Poll::Ready(());