From 6a3e15b3b1ca7f2f4f1d74740363ff9bd1569019 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 24 Jan 2024 13:21:10 +0100 Subject: [PATCH] [3.12] gh-114440: Close writer pipe in multiprocessing.Queue, not concurrent.futures (GH-114489) This was left out of the 3.12 backport for three related issues: - gh-107219 (which adds `self.call_queue._writer.close()` to `_ExecutorManagerThread` in `concurrent.futures`) - gh-109370 (which changes this to be only called on Windows) - gh-109047 (which moves the call to `multiprocessing.Queue`'s `_terminate_broken`) Without this change, ProcessPoolExecutor sometimes hangs on Windows when a worker process is terminated. Co-authored-by: Victor Stinner Co-authored-by: Serhiy Storchaka --- Lib/concurrent/futures/process.py | 5 ----- Lib/multiprocessing/queues.py | 5 +++++ .../Library/2023-09-25-02-11-14.gh-issue-114440.b2TrqG.rst | 6 ++++++ 3 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-114440.b2TrqG.rst diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 33e62fe231e1877..0e452883963c171 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -524,11 +524,6 @@ def _terminate_broken(self, cause): self.call_queue._terminate_broken() - # gh-107219: Close the connection writer which can unblock - # Queue._feed() if it was stuck in send_bytes(). - if sys.platform == 'win32': - self.call_queue._writer.close() - # clean up resources self._join_executor_internals(broken=True) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index d36de75749f4edf..852ae87b2768612 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -164,6 +164,11 @@ def _terminate_broken(self): # gh-94777: Prevent queue writing to a pipe which is no longer read. self._reader.close() + # gh-107219: Close the connection writer which can unblock + # Queue._feed() if it was stuck in send_bytes(). + if sys.platform == 'win32': + self._writer.close() + self.close() self.join_thread() diff --git a/Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-114440.b2TrqG.rst b/Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-114440.b2TrqG.rst new file mode 100644 index 000000000000000..7231a3c4ef83258 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-114440.b2TrqG.rst @@ -0,0 +1,6 @@ +On Windows, closing the connection writer when cleaning up a broken +:class:`multiprocessing.Queue` queue is now done for all queues, rather than +only in :mod:`concurrent.futures` manager thread. +This can prevent a deadlock when a ``multiprocessing`` worker process terminates +without cleaning up. +This completes the backport of patches by Victor Stinner and Serhiy Storchaka.