tox hangs but no subprocess running anymore

[jsteinhofff]

Issue

Some times (not reproducable every time but i run into the issue quite often) tox does not terminate.
I run it with

python3 -m tox run-parallel --conf "${config_dir}"/tox.ini --parallel auto --parallel-no-spinner --quiet

This command will then never terminate, but with htop it seems none of the external tools is running anymore:

When i add --exit-and-dump-after 10 it does terminate and will produce the below output.

Environment

Provide at least:

• OS: Ubuntu 20.04
• Python: 3.8.10

Output when terminated using exit-and-dump-after...

Timeout (0:00:10)!
Thread 0x00007f87e2ffd700 (most recent call first):
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/execute/local_sub_process/read_via_thread_unix.py", line 34 in _read_available
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/execute/local_sub_process/read_via_thread_unix.py", line 23 in _read_stream
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f87f9ffb700 (most recent call first):
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/execute/local_sub_process/read_via_thread_unix.py", line 34 in _read_available
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/execute/local_sub_process/read_via_thread_unix.py", line 23 in _read_stream
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f887dffb700 (most recent call first):
  File "/usr/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f887e7fc700 (most recent call first):
  File "/usr/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f887effd700 (most recent call first):
  File "/usr/lib/python3.8/subprocess.py", line 1764 in _try_wait
  File "/usr/lib/python3.8/subprocess.py", line 1806 in _wait
  File "/usr/lib/python3.8/subprocess.py", line 1083 in wait
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/execute/local_sub_process/__init__.py", line 100 in wait
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/tox_env/api.py", line 388 in execute
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/session/cmd/run/single.py", line 106 in run_command_set
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/session/cmd/run/single.py", line 85 in run_commands
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/session/cmd/run/single.py", line 48 in _evaluate
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/session/cmd/run/single.py", line 36 in run_one
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/session/cmd/run/common.py", line 322 in _run
  File "/usr/lib/python3.8/concurrent/futures/thread.py", line 57 in run
  File "/usr/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f887f7fe700 (most recent call first):
  File "/usr/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f887ffff700 (most recent call first):
  File "/usr/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f889cf27700 (most recent call first):
  File "/usr/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f889d728700 (most recent call first):
  File "/usr/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f889df69700 (most recent call first):
  File "/usr/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f889e76a700 (most recent call first):
  File "/usr/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f889ef6b700 (most recent call first):
  File "/usr/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f889f76c700 (most recent call first):
  File "/usr/lib/python3.8/threading.py", line 306 in wait
  File "/usr/lib/python3.8/threading.py", line 558 in wait
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/util/spinner.py", line 87 in render
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f889ff6d700 (most recent call first):
  File "/usr/lib/python3.8/threading.py", line 302 in wait
  File "/usr/lib/python3.8/threading.py", line 558 in wait
  File "/usr/lib/python3.8/concurrent/futures/_base.py", line 244 in as_completed
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/session/cmd/run/common.py", line 346 in _queue_and_wait
  File "/usr/lib/python3.8/threading.py", line 870 in run
  File "/usr/lib/python3.8/threading.py", line 932 in _bootstrap_inner
  File "/usr/lib/python3.8/threading.py", line 890 in _bootstrap

Thread 0x00007f88a1e38740 (most recent call first):
  File "/usr/lib/python3.8/threading.py", line 1027 in _wait_for_tstate_lock
  File "/usr/lib/python3.8/threading.py", line 1011 in join
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/session/cmd/run/common.py", line 251 in execute
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/session/cmd/run/parallel.py", line 85 in run_parallel
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/run.py", line 46 in main
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/run.py", line 20 in run
  File "/myhome.../.venv/lib/python3.8/site-packages/tox/__main__.py", line 6 in <module>
  File "/usr/lib/python3.8/runpy.py", line 87 in _run_code
  File "/usr/lib/python3.8/runpy.py", line 194 in _run_module_as_main

Output of running tox

Tox output is simply incomplete:

(special characters at the end accidentally added)

Minimal example

Not sure how to reproduce currently.

[glyph]

I suspect this is a duplicate of #2844 >?

[ziima]

I have troubles with the same error:

$ _TOX_SHOW_THREAD=1 timeout 900 tox --exit-and-dump-after 600 -vvv
ROOT: 104 D setup logging to NOTSET on pid 163 [tox/report.py:221]
Timeout (0:10:00)!
Thread 0x00007f154bfff6c0 (most recent call first):
  File "/usr/local/lib/python3.9/concurrent/futures/thread.py", line 81 in _worker
  File "/usr/local/lib/python3.9/threading.py", line 917 in run
  File "/usr/local/lib/python3.9/threading.py", line 980 in _bootstrap_inner
  File "/usr/local/lib/python3.9/threading.py", line 937 in _bootstrap
Thread 0x00007f156ffff6c0 (most recent call first):
  File "/usr/local/lib/python3.9/site-packages/virtualenv/util/lock.py", line 34 in release
  File "/usr/local/lib/python3.9/site-packages/virtualenv/util/lock.py", line 120 in _lock_file
  File "/usr/local/lib/python3.9/site-packages/virtualenv/util/lock.py", line 132 in lock_for_key
  File "/usr/local/lib/python3.9/contextlib.py", line 119 in __enter__
  File "/usr/local/lib/python3.9/site-packages/virtualenv/app_data/via_disk_folder.py", line 144 in locked
  File "/usr/local/lib/python3.9/contextlib.py", line 119 in __enter__
  File "/usr/local/lib/python3.9/site-packages/virtualenv/discovery/cached_py_info.py", line 63 in _get_via_file_cache
  File "/usr/local/lib/python3.9/site-packages/virtualenv/discovery/cached_py_info.py", line 46 in _get_from_cache
  File "/usr/local/lib/python3.9/site-packages/virtualenv/discovery/cached_py_info.py", line 30 in from_exe
  File "/usr/local/lib/python3.9/site-packages/virtualenv/discovery/py_info.py", line 381 in from_exe
  File "/usr/local/lib/python3.9/site-packages/virtualenv/discovery/builtin.py", line 155 in propose_interpreters
  File "/usr/local/lib/python3.9/site-packages/virtualenv/discovery/builtin.py", line 75 in get_interpreter
  File "/usr/local/lib/python3.9/site-packages/virtualenv/discovery/builtin.py", line 58 in run
  File "/usr/local/lib/python3.9/site-packages/virtualenv/discovery/discover.py", line 41 in interpreter
  File "/usr/local/lib/python3.9/site-packages/virtualenv/run/__init__.py", line 77 in build_parser
  File "/usr/local/lib/python3.9/site-packages/virtualenv/run/__init__.py", line 49 in session_via_cli
  File "/usr/local/lib/python3.9/site-packages/tox/tox_env/python/virtual_env/api.py", line 108 in session
  File "/usr/local/lib/python3.9/site-packages/tox/tox_env/python/virtual_env/api.py", line 127 in creator
  File "/usr/local/lib/python3.9/site-packages/tox/tox_env/python/virtual_env/api.py", line 135 in _get_python
  File "/usr/local/lib/python3.9/site-packages/tox/tox_env/python/api.py", line 252 in base_python
  File "/usr/local/lib/python3.9/site-packages/tox/tox_env/python/api.py", line 242 in python_cache
  File "/usr/local/lib/python3.9/site-packages/tox/tox_env/python/virtual_env/api.py", line 78 in python_cache
  File "/usr/local/lib/python3.9/site-packages/tox/tox_env/python/api.py", line 205 in ensure_python_env
  File "/usr/local/lib/python3.9/site-packages/tox/tox_env/python/api.py", line 201 in _setup_env
  File "/usr/local/lib/python3.9/site-packages/tox/tox_env/python/runner.py", line 108 in _setup_env
  File "/usr/local/lib/python3.9/site-packages/tox/tox_env/api.py", line 248 in setup
  File "/usr/local/lib/python3.9/site-packages/tox/session/cmd/run/single.py", line 47 in _evaluate
  File "/usr/local/lib/python3.9/site-packages/tox/session/cmd/run/single.py", line 36 in run_one
  File "/usr/local/lib/python3.9/site-packages/tox/session/cmd/run/common.py", line 252 in _run
  File "/usr/local/lib/python3.9/concurrent/futures/thread.py", line 58 in run
  File "/usr/local/lib/python3.9/concurrent/futures/thread.py", line 83 in _worker
  File "/usr/local/lib/python3.9/threading.py", line 917 in run
  File "/usr/local/lib/python3.9/threading.py", line 980 in _bootstrap_inner
  File "/usr/local/lib/python3.9/threading.py", line 937 in _bootstrap
Thread 0x00007f1584dcb6c0 (most recent call first):
  File "/usr/local/lib/python3.9/concurrent/futures/thread.py", line 81 in _worker
  File "/usr/local/lib/python3.9/threading.py", line 917 in run
  File "/usr/local/lib/python3.9/threading.py", line 980 in _bootstrap_inner
  File "/usr/local/lib/python3.9/threading.py", line 937 in _bootstrap
Thread 0x00007f156e7fc6c0 (most recent call first):
  File "/usr/local/lib/python3.9/concurrent/futures/thread.py", line 81 in _worker
  File "/usr/local/lib/python3.9/threading.py", line 917 in run
  File "/usr/local/lib/python3.9/threading.py", line 980 in _bootstrap_inner
  File "/usr/local/lib/python3.9/threading.py", line 937 in _bootstrap
Thread 0x00007f156dffb6c0 (most recent call first):
  File "/usr/local/lib/python3.9/concurrent/futures/thread.py", line 81 in _worker
  File "/usr/local/lib/python3.9/threading.py", line 917 in run
  File "/usr/local/lib/python3.9/threading.py", line 980 in _bootstrap_inner
  File "/usr/local/lib/python3.9/threading.py", line 937 in _bootstrap
Thread 0x00007f15858016c0 (most recent call first):
  File "/usr/local/lib/python3.9/concurrent/futures/thread.py", line 81 in _worker
  File "/usr/local/lib/python3.9/threading.py", line 917 in run
  File "/usr/local/lib/python3.9/threading.py", line 980 in _bootstrap_inner
  File "/usr/local/lib/python3.9/threading.py", line 937 in _bootstrap
Thread 0x00007f15860026c0 (most recent call first):
  File "/usr/local/lib/python3.9/concurrent/futures/thread.py", line 81 in _worker
  File "/usr/local/lib/python3.9/threading.py", line 917 in run
  File "/usr/local/lib/python3.9/threading.py", line 980 in _bootstrap_inner
  File "/usr/local/lib/python3.9/threading.py", line 937 in _bootstrap
Thread 0x00007f15868036c0 (most recent call first):
  File "/usr/local/lib/python3.9/threading.py", line 316 in wait
  File "/usr/local/lib/python3.9/threading.py", line 581 in wait
  File "/usr/local/lib/python3.9/site-packages/tox/util/spinner.py", line 87 in render
  File "/usr/local/lib/python3.9/threading.py", line 917 in run
  File "/usr/local/lib/python3.9/threading.py", line 980 in _bootstrap_inner
  File "/usr/local/lib/python3.9/threading.py", line 937 in _bootstrap
Thread 0x00007f15870046c0 (most recent call first):
  File "/usr/local/lib/python3.9/threading.py", line 312 in wait
  File "/usr/local/lib/python3.9/threading.py", line 581 in wait
  File "/usr/local/lib/python3.9/concurrent/futures/_base.py", line 245 in as_completed
  File "/usr/local/lib/python3.9/site-packages/tox/session/cmd/run/common.py", line 276 in _queue_and_wait
  File "/usr/local/lib/python3.9/threading.py", line 917 in run
  File "/usr/local/lib/python3.9/threading.py", line 980 in _bootstrap_inner
  File "/usr/local/lib/python3.9/threading.py", line 937 in _bootstrap
Thread 0x00007f1588db7b80 (most recent call first):
  File "/usr/local/lib/python3.9/threading.py", line 1080 in _wait_for_tstate_lock
  File "/usr/local/lib/python3.9/threading.py", line 1060 in join
  File "/usr/local/lib/python3.9/site-packages/tox/session/cmd/run/common.py", line 181 in execute
  File "/usr/local/lib/python3.9/site-packages/tox/session/cmd/run/parallel.py", line 85 in run_parallel
  File "/usr/local/lib/python3.9/site-packages/tox/session/cmd/legacy.py", line 114 in legacy
  File "/usr/local/lib/python3.9/site-packages/tox/run.py", line 46 in main
  File "/usr/local/lib/python3.9/site-packages/tox/run.py", line 20 in run
  File "/usr/local/bin/tox", line 8 in <module>