backport ThreadedChildWatcher

Author: mxschmitt

meta.yaml

@@ -26,7 +26,6 @@ requirements:
     - greenlet ==1.1.2
     - pyee ==8.1.0
     - websockets ==10.1
-    - psutil ==5.9.0 # [py<38]
     - typing_extensions # [py<39]
 test:
   requires:

playwright/__main__.py

@@ -19,9 +19,9 @@
 
 
 def main() -> None:
-    driver_executable = compute_driver_executable()
+    (driver_executable, cli_entrypoint) = compute_driver_executable()
     completed_process = subprocess.run(
-        [str(driver_executable), *sys.argv[1:]], env=get_driver_env()
+        [driver_executable, cli_entrypoint, *sys.argv[1:]], env=get_driver_env()
     )
     sys.exit(completed_process.returncode)
 

playwright/_impl/_connection.py

@@ -187,6 +187,7 @@ async def init() -> None:
     def stop_sync(self) -> None:
         self._transport.request_stop()
         self._dispatcher_fiber.switch()
+        self._loop.run_until_complete(self._transport.wait_until_stopped())
         self.cleanup()
 
     async def stop_async(self) -> None:

playwright/_impl/_driver.py

@@ -17,17 +17,17 @@
 import os
 import sys
 from pathlib import Path
+from typing import Tuple
 
 import playwright
 from playwright._repo_version import version
 
 
-def compute_driver_executable() -> Path:
+def compute_driver_executable() -> Tuple[str, str]:
+    node_binary = "node.exe" if sys.platform == "win32" else "node"
     package_path = Path(inspect.getfile(playwright)).parent
-    platform = sys.platform
-    if platform == "win32":
-        return package_path / "driver" / "playwright.cmd"
-    return package_path / "driver" / "playwright.sh"
+    base_args = package_path / "driver" / "package" / "cli.js"
+    return (str(package_path / "driver" / node_binary), str(base_args))
 
 
 if sys.version_info.major == 3 and sys.version_info.minor == 7:

playwright/_impl/_py37ThreadedChildWatcher.py

@@ -19,7 +19,6 @@
 import subprocess
 import sys
 from abc import ABC, abstractmethod
-from pathlib import Path
 from typing import Callable, Dict, Optional, Union
 
 import websockets
@@ -28,7 +27,7 @@
 from websockets.client import connect as websocket_connect
 
 from playwright._impl._api_types import Error
-from playwright._impl._driver import get_driver_env
+from playwright._impl._driver import compute_driver_executable, get_driver_env
 from playwright._impl._helper import ParsedMessagePayload
 
 
@@ -96,12 +95,9 @@ def deserialize_message(self, data: Union[str, bytes]) -> ParsedMessagePayload:
 
 
 class PipeTransport(Transport):
-    def __init__(
-        self, loop: asyncio.AbstractEventLoop, driver_executable: Path
-    ) -> None:
+    def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
         super().__init__(loop)
         self._stopped = False
-        self._driver_executable = driver_executable
 
     def request_stop(self) -> None:
         assert self._output
@@ -110,19 +106,6 @@ def request_stop(self) -> None:
 
     async def wait_until_stopped(self) -> None:
         await self._stopped_future
-        # In Python 3.7, self._proc.wait() hangs because it does not use ThreadedChildWatcher
-        # which is used in Python 3.8+. Hence waiting for child process is skipped in Python 3.7.
-        # See https://bugs.python.org/issue35621
-        # See https://stackoverflow.com/questions/28915607/does-asyncio-support-running-a-subprocess-from-a-non-main-thread/28917653#28917653
-        if sys.version_info >= (3, 8):
-            await self._proc.wait()
-        else:
-            import psutil
-
-            try:
-                psutil.Process(self._proc.pid).wait()
-            except psutil.NoSuchProcess:
-                pass
 
     async def connect(self) -> None:
         self._stopped_future: asyncio.Future = asyncio.Future()
@@ -131,14 +114,27 @@ async def connect(self) -> None:
         if sys.platform == "win32" and sys.stdout is None:
             creationflags = subprocess.CREATE_NO_WINDOW
 
+        original_child_watcher = asyncio.get_child_watcher()
         try:
             # For pyinstaller
             env = get_driver_env()
             if getattr(sys, "frozen", False):
                 env.setdefault("PLAYWRIGHT_BROWSERS_PATH", "0")
 
+            # In Python 3.7, self._proc.wait() hangs because it does not use ThreadedChildWatcher
+            # which is used in Python 3.8+. Since it also cleans up zombie processes, we backported it.
+            # See https://bugs.python.org/issue35621
+            if sys.version_info[0] == 3 and sys.version_info[1] == 7:
+                from ._py37ThreadedChildWatcher import (  # type: ignore
+                    ThreadedChildWatcher,
+                )
+
+                watcher = ThreadedChildWatcher()
+                asyncio.set_child_watcher(watcher)
+            (driver_executable, cli_entrypoint) = compute_driver_executable()
             self._proc = await asyncio.create_subprocess_exec(
-                str(self._driver_executable),
+                str(driver_executable),
+                cli_entrypoint,
                 "run-driver",
                 stdin=asyncio.subprocess.PIPE,
                 stdout=asyncio.subprocess.PIPE,
@@ -150,6 +146,9 @@ async def connect(self) -> None:
         except Exception as exc:
             self.on_error_future.set_exception(exc)
             raise exc
+        finally:
+            if sys.version_info[0] == 3 and sys.version_info[1] == 7:
+                asyncio.set_child_watcher(original_child_watcher)
 
         self._output = self._proc.stdin
 
@@ -159,22 +158,30 @@ async def run(self) -> None:
         while not self._stopped:
             try:
                 buffer = await self._proc.stdout.readexactly(4)
+                if self._stopped:
+                    break
                 length = int.from_bytes(buffer, byteorder="little", signed=False)
                 buffer = bytes(0)
                 while length:
                     to_read = min(length, 32768)
                     data = await self._proc.stdout.readexactly(to_read)
+                    if self._stopped:
+                        break
                     length -= to_read
                     if len(buffer):
                         buffer = buffer + data
                     else:
                         buffer = data
+                if self._stopped:
+                    break
 
                 obj = self.deserialize_message(buffer)
                 self.on_message(obj)
             except asyncio.IncompleteReadError:
                 break
             await asyncio.sleep(0)
+
+        await self._proc.wait()
         self._stopped_future.set_result(None)
 
     def send(self, message: Dict) -> None:

playwright/_impl/_transport.py

@@ -16,7 +16,6 @@
 from typing import Any
 
 from playwright._impl._connection import Connection
-from playwright._impl._driver import compute_driver_executable
 from playwright._impl._object_factory import create_remote_object
 from playwright._impl._transport import PipeTransport
 from playwright.async_api._generated import Playwright as AsyncPlaywright
@@ -31,7 +30,7 @@ async def __aenter__(self) -> AsyncPlaywright:
         self._connection = Connection(
             None,
             create_remote_object,
-            PipeTransport(loop, compute_driver_executable()),
+            PipeTransport(loop),
             loop,
         )
         loop.create_task(self._connection.run())

playwright/async_api/_context_manager.py

@@ -19,7 +19,6 @@
 
 from playwright._impl._api_types import Error
 from playwright._impl._connection import Connection
-from playwright._impl._driver import compute_driver_executable
 from playwright._impl._object_factory import create_remote_object
 from playwright._impl._playwright import Playwright
 from playwright._impl._transport import PipeTransport
@@ -51,7 +50,7 @@ def greenlet_main() -> None:
         self._connection = Connection(
             dispatcher_fiber,
             create_remote_object,
-            PipeTransport(self._loop, compute_driver_executable()),
+            PipeTransport(self._loop),
             self._loop,
         )
 
@@ -73,7 +72,6 @@ def start(self) -> SyncPlaywright:
 
     def __exit__(self, *args: Any) -> None:
         self._connection.stop_sync()
-        self._loop.run_until_complete(self._connection._transport.wait_until_stopped())
         if self._own_loop:
             self._loop.run_until_complete(self._loop.shutdown_asyncgens())
             self._loop.close()

playwright/sync_api/_context_manager.py

@@ -212,7 +212,6 @@ def _download_and_extract_local_driver(
         "greenlet==1.1.2",
         "pyee==8.1.0",
         "typing-extensions;python_version<='3.8'",
-        "psutil==5.9.0;python_version<='3.7'",
     ],
     classifiers=[
         "Topic :: Software Development :: Testing",