Avoid subprocess memory copy when c library supports posix_spawn

By default python 3.10 will use the fork() which has to
copy memory of the parent process (in our case
this can be huge since Home Assistant core can use
hundreds of megabytes of RAM). By using posix_spawn
this is avoided and subprocess creation does not
get discernibly slow the larger the Home Assistant
python process grows.

In python 3.11 vfork will also be available
python/cpython#80004 (comment)
python/cpython#11671 but we won't
always be able to use it and posix_spawn is considered safer
https://bugzilla.kernel.org/show_bug.cgi?id=215813#c14

The subprocess library doesn't know about musl though
even though it supports posix_spawn https://git.musl-libc.org/cgit/musl/log/src/process/posix_spawn.c
so we have to teach it since it only has checks for glibc
https://github.com/python/cpython/blob/1b736838e6ae1b4ef42cdd27c2708face908f92c/Lib/subprocess.py#L745

The constant is documented as being able to be flipped here:
https://docs.python.org/3/library/subprocess.html#disabling-use-of-vfork-or-posix-spawn

Author: bdraco

homeassistant/auth/providers/command_line.py

@@ -68,6 +68,7 @@ async def async_validate_login(self, username: str, password: str) -> None:
                 *self.config[CONF_ARGS],
                 env=env,
                 stdout=asyncio.subprocess.PIPE if self.config[CONF_META] else None,
+                close_fds=False,  # required for posix_spawn
             )
             stdout, _ = await process.communicate()
         except OSError as err:

homeassistant/components/command_line/__init__.py

@@ -18,7 +18,10 @@ def call_shell_with_timeout(
     try:
         _LOGGER.debug("Running command: %s", command)
         subprocess.check_output(
-            command, shell=True, timeout=timeout  # nosec # shell by design
+            command,
+            shell=True,  # nosec # shell by design
+            timeout=timeout,
+            close_fds=False,  # required for posix_spawn
         )
         return 0
     except subprocess.CalledProcessError as proc_exception:
@@ -41,7 +44,10 @@ def check_output_or_log(command: str, timeout: int) -> str | None:
     """Run a shell command with a timeout and return the output."""
     try:
         return_value = subprocess.check_output(
-            command, shell=True, timeout=timeout  # nosec # shell by design
+            command,
+            shell=True,  # nosec # shell by design
+            timeout=timeout,
+            close_fds=False,  # required for posix_spawn
         )
         return return_value.strip().decode("utf-8")
     except subprocess.CalledProcessError as err:

homeassistant/components/command_line/notify.py

@@ -52,6 +52,7 @@ def send_message(self, message="", **kwargs) -> None:
             self.command,
             universal_newlines=True,
             stdin=subprocess.PIPE,
+            close_fds=False,  # required for posix_spawn
             shell=True,  # nosec # shell by design
         ) as proc:
             try:

homeassistant/components/ping/binary_sensor.py

@@ -227,6 +227,7 @@ async def async_ping(self):
             stdin=None,
             stdout=asyncio.subprocess.PIPE,
             stderr=asyncio.subprocess.PIPE,
+            close_fds=False,  # required for posix_spawn
         )
         try:
             out_data, out_error = await asyncio.wait_for(

homeassistant/components/ping/device_tracker.py

@@ -55,7 +55,10 @@ def __init__(self, ip_address, dev_id, hass, config, privileged):
     def ping(self):
         """Send an ICMP echo request and return True if success."""
         with subprocess.Popen(
-            self._ping_cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL
+            self._ping_cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            close_fds=False,  # required for posix_spawn
         ) as pinger:
             try:
                 pinger.communicate(timeout=1 + PING_TIMEOUT)

homeassistant/components/rpi_camera/camera.py

@@ -32,7 +32,10 @@
 def kill_raspistill(*args):
     """Kill any previously running raspistill process.."""
     with subprocess.Popen(
-        ["killall", "raspistill"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT
+        ["killall", "raspistill"],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.STDOUT,
+        close_fds=False,  # required for posix_spawn
     ):
         pass
 
@@ -132,7 +135,10 @@ def __init__(self, device_info):
         # Therefore it must not be wrapped with "with", since that
         # waits for the subprocess to exit before continuing.
         subprocess.Popen(  # pylint: disable=consider-using-with
-            cmd_args, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT
+            cmd_args,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.STDOUT,
+            close_fds=False,  # required for posix_spawn
         )
 
     def camera_image(

homeassistant/components/shell_command/__init__.py

@@ -65,6 +65,7 @@ async def async_service_handler(service: ServiceCall) -> None:
                 stdin=None,
                 stdout=asyncio.subprocess.PIPE,
                 stderr=asyncio.subprocess.PIPE,
+                close_fds=False,  # required for posix_spawn
             )
         else:
             # Template used. Break into list and use create_subprocess_exec
@@ -76,6 +77,7 @@ async def async_service_handler(service: ServiceCall) -> None:
                 stdin=None,
                 stdout=asyncio.subprocess.PIPE,
                 stderr=asyncio.subprocess.PIPE,
+                close_fds=False,  # required for posix_spawn
             )
 
         process = await create_process

homeassistant/runner.py

@@ -4,6 +4,8 @@
 import asyncio
 import dataclasses
 import logging
+import os
+import subprocess
 import threading
 import traceback
 from typing import Any
@@ -26,6 +28,7 @@
 #
 MAX_EXECUTOR_WORKERS = 64
 TASK_CANCELATION_TIMEOUT = 5
+ALPINE_RELEASE_FILE = "/etc/alpine-release"
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -120,6 +123,16 @@ async def setup_and_run_hass(runtime_config: RuntimeConfig) -> int:
 
 def run(runtime_config: RuntimeConfig) -> int:
     """Run Home Assistant."""
+    if not subprocess._USE_POSIX_SPAWN:  # pylint: disable=protected-access
+        # The subprocess module does not know about Alpine Linux/musl
+        # and will use fork() instead of posix_spawn() which significantly
+        # less efficient. This is a workaround to force posix_spawn()
+        # on Alpine Linux which is supported by musl.
+        _exists = os.path.exists
+        subprocess._USE_POSIX_SPAWN = _exists(  # pylint: disable=protected-access
+            ALPINE_RELEASE_FILE
+        )
+
     asyncio.set_event_loop_policy(HassEventLoopPolicy(runtime_config.debug))
     # Backport of cpython 3.9 asyncio.run with a _cancel_all_tasks that times out
     loop = asyncio.new_event_loop()

homeassistant/util/package.py

@@ -121,6 +121,7 @@ async def async_get_user_site(deps_dir: str) -> str:
         stdout=asyncio.subprocess.PIPE,
         stderr=asyncio.subprocess.DEVNULL,
         env=env,
+        close_fds=False,  # required for posix_spawn
     )
     stdout, _ = await process.communicate()
     lib_dir = stdout.decode().strip()