Add make install-sysdeps target + re-enable BSD builds (#2448)

* Add make install-sysdeps target. I want a standardized way to satisfy all system deps from a central point across all UNIX platforms. From now on CI config files should always rely on make install-sysdeps.
* Rename setup-dev-env make target to install-pydeps (these are deps intended for running tests)
* Re-enable BSD builds

.github/workflows/bsd.yml

@@ -10,65 +10,41 @@ concurrency:
   group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.sha || '' }}
   cancel-in-progress: true
 jobs:
-  # here just so that I can comment the next jobs to temporarily disable them
-  empty-job:
-    if: false
-    runs-on: ubuntu-22.04
+  freebsd:
+    # if: false
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Run tests
+        uses: vmactions/freebsd-vm@v1
+        with:
+          usesh: true
+          run: |
+            scripts/internal/install-sysdeps.sh
+            PIP_BREAK_SYSTEM_PACKAGES=1 gmake install-pydeps install print-sysinfo test test-memleaks
 
-#   freebsd:
-#     runs-on: ubuntu-22.04
-#     steps:
-#       - uses: actions/checkout@v4
-#       - name: Run tests
-#         uses: vmactions/freebsd-vm@v1
-#         with:
-#           usesh: true
-#           prepare: |
-#             pkg install -y gcc python3
-#           run: |
-#             set -e -x
-#             make install-pip
-#             python3 -m pip install --user setuptools
-#             make install
-#             make test
-#             make test-memleaks
-#   openbsd:
-#     runs-on: ubuntu-22.04
-#     steps:
-#       - uses: actions/checkout@v4
-#       - name: Run tests
-#         uses: vmactions/openbsd-vm@v1
-#         with:
-#           usesh: true
-#           prepare: |
-#             set -e
-#             pkg_add gcc python3
-#           run: |
-#             set -e
-#             make install-pip
-#             python3 -m pip install --user setuptools
-#             make install
-#             make test
-#             make test-memleaks
-#   netbsd:
-#     runs-on: ubuntu-22.04
-#     steps:
-#       - uses: actions/checkout@v4
-#       - name: Run tests
-#         uses: vmactions/netbsd-vm@v1
-#         with:
-#           usesh: true
-#           prepare: |
-#             set -e
-#             /usr/sbin/pkg_add -v pkgin
-#             pkgin update
-#             pkgin -y install python311-* py311-setuptools-* gcc12-*
-#           run: |
-#             set -e
-#             make install-pip PYTHON=python3.11
-#             python3.11 -m pip install --user setuptools
-#             make install PYTHON=python3.11
-#             make test PYTHON=python3.11
-#             make test-memleaks PYTHON=python3.11
+  openbsd:
+    # if: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run tests
+        uses: vmactions/openbsd-vm@v1
+        with:
+          usesh: true
+          run: |
+            scripts/internal/install-sysdeps.sh
+            PIP_BREAK_SYSTEM_PACKAGES=1 gmake install-pydeps install print-sysinfo test test-memleaks
+
+  netbsd:
+    # if: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run tests
+        uses: vmactions/netbsd-vm@v1
+        with:
+          usesh: true
+          run: |
+            scripts/internal/install-sysdeps.sh
+            PIP_BREAK_SYSTEM_PACKAGES=1 gmake PYTHON=python3.11 install-pydeps install print-sysinfo test test-memleaks

.github/workflows/build.yml

@@ -59,7 +59,7 @@ jobs:
         CIBW_PRERELEASE_PYTHONS: True
         CIBW_TEST_EXTRAS: test
         CIBW_TEST_COMMAND:
-          make -C {project} PYTHON="env python" PSUTIL_SCRIPTS_DIR="{project}/scripts" setup-dev-env install print-sysinfo test test-memleaks
+          make -C {project} PYTHON="env python" PSUTIL_SCRIPTS_DIR="{project}/scripts" install-sysdeps install-pydeps install print-sysinfo test test-memleaks
 
     - name: Upload wheels
       uses: actions/upload-artifact@v4
@@ -87,7 +87,7 @@ jobs:
       CIBW_BUILD: 'cp27-*'
       CIBW_TEST_EXTRAS: test
       CIBW_TEST_COMMAND:
-        make -C {project} PYTHON="env python" PSUTIL_SCRIPTS_DIR="{project}/scripts" setup-dev-env install print-sysinfo test test-memleaks
+        make -C {project} PYTHON="env python" PSUTIL_SCRIPTS_DIR="{project}/scripts" install-sysdeps install-pydeps install print-sysinfo test test-memleaks
 
     steps:
     - uses: actions/checkout@v4

HISTORY.rst

@@ -10,6 +10,12 @@ XXXX-XX-XX
 - 2427_: psutil (segfault) on import in the free-threaded (no GIL) version of
   Python 3.13.  (patch by Sam Gross)
 
+**Enhancements**
+
+- 2448_: add ``make install-sysdeps`` target to install all necessary system
+  dependencies (python-dev, gcc, etc.) on different UNIX flavors. Also rename
+  ``make setup-dev-env`` to ``make install-pydeps``.
+
 6.0.0
 ======
 

INSTALL.rst

@@ -12,10 +12,36 @@ install a C compiler. All you have to do is::
 If wheels are not available for your platform or architecture, or you wish to
 build & install psutil from sources, keep reading.
 
-Linux (build)
--------------
+Compile psutil from sources
+===========================
+
+UNIX
+----
+
+On all UNIX systems you can use the
+`install-sysdeps.sh <https://github.com/giampaolo/psutil/blob/master/scripts/internal/install-sysdeps.sh>`__
+script to install the system dependencies necessary to compile psutil. You can
+invoke this script from the Makefile as::
+
+    make install-sysdeps
+
+If you're on a BSD platform you need to use ``gmake`` instead of ``make``::
+
+    gmake install-sysdeps
+
+After system deps are installed you can build & compile psutil with::
 
-Ubuntu / Debian::
+    make build
+    make install
+
+...or this, which will fetch the latest source distribution from `PyPI <https://pypi.org/project/psutil/>`__::
+
+    pip install --no-binary :all: psutil
+
+Linux
+-----
+
+Debian / Ubuntu::
 
     sudo apt-get install gcc python3-dev
     pip install --no-binary :all: psutil
@@ -30,10 +56,10 @@ Alpine::
     sudo apk add gcc python3-dev musl-dev linux-headers
     pip install --no-binary :all: psutil
 
-Windows (build)
----------------
+Windows
+-------
 
-In order to install psutil from sources on Windows you need Visual Studio
+In order to build / install psutil from sources on Windows you need Visual Studio
 (MinGW is not supported).
 Here's a couple of guides describing how to do it: `link <https://blog.ionelmc.ro/2014/12/21/compiling-python-extensions-on-windows/>`__
 and `link <https://cpython-core-tutorial.readthedocs.io/en/latest/build_cpython_windows.html>`__.
@@ -96,7 +122,7 @@ Install pip
 -----------
 
 Pip is shipped by default with Python 2.7.9+ and 3.4+.
-If you don't have pip you can install with wget::
+If you don't have pip you can install it with wget::
 
     wget https://bootstrap.pypa.io/get-pip.py -O - | python3
 

MANIFEST.in

@@ -187,6 +187,7 @@ include scripts/internal/download_wheels_appveyor.py
 include scripts/internal/download_wheels_github.py
 include scripts/internal/generate_manifest.py
 include scripts/internal/git_pre_commit.py
+include scripts/internal/install-sysdeps.sh
 include scripts/internal/print_access_denied.py
 include scripts/internal/print_announce.py
 include scripts/internal/print_api_speed.py

Makefile

@@ -14,25 +14,28 @@ PY3_DEPS = \
 	pytest-xdist
 
 # deps for local development
-ifndef CIBUILDWHEEL
-	PY3_DEPS += \
-		black \
-		check-manifest \
-		coverage \
-		packaging \
-		pylint \
-		pyperf \
-		pypinfo \
-		pytest-cov \
-		requests \
-		rstcheck \
-		ruff \
-		setuptools \
-		sphinx_rtd_theme \
-		toml-sort \
-		twine \
-		virtualenv \
-		wheel
+ifdef LINUX
+	ifndef CIBUILDWHEEL
+		PY3_DEPS += \
+			black \
+			check-manifest \
+			coverage \
+			packaging \
+			pylint \
+			pyperf \
+			pypinfo \
+			pytest-cov \
+			requests \
+			rstcheck \
+			ruff \
+			setuptools \
+			sphinx \
+			sphinx_rtd_theme \
+			toml-sort \
+			twine \
+			virtualenv \
+			wheel
+	endif
 endif
 
 # python 2 deps
@@ -117,6 +120,7 @@ uninstall:  ## Uninstall this package via pip.
 install-pip:  ## Install pip (no-op if already installed).
 	@$(PYTHON) -c \
 		"import sys, ssl, os, pkgutil, tempfile, atexit; \
+		print('pip already installed') if pkgutil.find_loader('pip') else None; \
 		sys.exit(0) if pkgutil.find_loader('pip') else None; \
 		PY3 = sys.version_info[0] == 3; \
 		pyexc = 'from urllib.request import urlopen' if PY3 else 'from urllib2 import urlopen'; \
@@ -135,7 +139,10 @@ install-pip:  ## Install pip (no-op if already installed).
 		f.close(); \
 		sys.exit(code);"
 
-setup-dev-env:  ## Install GIT hooks, pip, test deps (also upgrades them).
+install-sysdeps:
+	scripts/internal/install-sysdeps.sh
+
+install-pydeps:  ## Install GIT hooks, pip, test deps (also upgrades them).
 	${MAKE} install-git-hooks
 	${MAKE} install-pip
 	$(PYTHON) -m pip install $(INSTALL_OPTS) --trusted-host files.pythonhosted.org --trusted-host pypi.org --upgrade pip

appveyor.yml

@@ -36,7 +36,7 @@ init:
 
 install:
   - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version"
-  - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py setup-dev-env"
+  - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py install-pydeps"
   - "%WITH_COMPILER% %PYTHON%/python.exe -m pip freeze"
   - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py install"
 

docs/DEVGUIDE.rst

@@ -12,7 +12,7 @@ Once you have a compiler installed run:
 .. code-block:: bash
 
     git clone git@github.com:giampaolo/psutil.git
-    make setup-dev-env  # install useful dev libs (ruff, coverage, ...)
+    make install-pydeps  # install useful dev libs (ruff, coverage, ...)
     make build
     make install
     make test
@@ -118,7 +118,7 @@ Documentation
 -------------
 
 - doc source code is written in a single file: ``docs/index.rst``.
-- doc can be built with ``make setup-dev-env; cd docs; make html``.
+- doc can be built with ``make install-pydeps; cd docs; make html``.
 - public doc is hosted at https://psutil.readthedocs.io.
 
 .. _`CREDITS`: https://github.com/giampaolo/psutil/blob/master/CREDITS

docs/Makefile

@@ -229,6 +229,6 @@ dummy:
 	@echo
 	@echo "Build finished. Dummy builder generates no files."
 
-.PHONY: setup-dev-env
-setup-dev-env:  ## Install GIT hooks, pip, test deps (also upgrades them).
+.PHONY: install-pydeps
+install-pydeps:  ## Install GIT hooks, pip, test deps (also upgrades them).
 	$(PYTHON) -m pip install --user --upgrade --trusted-host files.pythonhosted.org $(DEPS)

psutil/tests/README.rst

@@ -9,6 +9,6 @@ Instructions for running tests
   As a "developer", if you have a copy of the source code and you wish to hack
   on psutil::
 
-    make setup-dev-env  # install missing third-party deps
+    make install-pydeps  # install missing third-party deps
     make test           # serial run
     make test-parallel  # parallel run

psutil/tests/test_windows.py

@@ -47,7 +47,7 @@
         import win32api  # requires "pip install pywin32"
         import win32con
         import win32process
-        import wmi  # requires "pip install wmi" / "make setup-dev-env"
+        import wmi  # requires "pip install wmi" / "make install-pydeps"
 
 if WINDOWS:
     from psutil._pswindows import convert_oserror

pyproject.toml

@@ -215,9 +215,6 @@ skip = [
 [tool.cibuildwheel.macos]
 archs = ["arm64", "x86_64"]
 
-[tool.cibuildwheel.linux]
-before-all = "yum install -y net-tools"
-
 [build-system]
 build-backend = "setuptools.build_meta"
 requires = ["setuptools>=43", "wheel"]

scripts/internal/install-sysdeps.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+
+# Depending on the UNIX platform, install the necessary system dependencies to:
+# * compile psutil
+# * run those unit tests that rely on CLI tools (netstat, ps, etc.)
+# NOTE: this script MUST be kept compatible with the `sh` shell.
+
+set -e
+
+UNAME_S=$(uname -s)
+
+case "$UNAME_S" in
+    Linux)
+        LINUX=true
+        if command -v apt > /dev/null 2>&1; then
+            HAS_APT=true
+        elif command -v yum > /dev/null 2>&1; then
+            HAS_YUM=true
+        elif command -v apk > /dev/null 2>&1; then
+            HAS_APK=true  # musl linux
+        fi
+        ;;
+    FreeBSD)
+        FREEBSD=true
+        ;;
+    NetBSD)
+        NETBSD=true
+        ;;
+    OpenBSD)
+        OPENBSD=true
+        ;;
+esac
+
+# Check if running as root
+if [ "$(id -u)" -ne 0 ]; then
+    SUDO=sudo
+fi
+
+# Function to install system dependencies
+main() {
+    if [ $HAS_APT ]; then
+        $SUDO apt-get install -y python3-dev gcc
+        $SUDO apt-get install -y net-tools coreutils util-linux  # for tests
+    elif [ $HAS_YUM ]; then
+        $SUDO yum install -y python3-devel gcc
+        $SUDO yum install -y net-tools coreutils util-linux  # for tests
+    elif [ $HAS_APK ]; then
+        $SUDO apk add python3-dev gcc musl-dev linux-headers coreutils procps
+    elif [ $FREEBSD ]; then
+        $SUDO pkg install -y gmake python3 gcc
+    elif [ $NETBSD ]; then
+        $SUDO /usr/sbin/pkg_add -v pkgin
+        $SUDO pkgin update
+        $SUDO pkgin -y install gmake python311-* gcc12-*
+    elif [ $OPENBSD ]; then
+        $SUDO pkg_add gmake gcc python3
+    else
+        echo "Unsupported platform: $UNAME_S"
+    fi
+}
+
+main