Provide test and dev python dependencies (#2449)

.github/workflows/bsd.yml

@@ -20,8 +20,7 @@ jobs:
         with:
           usesh: true
           run: |
-            scripts/internal/install-sysdeps.sh
-            PIP_BREAK_SYSTEM_PACKAGES=1 gmake install-pydeps install print-sysinfo test test-memleaks
+            PIP_BREAK_SYSTEM_PACKAGES=1 make install-sysdeps install-pydeps-test install print-sysinfo test test-memleaks
 
   openbsd:
     # if: false
@@ -33,8 +32,7 @@ jobs:
         with:
           usesh: true
           run: |
-            scripts/internal/install-sysdeps.sh
-            PIP_BREAK_SYSTEM_PACKAGES=1 gmake install-pydeps install print-sysinfo test test-memleaks
+            PIP_BREAK_SYSTEM_PACKAGES=1 make install-sysdeps install-pydeps-test install print-sysinfo test test-memleaks
 
   netbsd:
     # if: false
@@ -46,5 +44,4 @@ jobs:
         with:
           usesh: true
           run: |
-            scripts/internal/install-sysdeps.sh
-            PIP_BREAK_SYSTEM_PACKAGES=1 gmake PYTHON=python3.11 install-pydeps install print-sysinfo test test-memleaks
+            PIP_BREAK_SYSTEM_PACKAGES=1 make PYTHON=python3.11 install-sysdeps install-pydeps-test install print-sysinfo test test-memleaks

.github/workflows/build.yml

@@ -59,7 +59,7 @@ jobs:
         CIBW_PRERELEASE_PYTHONS: True
         CIBW_TEST_EXTRAS: test
         CIBW_TEST_COMMAND:
-          make -C {project} PYTHON="env python" PSUTIL_SCRIPTS_DIR="{project}/scripts" install-sysdeps install-pydeps install print-sysinfo test test-memleaks
+          make -C {project} PYTHON="env python" PSUTIL_SCRIPTS_DIR="{project}/scripts" install-sysdeps install-pydeps-test install print-sysinfo test test-memleaks
 
     - name: Upload wheels
       uses: actions/upload-artifact@v4
@@ -87,7 +87,7 @@ jobs:
       CIBW_BUILD: 'cp27-*'
       CIBW_TEST_EXTRAS: test
       CIBW_TEST_COMMAND:
-        make -C {project} PYTHON="env python" PSUTIL_SCRIPTS_DIR="{project}/scripts" install-sysdeps install-pydeps install print-sysinfo test test-memleaks
+        make -C {project} PYTHON="env python" PSUTIL_SCRIPTS_DIR="{project}/scripts" install-sysdeps install-pydeps-test install print-sysinfo test test-memleaks
 
     steps:
     - uses: actions/checkout@v4

HISTORY.rst

@@ -5,17 +5,20 @@
 
 XXXX-XX-XX
 
+**Enhancements**
+
+- 2448_: add ``make install-sysdeps`` target to install the necessary system
+  dependencies (python-dev, gcc, etc.) on all supported UNIX flavors.
+- 2449_: add ``make install-pydeps-test`` and ``make install-pydeps-dev``
+  targets. They can be used to install dependencies meant for running tests and
+  for local development. They can also be installed via ``pip install .[test]``
+  and ``pip install .[dev]``.
+
 **Bug fixes**
 
 - 2427_: psutil (segfault) on import in the free-threaded (no GIL) version of
   Python 3.13.  (patch by Sam Gross)
 
-**Enhancements**
-
-- 2448_: add ``make install-sysdeps`` target to install all necessary system
-  dependencies (python-dev, gcc, etc.) on different UNIX flavors. Also rename
-  ``make setup-dev-env`` to ``make install-pydeps``.
-
 6.0.0
 ======
 

INSTALL.rst

@@ -18,18 +18,14 @@ Compile psutil from sources
 UNIX
 ----
 
-On all UNIX systems you can use the
-`install-sysdeps.sh <https://github.com/giampaolo/psutil/blob/master/scripts/internal/install-sysdeps.sh>`__
-script to install the system dependencies necessary to compile psutil. You can
-invoke this script from the Makefile as::
+On all UNIX systems you can use the `install-sysdeps.sh
+<https://github.com/giampaolo/psutil/blob/master/scripts/internal/install-sysdeps.sh>`__
+script. This will install the system dependencies necessary to compile psutil
+from sources. You can invoke this script from the Makefile as::
 
     make install-sysdeps
 
-If you're on a BSD platform you need to use ``gmake`` instead of ``make``::
-
-    gmake install-sysdeps
-
-After system deps are installed you can build & compile psutil with::
+After system deps are installed, you can compile & install psutil with::
 
     make build
     make install
@@ -59,11 +55,10 @@ Alpine::
 Windows
 -------
 
-In order to build / install psutil from sources on Windows you need Visual Studio
-(MinGW is not supported).
-Here's a couple of guides describing how to do it: `link <https://blog.ionelmc.ro/2014/12/21/compiling-python-extensions-on-windows/>`__
-and `link <https://cpython-core-tutorial.readthedocs.io/en/latest/build_cpython_windows.html>`__.
-Once VS is installed do::
+In order to build / install psutil from sources on Windows you need to install
+`Visua Studio 2017 <https://visualstudio.microsoft.com/vs/older-downloads/>`__
+or later (see cPython `devguide <https://devguide.python.org/getting-started/setup-building/#windows>`__'s instructions).
+MinGW is not supported. Once Visual Studio is installed do::
 
     pip install --no-binary :all: psutil
 

Makefile

@@ -13,31 +13,6 @@ PY3_DEPS = \
 	pytest \
 	pytest-xdist
 
-# deps for local development
-ifdef LINUX
-	ifndef CIBUILDWHEEL
-		PY3_DEPS += \
-			black \
-			check-manifest \
-			coverage \
-			packaging \
-			pylint \
-			pyperf \
-			pypinfo \
-			pytest-cov \
-			requests \
-			rstcheck \
-			ruff \
-			setuptools \
-			sphinx \
-			sphinx_rtd_theme \
-			toml-sort \
-			twine \
-			virtualenv \
-			wheel
-	endif
-endif
-
 # python 2 deps
 PY2_DEPS = \
 	futures \
@@ -64,12 +39,16 @@ BUILD_OPTS = `$(PYTHON) -c \
 	print('--parallel %s' % cpus if cpus > 1 else '')"`
 
 # In not in a virtualenv, add --user options for install commands.
-INSTALL_OPTS = `$(PYTHON) -c \
+SETUP_INSTALL_OPTS = `$(PYTHON) -c \
 	"import sys; print('' if hasattr(sys, 'real_prefix') or hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix else '--user')"`
+PIP_INSTALL_OPTS = --trusted-host files.pythonhosted.org --trusted-host pypi.org --upgrade
 
 # if make is invoked with no arg, default to `make help`
 .DEFAULT_GOAL := help
 
+# install git hook
+_ := $(shell mkdir -p .git/hooks/ && ln -sf ../../scripts/internal/git_pre_commit.py .git/hooks/pre-commit && chmod +x .git/hooks/pre-commit)
+
 # ===================================================================
 # Install
 # ===================================================================
@@ -110,8 +89,7 @@ build:  ## Compile (in parallel) without installing.
 
 install:  ## Install this package as current user in "edit" mode.
 	${MAKE} build
-	$(PYTHON_ENV_VARS) $(PYTHON) setup.py develop $(INSTALL_OPTS)
-	$(PYTHON_ENV_VARS) $(PYTHON) -c "import psutil"  # make sure it actually worked
+	$(PYTHON_ENV_VARS) $(PYTHON) setup.py develop $(SETUP_INSTALL_OPTS)
 
 uninstall:  ## Uninstall this package via pip.
 	cd ..; $(PYTHON_ENV_VARS) $(PYTHON) -m pip uninstall -y -v psutil || true
@@ -140,13 +118,22 @@ install-pip:  ## Install pip (no-op if already installed).
 		sys.exit(code);"
 
 install-sysdeps:
-	scripts/internal/install-sysdeps.sh
+	./scripts/internal/install-sysdeps.sh
+
+install-pydeps-test:  ## Install python deps necessary to run unit tests.
+	${MAKE} install-pip
+	$(PYTHON) -m pip install $(PIP_INSTALL_OPTS) pip  # upgrade pip to latest version
+	$(PYTHON) -m pip install $(PIP_INSTALL_OPTS) `$(PYTHON) -c "import setup; print(' '.join(setup.TEST_DEPS))"`
 
-install-pydeps:  ## Install GIT hooks, pip, test deps (also upgrades them).
+install-pydeps-dev:  ## Install python deps meant for local development.
 	${MAKE} install-git-hooks
 	${MAKE} install-pip
-	$(PYTHON) -m pip install $(INSTALL_OPTS) --trusted-host files.pythonhosted.org --trusted-host pypi.org --upgrade pip
-	$(PYTHON) -m pip install $(INSTALL_OPTS) --trusted-host files.pythonhosted.org --trusted-host pypi.org --upgrade $(PY_DEPS)
+	$(PYTHON) -m pip install $(PIP_INSTALL_OPTS) pip  # upgrade pip to latest version
+	$(PYTHON) -m pip install $(PIP_INSTALL_OPTS) `$(PYTHON) -c "import setup; print(' '.join(setup.TEST_DEPS + setup.DEV_DEPS))"`
+
+install-git-hooks:  ## Install GIT pre-commit hook.
+	ln -sf ../../scripts/internal/git_pre_commit.py .git/hooks/pre-commit
+	chmod +x .git/hooks/pre-commit
 
 # ===================================================================
 # Tests
@@ -265,14 +252,6 @@ fix-all:  ## Run all code fixers.
 	${MAKE} fix-black
 	${MAKE} fix-toml
 
-# ===================================================================
-# GIT
-# ===================================================================
-
-install-git-hooks:  ## Install GIT pre-commit hook.
-	ln -sf ../../scripts/internal/git_pre_commit.py .git/hooks/pre-commit
-	chmod +x .git/hooks/pre-commit
-
 # ===================================================================
 # Distribution
 # ===================================================================

appveyor.yml

@@ -36,14 +36,14 @@ init:
 
 install:
   - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version"
-  - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py install-pydeps"
+  - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py install-pydeps-test"
   - "%WITH_COMPILER% %PYTHON%/python.exe -m pip freeze"
   - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py install"
+  - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py print-sysinfo"
 
 build: off
 
 test_script:
-  - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py print-sysinfo"
   - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py test"
   - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py test-memleaks"
 

docs/DEVGUIDE.rst

@@ -12,32 +12,36 @@ Once you have a compiler installed run:
 .. code-block:: bash
 
     git clone git@github.com:giampaolo/psutil.git
-    make install-pydeps  # install useful dev libs (ruff, coverage, ...)
+    make install-sysdeps      # install gcc and python headers
+    make install-pydeps-test  # install python deps necessary to run unit tests
     make build
     make install
     make test
 
-- ``make`` (see `Makefile`_) is the designated tool to run tests, build, install
-  try new features you're developing, etc. This also includes Windows (see
-  `make.bat`_ ).
-  Some useful commands are:
+- ``make`` (and the accompanying `Makefile`_) is the designated tool to build,
+  install, run tests and do pretty much anything that involves development.
+  This also includes Windows, meaning that you can run `make command` similarly
+  as if you were on UNIX (see `make.bat`_ and `winmake.py`_). Some useful
+  commands are:
 
 .. code-block:: bash
 
-    make test-parallel   # faster
-    make test-memleaks
-    make test-coverage
-    make lint-all        # Run Python and C linter
-    make fix-all         # Fix linting errors
+    make clean                # remove build files
+    make install-pydeps-dev   # install dev deps (ruff, black, ...)
+    make test                 # run tests
+    make test-parallel        # run tests in parallel (faster)
+    make test-memleaks        # run memory leak tests
+    make test-coverage        # run test coverage
+    make lint-all             # run linters
+    make fix-all              # fix linters errors
     make uninstall
     make help
 
-
 - To run a specific unit test:
 
 .. code-block:: bash
 
-    make test ARGS=psutil.tests.test_system.TestDiskAPIs
+    make test ARGS=psutil/tests/test_system.py::TestDiskAPIs::test_disk_usage
 
 - If you're working on a new feature and you wish to compile & test it "on the
   fly" from a test script, this is a quick & dirty way to do it:
@@ -48,7 +52,8 @@ Once you have a compiler installed run:
     make test test_script.py          # on Windows
 
 - Do not use ``sudo``. ``make install`` installs psutil as a limited user in
-  "edit" mode, meaning you can edit psutil code on the fly while you develop.
+  "edit" / development mode, meaning you can edit psutil code on the fly while
+  you develop.
 
 - If you want to target a specific Python version:
 
@@ -60,10 +65,12 @@ Once you have a compiler installed run:
 Coding style
 ------------
 
-- Oython code strictly follows `PEP-8`_ styling guides and this is enforced by
-  a commit GIT hook installed via ``make install-git-hooks`` which will reject
-  commits if code is not PEP-8 complieant.
-- C code should follow `PEP-7`_ styling guides.
+- Python code strictly follows `PEP-8`_ styling guide. In addition we use
+  ``black`` and ``ruff`` code formatters / linters. This is enforced by a GIT
+  commit hook, installed via ``make install-git-hooks``, which will reject the
+  commit if code is not compliant.
+- C code should follow `PEP-7`_ styling guides, but things are more relaxed.
+- Linters are enforced also for ``.rst`` and ``.toml`` files.
 
 Code organization
 -----------------
@@ -101,7 +108,7 @@ Make a pull request
 
 - Fork psutil (go to https://github.com/giampaolo/psutil and click on "fork")
 - Git clone the fork locally: ``git clone git@github.com:YOUR-USERNAME/psutil.git``
-- Create a branch:``git checkout -b new-feature``
+- Create a branch: ``git checkout -b new-feature``
 - Commit your changes: ``git commit -am 'add some feature'``
 - Push the branch: ``git push origin new-feature``
 - Create a new PR via the GitHub web interface and sign-off your work (see
@@ -118,13 +125,14 @@ Documentation
 -------------
 
 - doc source code is written in a single file: ``docs/index.rst``.
-- doc can be built with ``make install-pydeps; cd docs; make html``.
+- doc can be built with ``make install-pydeps-dev; cd docs; make html``.
 - public doc is hosted at https://psutil.readthedocs.io.
 
 .. _`CREDITS`: https://github.com/giampaolo/psutil/blob/master/CREDITS
 .. _`CONTRIBUTING.md`: https://github.com/giampaolo/psutil/blob/master/CONTRIBUTING.md
 .. _`HISTORY.rst`: https://github.com/giampaolo/psutil/blob/master/HISTORY.rst
 .. _`make.bat`: https://github.com/giampaolo/psutil/blob/master/make.bat
+.. _`winmake.py`: https://github.com/giampaolo/psutil/blob/master/scripts/internal/winmake.py
 .. _`Makefile`: https://github.com/giampaolo/psutil/blob/master/Makefile
 .. _`PEP-7`: https://www.python.org/dev/peps/pep-0007/
 .. _`PEP-8`: https://www.python.org/dev/peps/pep-0008/

psutil/tests/test_windows.py

@@ -47,7 +47,7 @@
         import win32api  # requires "pip install pywin32"
         import win32con
         import win32process
-        import wmi  # requires "pip install wmi" / "make install-pydeps"
+        import wmi  # requires "pip install wmi" / "make install-pydeps-test"
 
 if WINDOWS:
     from psutil._pswindows import convert_oserror

scripts/internal/install-sysdeps.sh

@@ -47,13 +47,13 @@ main() {
     elif [ $HAS_APK ]; then
         $SUDO apk add python3-dev gcc musl-dev linux-headers coreutils procps
     elif [ $FREEBSD ]; then
-        $SUDO pkg install -y gmake python3 gcc
+        $SUDO pkg install -y python3 gcc
     elif [ $NETBSD ]; then
         $SUDO /usr/sbin/pkg_add -v pkgin
         $SUDO pkgin update
-        $SUDO pkgin -y install gmake python311-* gcc12-*
+        $SUDO pkgin -y install python311-* gcc12-*
     elif [ $OPENBSD ]; then
-        $SUDO pkg_add gmake gcc python3
+        $SUDO pkg_add gcc python3
     else
         echo "Unsupported platform: $UNAME_S"
     fi