Add [testenv] constraints option (#3556)

* trivial: Dedent function

Signed-off-by: Stephen Finucane <stephen@that.guru>

* trivial: Address TODO

Signed-off-by: Stephen Finucane <stephen@that.guru>

* Remove unnecessary dependency

The minimum version is Python 3.9 now.

Signed-off-by: Stephen Finucane <stephen@that.guru>

* Add '[testenv] constraints' setting

This allows users to override constraints files for all installations.
Without this, it is not possible to use constraints files that include
the package under test without introducing a separate pre-tox processing
step to scrub said package from the constraints file.

Signed-off-by: Stephen Finucane <stephen@that.guru>

---------

Signed-off-by: Stephen Finucane <stephen@that.guru>

Author: stephenfin

docs/changelog/3350.feature.rst

@@ -0,0 +1 @@
+Added ``constraints`` to allow specifying constraints files for all dependencies.

docs/config.rst

@@ -942,15 +942,15 @@ Python run
    :keys: deps
    :default: <empty list>
 
-   Name of the Python dependencies. Installed into the environment prior to project after environment creation, but
+   Python dependencies. Installed into the environment prior to project after environment creation, but
    before package installation. All installer commands are executed using the :ref:`tox_root` as the current working
    directory. Each value must be one of:
 
    - a Python dependency as specified by :pep:`440`,
    - a `requirement file <https://pip.pypa.io/en/stable/user_guide/#requirements-files>`_ when the value starts with
-     ``-r`` (followed by a file path),
+     ``-r`` (followed by a file path or URL),
    - a `constraint file <https://pip.pypa.io/en/stable/user_guide/#constraints-files>`_ when the value starts with
-     ``-c`` (followed by a file path).
+     ``-c`` (followed by a file path or URL).
 
    If you are only defining :pep:`508` requirements (aka no pip requirement files), you should use
    :ref:`dependency_groups` instead.
@@ -977,6 +977,21 @@ Python run
             -r requirements.txt
             -c constraints.txt
 
+   .. note::
+
+      :ref:`constraints` is the preferred way to specify constraints files since they will apply to package dependencies
+      also.
+
+.. conf::
+   :keys: constraints
+   :default: <empty list>
+   :version_added: 4.28.0
+
+   `Constraints files <https://pip.pypa.io/en/stable/user_guide/#constraints-files>`_ to use during package and
+   dependency installation. Provided constraints files will be used when installing package dependencies and any
+   additional dependencies specified in :ref:`deps`, but will not be used when installing the package itself.
+   Each value must be a file path or URL.
+
 .. conf::
    :keys: use_develop, usedevelop
    :default: false
@@ -1210,7 +1225,6 @@ Pip installer
    This command will be executed only if executing on Continuous Integrations is detected (for example set environment
    variable ``CI=1``) or if journal is active.
 
-
 .. conf::
    :keys: pip_pre
    :default: false
@@ -1227,7 +1241,7 @@ Pip installer
 
    If ``constrain_package_deps`` is true, then tox will create and use ``{env_dir}{/}constraints.txt`` when installing
    package dependencies during ``install_package_deps`` stage. When this value is set to false, any conflicting package
-   dependencies will override explicit dependencies and constraints passed to ``deps``.
+   dependencies will override explicit dependencies and constraints passed to :ref:`deps`.
 
 .. conf::
    :keys: use_frozen_constraints

pyproject.toml

@@ -97,8 +97,7 @@ test = [
   "pytest-mock>=3.14",
   "pytest-xdist>=3.6.1",
   "re-assert>=1.1",
-  "setuptools>=75.3; python_version<='3.8'",
-  "setuptools>=75.8; python_version>'3.8'",
+  "setuptools>=75.8",
   "time-machine>=2.15; implementation_name!='pypy'",
   "wheel>=0.45.1",
 ]

src/tox/tox.schema.json

@@ -289,7 +289,14 @@
         },
         "deps": {
           "type": "string",
-          "description": "Name of the python dependencies as specified by PEP-440"
+          "description": "python dependencies with optional version specifiers, as specified by PEP-440"
+        },
+        "constraints": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "constraints to apply to installed python dependencies"
         },
         "dependency_groups": {
           "type": "array",

src/tox/tox_env/python/pip/pip_install.py

@@ -4,8 +4,9 @@
 import operator
 from abc import ABC, abstractmethod
 from collections import defaultdict
+from functools import partial
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Sequence
+from typing import TYPE_CHECKING, Any, Callable, Sequence, cast
 
 from packaging.requirements import Requirement
 
@@ -15,7 +16,7 @@
 from tox.tox_env.installer import Installer
 from tox.tox_env.python.api import Python
 from tox.tox_env.python.package import EditableLegacyPackage, EditablePackage, SdistPackage, WheelPackage
-from tox.tox_env.python.pip.req_file import PythonDeps
+from tox.tox_env.python.pip.req_file import PythonConstraints, PythonDeps
 
 if TYPE_CHECKING:
     from tox.config.main import Config
@@ -52,6 +53,7 @@ class Pip(PythonInstallerListDependencies):
 
     def _register_config(self) -> None:
         super()._register_config()
+        root = self._env.core["toxinidir"]
         self._env.conf.add_config(
             keys=["pip_pre"],
             of_type=bool,
@@ -65,6 +67,13 @@ def _register_config(self) -> None:
             post_process=self.post_process_install_command,
             desc="command used to install packages",
         )
+        self._env.conf.add_config(
+            keys=["constraints"],
+            of_type=PythonConstraints,
+            factory=partial(PythonConstraints.factory, root),
+            default=PythonConstraints("", root),
+            desc="constraints to apply to installed python dependencies",
+        )
         self._env.conf.add_config(
             keys=["constrain_package_deps"],
             of_type=bool,
@@ -110,6 +119,10 @@ def install(self, arguments: Any, section: str, of_type: str) -> None:
             logging.warning("pip cannot install %r", arguments)
             raise SystemExit(1)
 
+    @property
+    def constraints(self) -> PythonConstraints:
+        return cast("PythonConstraints", self._env.conf["constraints"])
+
     def constraints_file(self) -> Path:
         return Path(self._env.env_dir) / "constraints.txt"
 
@@ -121,16 +134,25 @@ def constrain_package_deps(self) -> bool:
     def use_frozen_constraints(self) -> bool:
         return bool(self._env.conf["use_frozen_constraints"])
 
-    def _install_requirement_file(self, arguments: PythonDeps, section: str, of_type: str) -> None:  # noqa: C901
+    def _install_requirement_file(self, arguments: PythonDeps, section: str, of_type: str) -> None:
+        new_requirements: list[str] = []
+        new_constraints: list[str] = []
+
         try:
             new_options, new_reqs = arguments.unroll()
         except ValueError as exception:
             msg = f"{exception} for tox env py within deps"
             raise Fail(msg) from exception
-        new_requirements: list[str] = []
-        new_constraints: list[str] = []
         for req in new_reqs:
             (new_constraints if req.startswith("-c ") else new_requirements).append(req)
+
+        try:
+            _, new_reqs = self.constraints.unroll()
+        except ValueError as exception:
+            msg = f"{exception} for tox env py within constraints"
+            raise Fail(msg) from exception
+        new_constraints.extend(new_reqs)
+
         constraint_options = {
             "constrain_package_deps": self.constrain_package_deps,
             "use_frozen_constraints": self.use_frozen_constraints,
@@ -159,17 +181,10 @@ def _install_requirement_file(self, arguments: PythonDeps, section: str, of_type
                         raise Recreate(msg)
                 args = arguments.as_root_args
                 if args:  # pragma: no branch
+                    args.extend(self.constraints.as_root_args)
                     self._execute_installer(args, of_type)
                     if self.constrain_package_deps and not self.use_frozen_constraints:
-                        # when we drop Python 3.8 we can use the builtin `.removeprefix`
-                        def remove_prefix(text: str, prefix: str) -> str:
-                            if text.startswith(prefix):
-                                return text[len(prefix) :]
-                            return text
-
-                        combined_constraints = new_requirements + [
-                            remove_prefix(text=c, prefix="-c ") for c in new_constraints
-                        ]
+                        combined_constraints = new_requirements + [c.removeprefix("-c ") for c in new_constraints]
                         self.constraints_file().write_text("\n".join(combined_constraints))
 
     @staticmethod
@@ -215,13 +230,18 @@ def _install_list_of_deps(  # noqa: C901
                     raise Recreate(msg)  # pragma: no branch
                 new_deps = sorted(set(groups["req"]) - set(old or []))
                 if new_deps:  # pragma: no branch
+                    new_deps.extend(self.constraints.as_root_args)
                     self._execute_installer(new_deps, req_of_type)
         install_args = ["--force-reinstall", "--no-deps"]
         if groups["pkg"]:
+            # we intentionally ignore constraints when installing the package itself
+            # https://github.com/tox-dev/tox/issues/3550
             self._execute_installer(install_args + groups["pkg"], of_type)
         if groups["dev_pkg"]:
             for entry in groups["dev_pkg"]:
                 install_args.extend(("-e", str(entry)))
+            # we intentionally ignore constraints when installing the package itself
+            # https://github.com/tox-dev/tox/issues/3550
             self._execute_installer(install_args, of_type)
 
     def _execute_installer(self, deps: Sequence[Any], of_type: str) -> None:

src/tox/tox_env/python/pip/req_file.py

@@ -56,26 +56,26 @@ def _is_url_self(self, url: str) -> bool:
     def _pre_process(self, content: str) -> ReqFileLines:
         for at, line in super()._pre_process(content):
             if line.startswith("-r") or (line.startswith("-c") and line[2].isalpha()):
-                found_line = f"{line[0:2]} {line[2:]}"
+                found_line = f"{line[0:2]} {line[2:]}"  # normalize
             else:
                 found_line = line
             yield at, found_line
 
     def lines(self) -> list[str]:
         return self._raw.splitlines()
 
-    @staticmethod
-    def _normalize_raw(raw: str) -> str:
+    @classmethod
+    def _normalize_raw(cls, raw: str) -> str:
         # a line ending in an unescaped \ is treated as a line continuation and the newline following it is effectively
         # ignored
         raw = "".join(raw.replace("\r", "").split("\\\n"))
         # for tox<4 supporting requirement/constraint files via -rreq.txt/-creq.txt
-        lines: list[str] = [PythonDeps._normalize_line(line) for line in raw.splitlines()]
+        lines: list[str] = [cls._normalize_line(line) for line in raw.splitlines()]
         adjusted = "\n".join(lines)
         return f"{adjusted}\n" if raw.endswith("\\\n") else adjusted  # preserve trailing newline if input has it
 
-    @staticmethod
-    def _normalize_line(line: str) -> str:
+    @classmethod
+    def _normalize_line(cls, line: str) -> str:
         arg_match = next(
             (
                 arg
@@ -138,6 +138,113 @@ def factory(cls, root: Path, raw: object) -> PythonDeps:
         return cls(raw, root)
 
 
+class PythonConstraints(RequirementsFile):
+    def __init__(self, raw: str | list[str] | list[Requirement], root: Path) -> None:
+        super().__init__(root / "tox.ini", constraint=True)
+        got = raw if isinstance(raw, str) else "\n".join(str(i) for i in raw)
+        self._raw = self._normalize_raw(got)
+        self._unroll: tuple[list[str], list[str]] | None = None
+        self._req_parser_: RequirementsFile | None = None
+
+    @property
+    def _req_parser(self) -> RequirementsFile:
+        if self._req_parser_ is None:
+            self._req_parser_ = RequirementsFile(path=self._path, constraint=True)
+        return self._req_parser_
+
+    def _get_file_content(self, url: str) -> str:
+        if self._is_url_self(url):
+            return self._raw
+        return super()._get_file_content(url)
+
+    def _is_url_self(self, url: str) -> bool:
+        return url == str(self._path)
+
+    def _pre_process(self, content: str) -> ReqFileLines:
+        for at, line in super()._pre_process(content):
+            if line.startswith("-r") or (line.startswith("-c") and line[2].isalpha()):
+                found_line = f"{line[0:2]} {line[2:]}"  # normalize
+            else:
+                found_line = line
+            yield at, found_line
+
+    def lines(self) -> list[str]:
+        return self._raw.splitlines()
+
+    @classmethod
+    def _normalize_raw(cls, raw: str) -> str:
+        # a line ending in an unescaped \ is treated as a line continuation and the newline following it is effectively
+        # ignored
+        raw = "".join(raw.replace("\r", "").split("\\\n"))
+        # for tox<4 supporting requirement/constraint files via -rreq.txt/-creq.txt
+        lines: list[str] = [cls._normalize_line(line) for line in raw.splitlines()]
+
+        if any(line.startswith("-") for line in lines):
+            msg = "only constraints files or URLs can be provided"
+            raise ValueError(msg)
+
+        adjusted = "\n".join([f"-c {line}" for line in lines])
+        return f"{adjusted}\n" if raw.endswith("\\\n") else adjusted  # preserve trailing newline if input has it
+
+    @classmethod
+    def _normalize_line(cls, line: str) -> str:
+        arg_match = next(
+            (
+                arg
+                for arg in ONE_ARG
+                if line.startswith(arg)
+                and len(line) > len(arg)
+                and not (line[len(arg)].isspace() or line[len(arg)] == "=")
+            ),
+            None,
+        )
+        if arg_match is not None:
+            values = line[len(arg_match) :]
+            line = f"{arg_match} {values}"
+        # escape spaces
+        escape_match = next((e for e in ONE_ARG_ESCAPE if line.startswith(e) and line[len(e)].isspace()), None)
+        if escape_match is not None:
+            # escape not already escaped spaces
+            escaped = re.sub(r"(?<!\\)(\s)", r"\\\1", line[len(escape_match) + 1 :])
+            line = f"{line[: len(escape_match)]} {escaped}"
+        return line
+
+    def _parse_requirements(self, opt: Namespace, recurse: bool) -> list[ParsedRequirement]:  # noqa: FBT001
+        # check for any invalid options in the deps list
+        # (requirements recursively included from other files are not checked)
+        requirements = super()._parse_requirements(opt, recurse)
+        for req in requirements:
+            if req.from_file != str(self.path):
+                continue
+            if req.options:
+                msg = f"Cannot provide options in constraints list, only paths or URL can be provided. ({req})"
+                raise ValueError(msg)
+        return requirements
+
+    def unroll(self) -> tuple[list[str], list[str]]:
+        if self._unroll is None:
+            opts_dict = vars(self.options)
+            if not self.requirements and opts_dict:
+                msg = "no dependencies"
+                raise ValueError(msg)
+            result_opts: list[str] = [f"{key}={value}" for key, value in opts_dict.items()]
+            result_req = [str(req) for req in self.requirements]
+            self._unroll = result_opts, result_req
+        return self._unroll
+
+    @classmethod
+    def factory(cls, root: Path, raw: object) -> PythonConstraints:
+        if not (
+            isinstance(raw, str)
+            or (
+                isinstance(raw, list)
+                and (all(isinstance(i, str) for i in raw) or all(isinstance(i, Requirement) for i in raw))
+            )
+        ):
+            raise TypeError(raw)
+        return cls(raw, root)
+
+
 ONE_ARG = {
     "-i",
     "--index-url",

src/tox/tox_env/python/runner.py

@@ -34,11 +34,11 @@ def register_config(self) -> None:
         super().register_config()
         root = self.core["toxinidir"]
         self.conf.add_config(
-            keys="deps",
+            keys=["deps"],
             of_type=PythonDeps,
             factory=partial(PythonDeps.factory, root),
             default=PythonDeps("", root),
-            desc="Name of the python dependencies as specified by PEP-440",
+            desc="python dependencies with optional version specifiers, as specified by PEP-440",
         )
         self.conf.add_config(
             keys=["dependency_groups"],