fix #8991 and enhance collection performance

changelog/9125.improvement.rst

@@ -0,0 +1 @@
+Improve collection performance by caching the internal hook proxy objects.

src/_pytest/main.py

@@ -22,6 +22,11 @@
 
 import attr
 
+try:
+    from pluggy.hooks import _HookCaller
+except ImportError:
+    from pluggy._hooks import _HookCaller
+
 import _pytest._code
 from _pytest import nodes
 from _pytest.compat import final
@@ -34,6 +39,7 @@
 from _pytest.config import PytestPluginManager
 from _pytest.config import UsageError
 from _pytest.config.argparsing import Parser
+from _pytest.config.compat import PathAwareHookProxy
 from _pytest.fixtures import FixtureManager
 from _pytest.outcomes import exit
 from _pytest.pathlib import absolutepath
@@ -478,6 +484,12 @@ def __init__(self, config: Config) -> None:
         self.trace = config.trace.root.get("collection")
         self._initialpaths: FrozenSet[Path] = frozenset()
 
+        # Cache of hookproxy objects, which speeds up collection.
+        # The cache is cleared whenever a new plugin got registered.
+        # See #9125 for a detailed summary of the gains.
+        self._fspath_hookproxy_cache: Dict[
+            os.PathLike[str], Union[_HookCaller, PathAwareHookProxy]
+        ] = {}
         self._bestrelpathcache: Dict[Path, str] = _bestrelpath_cache(config.rootpath)
 
         self.config.pluginmanager.register(self, name="session")
@@ -542,12 +554,20 @@ def isinitpath(self, path: Union[str, "os.PathLike[str]"]) -> bool:
         path_ = path if isinstance(path, Path) else Path(path)
         return path_ in self._initialpaths
 
-    def gethookproxy(self, fspath: "os.PathLike[str]"):
+    def gethookproxy(
+        self, fspath: "os.PathLike[str]"
+    ) -> Union[_HookCaller, PathAwareHookProxy]:
         # Optimization: Path(Path(...)) is much slower than isinstance.
         path = fspath if isinstance(fspath, Path) else Path(fspath)
-        pm = self.config.pluginmanager
         # Check if we have the common case of running
         # hooks with all conftest.py files.
+
+        pm = self.config.pluginmanager
+        proxy: Optional[PathAwareHookProxy] = self._fspath_hookproxy_cache.get(fspath)
+
+        if proxy is not None:
+            return proxy
+
         my_conftestmodules = pm._getconftestmodules(
             path,
             self.config.getoption("importmode"),
@@ -556,14 +576,16 @@ def gethookproxy(self, fspath: "os.PathLike[str]"):
         remove_mods = pm._conftest_plugins.difference(my_conftestmodules)
         if remove_mods:
             # One or more conftests are not in use at this fspath.
-            from .config.compat import PathAwareHookProxy
-
             proxy = PathAwareHookProxy(FSHookProxy(pm, remove_mods))
         else:
             # All plugins are active for this fspath.
             proxy = self.config.hook
+        self._fspath_hookproxy_cache[fspath] = proxy
         return proxy
 
+    def pytest_plugin_registered(self, plugin):
+        self._fspath_hookproxy_cache.clear()
+
     def _recurse(self, direntry: "os.DirEntry[str]") -> bool:
         if direntry.name == "__pycache__":
             return False