Add session-level file cache, with data_loader_max_cache_bytes and data_loader_max_open_files INI options

README.md

@@ -423,7 +423,6 @@ def test_env_specific_cases(data_loader: DataLoaderFixture, env: str, filename:
 
 
 
-
 ## Lazy Loading
 
 Lazy loading is enabled by default for all data loaders to improve efficiency, especially with large datasets. During 
@@ -435,7 +434,7 @@ If you need to disable this behavior for a specific test, pass `lazy_loading=Fal
 > Lazy loading for the `@parametrize` loader works slightly differently from other loaders. Since pytest needs to know 
 > the total number of parameters in advance, the plugin still needs to inspect the file data and split it once during 
 > the test collection phase. But once it's done, the split data will not be kept as parameter values and will be loaded 
-> lazily later
+> lazily at test setup
 
 
 
@@ -600,6 +599,25 @@ def test_something2(data):
 
 
 
+## Caching
+
+To reduce repeated I/O and parsing work during a test session, the plugin uses two layers of caching:
+
+- **Session-scoped cache**
+  - Shared across all data loaders
+  - Caches raw file contents by resolved file path
+  - Maintains a bounded pool of open file handles using an LRU strategy
+
+- **Per-loader cache**
+  - Scoped to an individual loader on a test function
+  - Caches the loaded data after the per-file loader pipeline
+  - Reduces repeated parsing and transformation work across parametrized test cases that reuse the same source data
+
+The session-scoped cache can be tuned or disabled via the INI options.
+
+
+
+
 ## Loader Options
 
 Each data loader supports different optional parameters you can use to change how your data is loaded.
@@ -696,3 +714,14 @@ Plugin default: `utf-8`
 >
 > If you use one of these encodings as the default, configure data loaders for known binary formats with
 > `read_options={"mode": "rb"}` to force binary reads.
+
+### `data_loader_max_cache_bytes`
+Maximum cumulative approximate in-memory size of the session-scoped LRU cache for raw file contents. When the total
+cached size exceeds this limit, the least recently used entries are evicted until the cache size falls below the limit.
+Set to `0` to disable raw-content caching entirely.  
+Plugin default: `268435456` (256 MiB)
+
+### `data_loader_max_open_files`
+Maximum number of open file handles retained in the session-scoped file-handle pool. When the pool reaches this limit,
+the least recently used handle is closed before a new handle is added. Set to `0` to disable handle pooling.  
+Plugin default: `64`

src/pytest_data_loader/constants.py

@@ -6,12 +6,18 @@
 from pytest import StashKey
 
 if TYPE_CHECKING:
+    from pytest_data_loader.loaders.cache import SessionFileCache
     from pytest_data_loader.types import DataLoaderOption
 
 
 DEFAULT_LOADER_DIR_NAME = "data"
 DEFAULT_ENCODING = "utf-8"
+DEFAULT_MAX_CACHED_CONTENT_BYTES = 256 * 1024 * 1024  # 256 MiB cumulative raw-content cap
+DEFAULT_MAX_OPEN_FILE_HANDLES = 64  # max simultaneously pooled open file handles
 PYTEST_DATA_LOADER_ATTRS = "_pytest_data_loader_attrs"
 PYTEST_DATA_LOADER_MODULE_CACHE = "_pytest_data_loader_module_cache"
 ROOT_DIR = Path(".").resolve().anchor
+
+# Pytest stash keys
 STASH_KEY_DATA_LOADER_OPTION: StashKey[DataLoaderOption] = StashKey()
+STASH_KEY_FILE_CACHE: StashKey[SessionFileCache] = StashKey()

src/pytest_data_loader/fixtures.py

@@ -10,7 +10,11 @@
 from _pytest.fixtures import FixtureRequest
 from pytest import Config
 
-from pytest_data_loader.constants import PYTEST_DATA_LOADER_MODULE_CACHE, STASH_KEY_DATA_LOADER_OPTION
+from pytest_data_loader.constants import (
+    PYTEST_DATA_LOADER_MODULE_CACHE,
+    STASH_KEY_DATA_LOADER_OPTION,
+    STASH_KEY_FILE_CACHE,
+)
 from pytest_data_loader.exceptions import DataNotFound
 from pytest_data_loader.loaders.impl import create_loaders
 from pytest_data_loader.loaders.loaders import load
@@ -29,6 +33,7 @@
 from pytest_data_loader.validators import validate_loader_func, validate_path, validate_read_options, validate_reader
 
 if TYPE_CHECKING:
+    from pytest_data_loader.loaders.cache import SessionFileCache
     from pytest_data_loader.loaders.impl import Loader
 
 
@@ -44,7 +49,8 @@ def data_loader(request: FixtureRequest, pytestconfig: Config) -> DataLoaderFixt
     This is an alternative to the @load data loader for cases where the file path is only known at test time.
     """
     data_loader_option = pytestconfig.stash[STASH_KEY_DATA_LOADER_OPTION]
-    return DataLoaderFixture(request, data_loader_option)
+    file_cache = pytestconfig.stash[STASH_KEY_FILE_CACHE]
+    return DataLoaderFixture(request, data_loader_option, file_cache=file_cache)
 
 
 class DataLoaderFixture:
@@ -55,14 +61,22 @@ class DataLoaderFixture:
     Repeated calls with the same arguments within a single test return the cached result without re-reading the file.
     """
 
-    def __init__(self, request: FixtureRequest, data_loader_option: DataLoaderOption) -> None:
+    def __init__(
+        self,
+        request: FixtureRequest,
+        data_loader_option: DataLoaderOption,
+        *,
+        file_cache: SessionFileCache | None = None,
+    ) -> None:
         """Initialize the data loader fixture.
 
         :param request: The active fixture request, used to locate the test file and module cache.
         :param data_loader_option: Parsed data loader INI options.
+        :param file_cache: Session-scoped file cache. None disables session caching.
         """
         self._request = request
         self._data_loader_option = data_loader_option
+        self._file_cache = file_cache
         self._search_from = request.path
         self._cache: dict[tuple[Any, ...], Any] = {}
 
@@ -107,7 +121,9 @@ def __call__(
                 onload_func=onload,
             )
 
-            (file_loader,) = create_loaders(validated_path, load_attrs, self._data_loader_option)
+            (file_loader,) = create_loaders(
+                validated_path, load_attrs, self._data_loader_option, file_cache=self._file_cache
+            )
             file_loader.register_cleanup(self._request.module)
             loaded = file_loader.load()
             assert isinstance(loaded, LoadedData)

src/pytest_data_loader/loaders/cache.py

@@ -0,0 +1,162 @@
+from __future__ import annotations
+
+import logging
+import sys
+from collections import OrderedDict
+from collections.abc import Callable
+from typing import IO, Any, Literal
+
+from pytest_data_loader.constants import DEFAULT_MAX_CACHED_CONTENT_BYTES, DEFAULT_MAX_OPEN_FILE_HANDLES
+from pytest_data_loader.types import HashableDict
+
+logger = logging.getLogger(__name__)
+
+# (abs_path, mtime_ns, file_size, read_options)
+CacheKey = tuple[str, int, int, HashableDict]
+# (abs_path, mtime_ns, file_size, effective_encoding)
+ReadModeKey = tuple[str, int, int, str]
+
+
+class SessionFileCache:
+    """Session-scoped cache shared across all FileLoader instances within a pytest session.
+
+    Provides three complementary caches that sit *below* any user transform chain:
+
+    **Raw-content LRU** — caches the str/bytes returned by file reads. mtime_ns and file_size in the key make entries
+    self-invalidating when a file changes during a run. Using the full read_options (not just encoding) ensures
+    loaders with different errors or newline parameters never share a cache entry.  Only plain _read_file() reads
+    are cached. file_reader output and all user transform results are excluded because those callables may be
+    non-idempotent or return generators.
+
+    **Bounded handle pool** — keeps at most MAX_OPEN_FILE_HANDLES open file handles, evicting (and closing) the
+    least-recently-used handle when the pool is full.  Keyed by (abs_path, mtime_ns, file_size, read_options) —
+    the same fields as the content key — so a file changed mid-run maps to a new handle key and the stale FD is evicted
+    automatically.  Each access re-fetches via get_handle() and seeks to the required position, so eviction between
+    accesses is safe (the handle is simply reopened on next access).  Set max_open_handles=0 to disable pooling.
+    FileLoader will fall back to its per-instance handle instead (check pooling_enabled before calling get_handle).
+
+    **Read-mode memo** — caches the detected "r"/"rb" result of the binary-probe open. Eliminates redundant 4 KiB
+    probe opens when multiple FileLoader instances target the same file with the same encoding in the same session.
+
+    Invalidation note: cache keys include mtime_ns and file_size captured when a FileLoader is constructed, so entries
+    are self-invalidating across new loader instances if a file changes between test runs. Within a single reused
+    FileLoader (same parametrized test function, multiple cases) the stat is frozen at first access; mid-session
+    file mutations are not detected for that loader's lifetime.
+
+    NOTE: None of the three caches is thread-safe. Under pytest-xdist each worker is a separate process with its own
+          SessionFileCache instance. The configured byte cap therefore applies per worker, not globally.
+    """
+
+    def __init__(
+        self,
+        max_content_bytes: int = DEFAULT_MAX_CACHED_CONTENT_BYTES,
+        max_open_handles: int = DEFAULT_MAX_OPEN_FILE_HANDLES,
+    ) -> None:
+        """Initialize the session file cache.
+
+        :param max_content_bytes: Cumulative byte cap for the raw-content LRU cache.
+        :param max_open_handles: Maximum number of simultaneously open pooled file handles.
+        """
+        self._max_content_bytes = max_content_bytes
+        self._max_open_handles = max_open_handles
+        # (content, measured byte size) ordered oldest → newest
+        self._content: OrderedDict[CacheKey, tuple[str | bytes, int]] = OrderedDict()
+        self._content_bytes: int = 0
+        # open handles ordered oldest → newest
+        self._handles: OrderedDict[CacheKey, IO[Any]] = OrderedDict()
+        # detected read mode ("r" or "rb") keyed by (abs_path, mtime_ns, file_size, encoding).
+        # Intentionally unbounded: keys are O(bytes) tiny and the number of distinct (file, encoding)
+        # pairs in a session is bounded by the test suite itself.
+        self._read_modes: dict[ReadModeKey, Literal["r", "rb"]] = {}
+
+    @property
+    def pooling_enabled(self) -> bool:
+        """Return True when handle pooling is active (max_open_handles > 0)."""
+        return self._max_open_handles > 0
+
+    def get_content(self, key: CacheKey, on_miss: Callable[[], str | bytes]) -> str | bytes:
+        """Return cached raw file content, calling on_miss on a cache miss.
+
+        The result is stored only if its measured size does not exceed max_content_bytes alone
+        (a single file larger than the cap is returned uncached to avoid evict-thrash).
+        Size is len(data) for binary content and sys.getsizeof(data) for text.
+
+        :param key: Cache key
+        :param on_miss: Zero-argument callable that reads and returns the raw file content.
+        """
+        if key in self._content:
+            self._content.move_to_end(key)
+            return self._content[key][0]
+
+        data = on_miss()
+        # len() for bytes gives exact payload size; sys.getsizeof() for str is an approximation
+        # (includes interpreter overhead, varies with code-point width) — deliberate: exact for
+        # binary, close-enough for text, consistent cap enforcement across both.
+        byte_size = len(data) if isinstance(data, (bytes, bytearray)) else sys.getsizeof(data)
+        if byte_size <= self._max_content_bytes:
+            while self._content and self._content_bytes + byte_size > self._max_content_bytes:
+                _, (_, evicted_size) = self._content.popitem(last=False)
+                self._content_bytes -= evicted_size
+            self._content[key] = (data, byte_size)
+            self._content_bytes += byte_size
+        return data
+
+    def get_handle(self, key: CacheKey, on_miss: Callable[[], IO[Any]]) -> IO[Any]:
+        """Return a pooled open file handle, opening it via opener when missing or closed.
+
+        When the pool is at capacity the least-recently-used handle is closed and removed
+        before inserting the new one.  Callers must seek to the desired position after
+        receiving the handle.
+
+        When max_open_handles is 0 (pooling disabled) the handle is opened via on_miss and
+        returned directly without touching the pool.
+
+        :param key: Cache key
+        :param on_miss: Zero-argument callable that opens and returns a new file handle.
+        """
+        if self._max_open_handles == 0:
+            return on_miss()
+
+        f = self._handles.get(key)
+        if f is not None and not f.closed:
+            self._handles.move_to_end(key)
+            return f
+
+        if key in self._handles:
+            del self._handles[key]
+
+        # New handle opened before evicting the LRU — pool briefly holds max+1 fds.
+        f = on_miss()
+        if len(self._handles) >= self._max_open_handles:
+            _, lru = self._handles.popitem(last=False)
+            try:
+                lru.close()
+            except Exception:
+                logger.exception("Failed to close evicted file handle")
+        self._handles[key] = f
+        return f
+
+    def get_read_mode(self, key: ReadModeKey, on_miss: Callable[[], Literal["r", "rb"]]) -> Literal["r", "rb"]:
+        """Return the cached detected read mode ("r" or "rb"), probing on a miss.
+
+        Eliminates redundant binary-probe opens across FileLoader instances that target the same file and encoding
+        within a session.
+
+        :param key: Cache key
+        :param on_miss: Zero-argument callable that runs the binary probe and returns "r" or "rb".
+        """
+        if key not in self._read_modes:
+            self._read_modes[key] = on_miss()
+        return self._read_modes[key]
+
+    def clear(self) -> None:
+        """Close all pooled handles and drop all cached content and memos. Idempotent."""
+        for f in self._handles.values():
+            try:
+                f.close()
+            except Exception:
+                logger.exception("Failed to close pooled file handle during cache clear")
+        self._handles.clear()
+        self._content.clear()
+        self._content_bytes = 0
+        self._read_modes.clear()