Refactor lazy caching (#9361)

* changes

* lazy

* redo lazy

* add changeset

* changes

* helpers

* docstrings'

* lint

* Update guides/04_additional-features/09_environment-variables.md

Co-authored-by: Charles <charles@huggingface.co>

* Update gradio/chat_interface.py

Co-authored-by: Dawood Khan <dawoodkhan82@gmail.com>

* Update gradio/chat_interface.py

Co-authored-by: Dawood Khan <dawoodkhan82@gmail.com>

* tolerant

---------

Co-authored-by: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com>
Co-authored-by: Charles <charles@huggingface.co>
Co-authored-by: Dawood Khan <dawoodkhan82@gmail.com>

.changeset/young-ears-vanish.md

@@ -0,0 +1,5 @@
+---
+"gradio": minor
+---
+
+feat:Refactor lazy caching

gradio/chat_interface.py

@@ -67,7 +67,8 @@ def __init__(
         additional_inputs: str | Component | list[str | Component] | None = None,
         additional_inputs_accordion: str | Accordion | None = None,
         examples: list[str] | list[dict[str, str | list]] | list[list] | None = None,
-        cache_examples: bool | Literal["lazy"] | None = None,
+        cache_examples: bool | None = None,
+        cache_mode: Literal["eager", "lazy"] | None = None,
         examples_per_page: int = 10,
         title: str | None = None,
         description: str | None = None,
@@ -95,7 +96,8 @@ def __init__(
             additional_inputs: an instance or list of instances of gradio components (or their string shortcuts) to use as additional inputs to the chatbot. If components are not already rendered in a surrounding Blocks, then the components will be displayed under the chatbot, in an accordion.
             additional_inputs_accordion: if a string is provided, this is the label of the `gr.Accordion` to use to contain additional inputs. A `gr.Accordion` object can be provided as well to configure other properties of the container holding the additional inputs. Defaults to a `gr.Accordion(label="Additional Inputs", open=False)`. This parameter is only used if `additional_inputs` is provided.
             examples: sample inputs for the function; if provided, appear below the chatbot and can be clicked to populate the chatbot input. Should be a list of strings if `multimodal` is False, and a list of dictionaries (with keys `text` and `files`) if `multimodal` is True.
-            cache_examples: if True, caches examples in the server for fast runtime in examples. The default option in HuggingFace Spaces is True. The default option elsewhere is False.
+            cache_examples: If True, caches examples in the server for fast runtime in examples. If None, will use the GRADIO_CACHE_EXAMPLES environment variable, which should be either "true" or "false". In HuggingFace Spaces, this parameter is True (as long as `fn` and `outputs` are also provided). The default option otherwise is False.
+            cache_mode: If "lazy", then examples are cached (for all users of the app) after their first use (by any user of the app). If "eager", all examples are cached at app launch. If None, will use the GRADIO_CACHE_MODE environment variable if defined, or default to "eager".
             examples_per_page: if examples are provided, how many to display per page.
             title: a title for the interface; if provided, appears above chatbot in large font. Also used as the tab title when opened in a browser window.
             description: a description for the interface; if provided, appears above the chatbot and beneath the title in regular font. Accepts Markdown and HTML content.
@@ -138,6 +140,7 @@ def __init__(
 
         self.examples = examples
         self.cache_examples = cache_examples
+        self.cache_mode = cache_mode
 
         if additional_inputs:
             if not isinstance(additional_inputs, list):
@@ -237,6 +240,7 @@ def __init__(
                     outputs=self.chatbot,
                     fn=examples_fn,
                     cache_examples=self.cache_examples,
+                    cache_mode=self.cache_mode,
                     _defer_caching=True,
                     examples_per_page=examples_per_page,
                 )

gradio/helpers.py

@@ -38,7 +38,8 @@ def create_examples(
     inputs: Component | Sequence[Component],
     outputs: Component | Sequence[Component] | None = None,
     fn: Callable | None = None,
-    cache_examples: bool | Literal["lazy"] | None = None,
+    cache_examples: bool | None = None,
+    cache_mode: Literal["eager", "lazy"] | None = None,
     examples_per_page: int = 10,
     _api_mode: bool = False,
     label: str | None = None,
@@ -60,6 +61,7 @@ def create_examples(
         outputs=outputs,
         fn=fn,
         cache_examples=cache_examples,
+        cache_mode=cache_mode,
         examples_per_page=examples_per_page,
         _api_mode=_api_mode,
         label=label,
@@ -96,7 +98,8 @@ def __init__(
         inputs: Component | Sequence[Component],
         outputs: Component | Sequence[Component] | None = None,
         fn: Callable | None = None,
-        cache_examples: bool | Literal["lazy"] | None = None,
+        cache_examples: bool | None = None,
+        cache_mode: Literal["eager", "lazy"] | None = None,
         examples_per_page: int = 10,
         _api_mode: bool = False,
         label: str | None = "Examples",
@@ -118,7 +121,8 @@ def __init__(
             inputs: the component or list of components corresponding to the examples
             outputs: optionally, provide the component or list of components corresponding to the output of the examples. Required if `cache_examples` is not False.
             fn: optionally, provide the function to run to generate the outputs corresponding to the examples. Required if `cache_examples` is not False. Also required if `run_on_click` is True.
-            cache_examples: If True, caches examples in the server for fast runtime in examples. If "lazy", then examples are cached after their first use. Can also be set by the GRADIO_CACHE_EXAMPLES environment variable, which takes a case-insensitive value, one of: {"true", "lazy", or "false"} (for the first two to take effect, `fn` and `outputs` should also be provided). In HuggingFace Spaces, this is True (as long as `fn` and `outputs` are also provided). The default option otherwise is False.
+            cache_examples: If True, caches examples in the server for fast runtime in examples. If "lazy", then examples are cached (for all users of the app) after their first use (by any user of the app). If None, will use the GRADIO_CACHE_EXAMPLES environment variable, which should be either "true" or "false". In HuggingFace Spaces, this parameter is True (as long as `fn` and `outputs` are also provided). The default option otherwise is False.
+            cache_mode: if "lazy", examples are cached after their first use. If "eager", all examples are cached at app launch. If None, will use the GRADIO_CACHE_MODE environment variable if defined, or default to "eager".
             examples_per_page: how many examples to show per page.
             label: the label to use for the examples component (by default, "Examples")
             elem_id: an optional string that is assigned as the id of this component in the HTML DOM.
@@ -142,30 +146,35 @@ def __init__(
                         self.cache_examples = True
                     else:
                         self.cache_examples = False
-                elif cache_examples_env.lower() == "lazy":
-                    if fn is not None and outputs is not None:
-                        self.cache_examples = "lazy"
-                    else:
-                        self.cache_examples = False
-                elif cache_examples_env.lower() == "false":
-                    self.cache_examples = False
-                else:
-                    raise ValueError(
-                        "The `GRADIO_CACHE_EXAMPLES` env variable must be one of: 'true', 'false', 'lazy' (case-insensitive)."
-                    )
             elif utils.get_space() and fn is not None and outputs is not None:
                 self.cache_examples = True
             else:
                 self.cache_examples = cache_examples or False
         else:
-            if cache_examples not in [True, False, "lazy"]:
+            if cache_examples not in [True, False]:
                 raise ValueError(
-                    "The `cache_examples` parameter must be one of: True, False, 'lazy'."
+                    "The `cache_examples` parameter must either: True or False."
                 )
             self.cache_examples = cache_examples
 
         if self.cache_examples and (fn is None or outputs is None):
             raise ValueError("If caching examples, `fn` and `outputs` must be provided")
+
+        if (cache_mode_env := os.getenv("GRADIO_CACHE_MODE")) and cache_mode is None:
+            if cache_mode_env.lower() == "eager":
+                cache_mode = "eager"
+            elif cache_mode_env.lower() == "lazy":
+                cache_mode = "lazy"
+            else:
+                cache_mode = "eager"
+                warnings.warn(
+                    "The `GRADIO_CACHE_MODE` environment variable must be either 'eager' or 'lazy'. "
+                    "Defaulting to 'eager'."
+                )
+
+        if self.cache_examples and cache_mode == "lazy":
+            self.cache_examples = "lazy"
+
         self._defer_caching = _defer_caching
 
         if not isinstance(inputs, Sequence):

gradio/interface.py

@@ -101,7 +101,8 @@ def __init__(
         outputs: str | Component | Sequence[str | Component] | None,
         examples: list[Any] | list[list[Any]] | str | None = None,
         *,
-        cache_examples: bool | Literal["lazy"] | None = None,
+        cache_examples: bool | None = None,
+        cache_mode: Literal["eager", "lazy"] | None = None,
         examples_per_page: int = 10,
         example_labels: list[str] | None = None,
         live: bool = False,
@@ -146,7 +147,8 @@ def __init__(
             inputs: a single Gradio component, or list of Gradio components. Components can either be passed as instantiated objects, or referred to by their string shortcuts. The number of input components should match the number of parameters in fn. If set to None, then only the output components will be displayed.
             outputs: a single Gradio component, or list of Gradio components. Components can either be passed as instantiated objects, or referred to by their string shortcuts. The number of output components should match the number of values returned by fn. If set to None, then only the input components will be displayed.
             examples: sample inputs for the function; if provided, appear below the UI components and can be clicked to populate the interface. Should be nested list, in which the outer list consists of samples and each inner list consists of an input corresponding to each input component. A string path to a directory of examples can also be provided, but it should be within the directory with the python file running the gradio app. If there are multiple input components and a directory is provided, a log.csv file must be present in the directory to link corresponding inputs.
-            cache_examples: if True, caches examples in the server for fast runtime in examples. If "lazy", then examples are cached after their first use. If `fn` is a generator function, then the last yielded value will be used as the output. Can also be set by the GRADIO_CACHE_EXAMPLES environment variable, which takes a case-insensitive value, one of: {"true", "false", "lazy"}. The default option in HuggingFace Spaces is True. The default option elsewhere is False.
+            cache_examples: If True, caches examples in the server for fast runtime in examples. If "lazy", then examples are cached (for all users of the app) after their first use (by any user of the app). If None, will use the GRADIO_CACHE_EXAMPLES environment variable, which should be either "true" or "false". In HuggingFace Spaces, this parameter is True (as long as `fn` and `outputs` are also provided). The default option otherwise is False.
+            cache_mode: if "lazy", examples are cached after their first use. If "eager", all examples are cached at app launch. If None, will use the GRADIO_CACHE_MODE environment variable if defined, or default to "eager".
             examples_per_page: if examples are provided, how many to display per page.
             live: whether the interface should automatically rerun if any of the inputs change.
             title: a title for the interface; if provided, appears above the input and output components in large font. Also used as the tab title when opened in a browser window.
@@ -222,6 +224,7 @@ def __init__(
             outputs = [outputs]
 
         self.cache_examples = cache_examples
+        self.cache_mode: Literal["eager", "lazy"] | None = cache_mode
 
         state_input_indexes = [
             idx for idx, i in enumerate(inputs) if i == "state" or isinstance(i, State)
@@ -875,10 +878,11 @@ def render_examples(self):
             ]
             self.examples_handler = Examples(
                 examples=self.examples,
-                inputs=non_state_inputs,  # type: ignore
-                outputs=non_state_outputs,  # type: ignore
+                inputs=non_state_inputs,
+                outputs=non_state_outputs,
                 fn=self.fn,
-                cache_examples=self.cache_examples,  # type: ignore
+                cache_examples=self.cache_examples,
+                cache_mode=self.cache_mode,
                 examples_per_page=self.examples_per_page,
                 _api_mode=self.api_mode,
                 batch=self.batch,

guides/04_additional-features/09_environment-variables.md

@@ -106,16 +106,38 @@ Environment variables in Gradio provide a way to customize your applications and
   export FORWARDED_ALLOW_IPS="127.0.0.1,192.168.1.100"
   ```
 
+### 12. `GRADIO_CACHE_EXAMPLES`
+
+- **Description**: Whether or not to cache examples by default in `gr.Interface()`, `gr.ChatInterface()` or in `gr.Examples()` when no explicit argument is passed for the `cache_examples` parameter. You can set this environment variable to either the string "true" or "false".
+- **Default**: `"false"`
+- **Example**:
+  ```sh
+  export GRADIO_CACHE_EXAMPLES="true"
+  ```
+
+
+### 13. `GRADIO_CACHE_MODE`
+
+- **Description**: How to cache examples. Only applies if `cache_examples` is set to `True` either via enviornment variable or by an explicit parameter, AND no no explicit argument is passed for the `cache_mode` parameter in `gr.Interface()`, `gr.ChatInterface()` or in `gr.Examples()`. Can be set to either the strings "lazy" or "eager." If "lazy", examples are cached after their first use for all users of the app. If "eager", all examples are cached at app launch.
+
+- **Default**: `"eager"`
+- **Example**:
+  ```sh
+  export GRADIO_CACHE_MODE="lazy"
+  ```
 
-### 12. `GRADIO_EXAMPLES_CACHE`
 
-- **Description**:  If you set `cache_examples=True` or `cache_examples="lazy"` in `gr.Interface()`, `gr.ChatInterface()` or in `gr.Examples()`, Gradio will run your prediction function and save the results to disk. By default, this is in the `gradio_cached_examples/` subdirectory within your app's working directory. You can customize the location of cached example files created by Gradio by setting the environment variable `GRADIO_EXAMPLES_CACHE` to an absolute path or a path relative to your working directory.
-- **Default**: `"gradio_cached_examples/"`
+### 14. `GRADIO_EXAMPLES_CACHE`
+
+- **Description**:  If you set `cache_examples=True` in `gr.Interface()`, `gr.ChatInterface()` or in `gr.Examples()`, Gradio will run your prediction function and save the results to disk. By default, this is in the `.gradio/cached_examples//` subdirectory within your app's working directory. You can customize the location of cached example files created by Gradio by setting the environment variable `GRADIO_EXAMPLES_CACHE` to an absolute path or a path relative to your working directory.
+- **Default**: `".gradio/cached_examples/"`
 - **Example**:
   ```sh
   export GRADIO_EXAMPLES_CACHE="custom_cached_examples/"
   ```
 
+
+
 ## How to Set Environment Variables
 
 To set environment variables in your terminal, use the `export` command followed by the variable name and its value. For example:

test/test_chat_interface.py

@@ -99,7 +99,10 @@ async def test_example_caching_lazy(self):
             "gradio.utils.get_cache_folder", return_value=Path(tempfile.mkdtemp())
         ):
             chatbot = gr.ChatInterface(
-                double, examples=["hello", "hi"], cache_examples="lazy"
+                double,
+                examples=["hello", "hi"],
+                cache_examples=True,
+                cache_mode="lazy",
             )
             async for _ in chatbot.examples_handler.async_lazy_cache(
                 (0, ["hello"]), "hello"

test/test_helpers.py

@@ -618,7 +618,8 @@ def image_identity(image, string):
                 inputs=[i1, t],
                 outputs=[i2],
                 fn=image_identity,
-                cache_examples="lazy",
+                cache_examples=True,
+                cache_mode="lazy",
                 api_name="load_example",
             )