Merge pull request stanfordnlp#1268 from stanfordnlp/backoff_handler_configure

added configurable backoff max time

Author: arnavsinghvi11

docs/docs/faqs.md

@@ -137,7 +137,25 @@ Firstly, please refer to your LM/RM provider to ensure stable status or sufficie
 
 Additionally, try reducing the number of threads you are testing on as the corresponding servers may get overloaded with requests and trigger a backoff + retry mechanism.
 
-If all variables seem stable, you may be experiencing timeouts or backoff errors due to incorrect payload requests sent to the api providers. Please verify your arguments are compatible with the SDK you are interacting with. At times, DSPy may have hard-coded arguments that are not relevant for your compatible, in which case, please free to open a PR alerting this or comment out these default settings for your usage. 
+If all variables seem stable, you may be experiencing timeouts or backoff errors due to incorrect payload requests sent to the api providers. Please verify your arguments are compatible with the SDK you are interacting with. 
+
+You can configure backoff times for your LM/RM provider by setting `dspy.settings.backoff_time` while configuring your DSPy workflow. 
+
+```python
+dspy.settings.configure(backoff_time = ...)
+```
+
+Additionally, if you'd like to set individual backoff times for specific providers, you can do so through the DSPy context manager: 
+
+```python
+with dspy.context(backoff_time = ..):
+      dspy.OpenAI(...) # example
+
+with dspy.context(backoff_time = ..):
+      dspy.AzureOpenAI(...) # example
+```
+
+At times, DSPy may have hard-coded arguments that are not relevant for your compatible, in which case, please free to open a PR alerting this or comment out these default settings for your usage. 
 
 ## Contributing
 

dsp/modules/anthropic.py

@@ -3,6 +3,7 @@
 from typing import Any, Optional
 
 import backoff
+from dsp.utils.settings import settings
 
 from dsp.modules.lm import LM
 
@@ -90,7 +91,7 @@ def basic_request(self, prompt: str, **kwargs):
     @backoff.on_exception(
         backoff.expo,
         (anthropic_rate_limit),
-        max_time=1000,
+        max_time=settings.backoff_time,
         max_tries=8,
         on_backoff=backoff_hdlr,
         giveup=giveup_hdlr,

dsp/modules/aws_providers.py

@@ -4,6 +4,7 @@
 from typing import Any, Optional
 
 import backoff
+from dsp.utils.settings import settings
 
 try:
     import boto3
@@ -80,7 +81,7 @@ def get_provider_name(self) -> str:
     @backoff.on_exception(
         backoff.expo,
         ERRORS,
-        max_time=1000,
+        max_time=settings.backoff_time,
         max_tries=8,
         on_backoff=backoff_hdlr,
         giveup=giveup_hdlr,
@@ -155,7 +156,7 @@ def __init__(
     @backoff.on_exception(
         backoff.expo,
         ERRORS,
-        max_time=1000,
+        max_time=settings.backoff_time,
         max_tries=8,
         on_backoff=backoff_hdlr,
         giveup=giveup_hdlr,

dsp/modules/azure_openai.py

@@ -8,6 +8,8 @@
 
 from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_on
 from dsp.modules.lm import LM
+from dsp.utils.settings import settings
+
 
 try:
     OPENAI_LEGACY = int(openai.version.__version__[0]) == 0
@@ -166,7 +168,7 @@ def basic_request(self, prompt: str, **kwargs):
     @backoff.on_exception(
         backoff.expo,
         ERRORS,
-        max_time=1000,
+        max_time=settings.backoff_time,
         on_backoff=backoff_hdlr,
     )
     def request(self, prompt: str, **kwargs):

dsp/modules/cloudflare.py

@@ -7,6 +7,7 @@
 from pydantic import BaseModel, ValidationError
 
 from dsp.modules.lm import LM
+from dsp.utils.settings import settings
 
 
 def backoff_hdlr(details) -> None:
@@ -69,7 +70,7 @@ def __init__(
     @backoff.on_exception(
         backoff.expo,
         requests.exceptions.RequestException,
-        max_time=1000,
+        max_time=settings.backoff_time,
         on_backoff=backoff_hdlr,
         on_giveup=giveup_hdlr,
     )

dsp/modules/cohere.py

@@ -3,6 +3,7 @@
 import backoff
 
 from dsp.modules.lm import LM
+from dsp.utils.settings import settings
 
 try:
     import cohere
@@ -101,7 +102,7 @@ def basic_request(self, prompt: str, **kwargs):
     @backoff.on_exception(
         backoff.expo,
         (cohere_api_error),
-        max_time=1000,
+        max_time=settings.backoff_time,
         on_backoff=backoff_hdlr,
         giveup=giveup_hdlr,
     )

dsp/modules/google.py

@@ -5,6 +5,7 @@
 import backoff
 
 from dsp.modules.lm import LM
+from dsp.utils.settings import settings
 
 try:
     import google.generativeai as genai
@@ -133,7 +134,7 @@ def basic_request(self, prompt: str, **kwargs):
     @backoff.on_exception(
         backoff.expo,
         (google_api_error),
-        max_time=1000,
+        max_time=settings.backoff_time,
         max_tries=8,
         on_backoff=backoff_hdlr,
         giveup=giveup_hdlr,

dsp/modules/googlevertexai.py

@@ -6,6 +6,7 @@
 from pydantic_core import PydanticCustomError
 
 from dsp.modules.lm import LM
+from dsp.utils.settings import settings
 
 try:
     import vertexai  # type: ignore[import-untyped]
@@ -184,7 +185,7 @@ def basic_request(self, prompt: str, **kwargs):
     @backoff.on_exception(
         backoff.expo,
         (Exception),
-        max_time=1000,
+        max_time=settings.backoff_time,
         on_backoff=backoff_hdlr,
         giveup=giveup_hdlr,
     )

dsp/modules/gpt3.py

@@ -8,6 +8,7 @@
 
 from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_on
 from dsp.modules.lm import LM
+from dsp.utils.settings import settings
 
 try:
     OPENAI_LEGACY = int(openai.version.__version__[0]) == 0
@@ -133,11 +134,11 @@ def basic_request(self, prompt: str, **kwargs):
     @backoff.on_exception(
         backoff.expo,
         ERRORS,
-        max_time=1000,
+        max_time=settings.backoff_time,
         on_backoff=backoff_hdlr,
     )
     def request(self, prompt: str, **kwargs):
-        """Handles retreival of GPT-3 completions whilst handling rate limiting and caching."""
+        """Handles retrieval of GPT-3 completions whilst handling rate limiting and caching."""
         if "model_type" in kwargs:
             del kwargs["model_type"]
 

dsp/modules/groq_client.py

@@ -13,6 +13,7 @@
 
 
 from dsp.modules.lm import LM
+from dsp.utils.settings import settings
 
 
 def backoff_hdlr(details):
@@ -91,7 +92,7 @@ def basic_request(self, prompt: str, **kwargs):
     @backoff.on_exception(
         backoff.expo,
         groq_api_error,
-        max_time=1000,
+        max_time=settings.backoff_time,
         on_backoff=backoff_hdlr,
     )
     def request(self, prompt: str, **kwargs):

dsp/modules/hf_client.py

@@ -11,6 +11,7 @@
 
 from dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory
 from dsp.modules.hf import HFModel, openai_to_hf
+from dsp.utils.settings import settings
 
 ERRORS = (Exception)
 
@@ -340,7 +341,7 @@ def __init__(self, model, api_base="https://api.together.xyz/v1", api_key=None,
     @backoff.on_exception(
         backoff.expo,
         ERRORS,
-        max_time=1000,
+        max_time=settings.backoff_time,
         on_backoff=backoff_hdlr,
     )
     def _generate(self, prompt, use_chat_api=False, **kwargs):

dsp/modules/mistral.py

@@ -3,6 +3,7 @@
 import backoff
 
 from dsp.modules.lm import LM
+from dsp.utils.settings import settings
 
 try:
     import mistralai
@@ -99,7 +100,7 @@ def basic_request(self, prompt: str, **kwargs):
     @backoff.on_exception(
         backoff.expo,
         (mistralai_api_error),
-        max_time=1000,
+        max_time=settings.backoff_time,
         on_backoff=backoff_hdlr,
         giveup=giveup_hdlr,
     )

dsp/modules/premai.py

@@ -5,6 +5,7 @@
 import backoff
 
 from dsp.modules.lm import LM
+from dsp.utils.settings import settings
 
 try:
     import premai
@@ -179,7 +180,7 @@ def basic_request(self, prompt, **kwargs) -> list[str]:
     @backoff.on_exception(
         backoff.expo,
         (premai_api_error),
-        max_time=1000,
+        max_time=settings.backoff_time,
         on_backoff=backoff_hdlr,
         giveup=giveup_hdlr,
     )

dsp/modules/snowflake.py

@@ -6,6 +6,7 @@
 from pydantic_core import PydanticCustomError
 
 from dsp.modules.lm import LM
+from dsp.utils.settings import settings
 
 try:
     from snowflake.snowpark import Session
@@ -146,7 +147,7 @@ def basic_request(self, prompt: str, **kwargs) -> list:
     @backoff.on_exception(
         backoff.expo,
         (Exception),
-        max_time=1000,
+        max_time=settings.backoff_time,
         on_backoff=backoff_hdlr,
         giveup=giveup_hdlr,
     )

dsp/utils/settings.py

@@ -42,6 +42,7 @@ def __new__(cls):
                 suggest_failures=0,
                 langchain_history=[],
                 experimental=False,
+                backoff_time = 10
             )
             cls._instance.__append(config)
 

dspy/retrieve/chromadb_rm.py

@@ -7,7 +7,7 @@
 import backoff
 import openai
 
-import dspy
+from dsp.utils.settings import settings
 from dsp.utils import dotdict
 
 try:
@@ -124,7 +124,7 @@ def _init_chromadb(
     @backoff.on_exception(
         backoff.expo,
         ERRORS,
-        max_time=15,
+        max_time=settings.backoff_time,
     )
     def _get_embeddings(self, queries: List[str]) -> List[List[float]]:
         """Return query vector after creating embedding using OpenAI

dspy/retrieve/mongodb_atlas_rm.py

@@ -10,7 +10,7 @@
     UnprocessableEntityError,
 )
 
-import dspy
+from dsp.utils.settings import settings
 
 try:
     from pymongo import MongoClient
@@ -61,7 +61,7 @@ def __init__(self, provider: str, model: str):
             RateLimitError,
             UnprocessableEntityError,
         ),
-        max_time=15,
+        max_time=settings.backoff_time,
     )
     def __call__(self, queries) -> Any:
         embedding = self.client.embeddings.create(input=queries, model=self.model)

dspy/retrieve/neo4j_rm.py

@@ -10,7 +10,7 @@
     UnprocessableEntityError,
 )
 
-import dspy
+from dsp.utils.settings import settings
 from dsp.utils import dotdict
 
 try:
@@ -42,7 +42,7 @@ def __init__(self, provider: str, model: str):
             RateLimitError,
             UnprocessableEntityError,
         ),
-        max_time=15,
+        max_time=settings.backoff_time,
     )
     def __call__(self, queries) -> Any:
         embedding = self.client.embeddings.create(input=queries, model=self.model)

dspy/retrieve/pinecone_rm.py

@@ -7,7 +7,7 @@
 
 import backoff
 
-import dspy
+from dsp.utils.settings import settings
 from dsp.utils import dotdict
 
 try:
@@ -178,7 +178,7 @@ def _mean_pooling(
     @backoff.on_exception(
         backoff.expo,
         ERRORS,
-        max_time=15,
+        max_time=settings.backoff_time,
     )
     def _get_embeddings(
         self,