Enable prompts on the Hub (#23662)

* Enable prompts on the Hub

* Update src/transformers/tools/prompts.py

Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com>

* Address review comments

---------

Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com>

docs/source/en/custom_tools.mdx

@@ -414,6 +414,13 @@ of the tools, it has available to it.
 
 </Tip>
 
+In both cases, you can pass a repo ID instead of the prompt template if you would like to use a template hosted by someone in the community. The default prompts live in [this repo](https://huggingface.co/datasets/huggingface-tools/default-prompts) as an example.
+
+To upload your custom prompt on a repo on the Hub and share it with the community just make sure:
+- to use a dataset repository
+- to put the prompt template for the `run` command in a file named `run_prompt_template.txt`
+- to put the prompt template for the `chat` command in a file named `chat_prompt_template.txt`
+
 ## Using custom tools
 
 In this section, we'll be leveraging two existing custom tools that are specific to image generation:

src/transformers/tools/agents.py

@@ -27,7 +27,7 @@
 from ..models.auto import AutoTokenizer
 from ..utils import is_openai_available, is_torch_available, logging
 from .base import TASK_MAPPING, TOOL_CONFIG_FILE, Tool, load_tool, supports_remote
-from .prompts import CHAT_MESSAGE_PROMPT, CHAT_PROMPT_TEMPLATE, RUN_PROMPT_TEMPLATE
+from .prompts import CHAT_MESSAGE_PROMPT, download_prompt
 from .python_interpreter import evaluate
 
 
@@ -193,9 +193,13 @@ class Agent:
 
     Args:
         chat_prompt_template (`str`, *optional*):
-            Pass along your own prompt if you want to override the default template for the `chat` method.
+            Pass along your own prompt if you want to override the default template for the `chat` method. Can be the
+            actual prompt template or a repo ID (on the Hugging Face Hub). The prompt should be in a file named
+            `chat_prompt_template.txt` in this repo in this case.
         run_prompt_template (`str`, *optional*):
-            Pass along your own prompt if you want to override the default template for the `run` method.
+            Pass along your own prompt if you want to override the default template for the `run` method. Can be the
+            actual prompt template or a repo ID (on the Hugging Face Hub). The prompt should be in a file named
+            `run_prompt_template.txt` in this repo in this case.
         additional_tools ([`Tool`], list of tools or dictionary with tool values, *optional*):
             Any additional tools to include on top of the default ones. If you pass along a tool with the same name as
             one of the default tools, that default tool will be overridden.
@@ -204,8 +208,9 @@ class Agent:
     def __init__(self, chat_prompt_template=None, run_prompt_template=None, additional_tools=None):
         _setup_default_tools()
 
-        self.chat_prompt_template = CHAT_PROMPT_TEMPLATE if chat_prompt_template is None else chat_prompt_template
-        self.run_prompt_template = RUN_PROMPT_TEMPLATE if run_prompt_template is None else run_prompt_template
+        agent_name = self.__class__.__name__
+        self.chat_prompt_template = download_prompt(chat_prompt_template, agent_name, mode="chat")
+        self.run_prompt_template = download_prompt(run_prompt_template, agent_name, mode="run")
         self._toolbox = HUGGINGFACE_DEFAULT_TOOLS.copy()
         self.log = print
         if additional_tools is not None:
@@ -367,9 +372,13 @@ class OpenAiAgent(Agent):
         api_key (`str`, *optional*):
             The API key to use. If unset, will look for the environment variable `"OPENAI_API_KEY"`.
         chat_prompt_template (`str`, *optional*):
-            Pass along your own prompt if you want to override the default template for the `chat` method.
+            Pass along your own prompt if you want to override the default template for the `chat` method. Can be the
+            actual prompt template or a repo ID (on the Hugging Face Hub). The prompt should be in a file named
+            `chat_prompt_template.txt` in this repo in this case.
         run_prompt_template (`str`, *optional*):
-            Pass along your own prompt if you want to override the default template for the `run` method.
+            Pass along your own prompt if you want to override the default template for the `run` method. Can be the
+            actual prompt template or a repo ID (on the Hugging Face Hub). The prompt should be in a file named
+            `run_prompt_template.txt` in this repo in this case.
         additional_tools ([`Tool`], list of tools or dictionary with tool values, *optional*):
             Any additional tools to include on top of the default ones. If you pass along a tool with the same name as
             one of the default tools, that default tool will be overridden.
@@ -455,9 +464,13 @@ class HfAgent(Agent):
             The token to use as HTTP bearer authorization for remote files. If unset, will use the token generated when
             running `huggingface-cli login` (stored in `~/.huggingface`).
         chat_prompt_template (`str`, *optional*):
-            Pass along your own prompt if you want to override the default template for the `chat` method.
+            Pass along your own prompt if you want to override the default template for the `chat` method. Can be the
+            actual prompt template or a repo ID (on the Hugging Face Hub). The prompt should be in a file named
+            `chat_prompt_template.txt` in this repo in this case.
         run_prompt_template (`str`, *optional*):
-            Pass along your own prompt if you want to override the default template for the `run` method.
+            Pass along your own prompt if you want to override the default template for the `run` method. Can be the
+            actual prompt template or a repo ID (on the Hugging Face Hub). The prompt should be in a file named
+            `run_prompt_template.txt` in this repo in this case.
         additional_tools ([`Tool`], list of tools or dictionary with tool values, *optional*):
             Any additional tools to include on top of the default ones. If you pass along a tool with the same name as
             one of the default tools, that default tool will be overridden.
@@ -521,9 +534,13 @@ class LocalAgent(Agent):
         tokenizer ([`PreTrainedTokenizer`]):
             The tokenizer to use for the agent.
         chat_prompt_template (`str`, *optional*):
-            Pass along your own prompt if you want to override the default template for the `chat` method.
+            Pass along your own prompt if you want to override the default template for the `chat` method. Can be the
+            actual prompt template or a repo ID (on the Hugging Face Hub). The prompt should be in a file named
+            `chat_prompt_template.txt` in this repo in this case.
         run_prompt_template (`str`, *optional*):
-            Pass along your own prompt if you want to override the default template for the `run` method.
+            Pass along your own prompt if you want to override the default template for the `run` method. Can be the
+            actual prompt template or a repo ID (on the Hugging Face Hub). The prompt should be in a file named
+            `run_prompt_template.txt` in this repo in this case.
         additional_tools ([`Tool`], list of tools or dictionary with tool values, *optional*):
             Any additional tools to include on top of the default ones. If you pass along a tool with the same name as
             one of the default tools, that default tool will be overridden.

src/transformers/tools/prompts.py

@@ -14,173 +14,35 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import re
 
-# docstyle-ignore
-RUN_PROMPT_TEMPLATE = """I will ask you to perform a task, your job is to come up with a series of simple commands in Python that will perform the task.
-To help you, I will give you access to a set of tools that you can use. Each tool is a Python function and has a description explaining the task it performs, the inputs it expects and the outputs it returns.
-You should first explain which tool you will use to perform the task and for what reason, then write the code in Python.
-Each instruction in Python should be a simple assignment. You can print intermediate results if it makes sense to do so.
-
-Tools:
-<<all_tools>>
-
-
-Task: "Answer the question in the variable `question` about the image stored in the variable `image`. The question is in French."
-
-I will use the following tools: `translator` to translate the question into English and then `image_qa` to answer the question on the input image.
-
-Answer:
-```py
-translated_question = translator(question=question, src_lang="French", tgt_lang="English")
-print(f"The translated question is {translated_question}.")
-answer = image_qa(image=image, question=translated_question)
-print(f"The answer is {answer}")
-```
-
-Task: "Identify the oldest person in the `document` and create an image showcasing the result."
-
-I will use the following tools: `document_qa` to find the oldest person in the document, then `image_generator` to generate an image according to the answer.
-
-Answer:
-```py
-answer = document_qa(document, question="What is the oldest person?")
-print(f"The answer is {answer}.")
-image = image_generator(answer)
-```
-
-Task: "Generate an image using the text given in the variable `caption`."
-
-I will use the following tool: `image_generator` to generate an image.
-
-Answer:
-```py
-image = image_generator(prompt=caption)
-```
-
-Task: "Summarize the text given in the variable `text` and read it out loud."
-
-I will use the following tools: `summarizer` to create a summary of the input text, then `text_reader` to read it out loud.
-
-Answer:
-```py
-summarized_text = summarizer(text)
-print(f"Summary: {summarized_text}")
-audio_summary = text_reader(summarized_text)
-```
-
-Task: "Answer the question in the variable `question` about the text in the variable `text`. Use the answer to generate an image."
-
-I will use the following tools: `text_qa` to create the answer, then `image_generator` to generate an image according to the answer.
-
-Answer:
-```py
-answer = text_qa(text=text, question=question)
-print(f"The answer is {answer}.")
-image = image_generator(answer)
-```
-
-Task: "Caption the following `image`."
-
-I will use the following tool: `image_captioner` to generate a caption for the image.
-
-Answer:
-```py
-caption = image_captioner(image)
-```
-
-Task: "<<prompt>>"
-
-I will use the following"""
+from ..utils import cached_file
 
 
 # docstyle-ignore
-CHAT_PROMPT_TEMPLATE = """Below are a series of dialogues between various people and an AI assistant specialized in coding. The AI assistant tries to be helpful, polite, honest, and humble-but-knowledgeable.
-
-The job of the AI assistant is to come up with a series of simple commands in Python that will perform the task the human wants to perform.
-To help with that, the AI assistant has access to a set of tools. Each tool is a Python function and has a description explaining the task it performs, the inputs it expects and the outputs it returns.
-The AI assistant should first explain the tools it will use to perform the task and for what reason, then write the code in Python.
-Each instruction in Python should be a simple assignment. The AI assistant can print intermediate results if it makes sense to do so.
-
-Tools:
-<<all_tools>>
-
-=====
-
-Human: Answer the question in the variable `question` about the image stored in the variable `image`.
-
-Assistant: I will use the tool `image_qa` to answer the question on the input image.
-
-```py
-answer = image_qa(text=question, image=image)
-print(f"The answer is {answer}")
-```
-
-Human: I tried this code, it worked but didn't give me a good result. The question is in French
-
-Assistant: In this case, the question needs to be translated first. I will use the tool `translator` to do this.
-
-```py
-translated_question = translator(question=question, src_lang="French", tgt_lang="English")
-print(f"The translated question is {translated_question}.")
-answer = image_qa(text=translated_question, image=image)
-print(f"The answer is {answer}")
-```
-
-=====
-
-Human: Identify the oldest person in the `document`.
-
-Assistant: I will use the tool `document_qa` to find the oldest person in the document.
-
-```py
-answer = document_qa(document, question="What is the oldest person?")
-print(f"The answer is {answer}.")
-```
-
-Human: Can you generate an image with the result?
-
-Assistant: I will use the tool `image_generator` to do that.
-
-```py
-image = image_generator(answer)
-```
-
-=====
-
-Human: Summarize the text given in the variable `text` and read it out loud.
-
-Assistant: I will use the tool `summarizer` to create a summary of the input text, then the tool `text_reader` to read it out loud.
-
-```py
-summarized_text = summarizer(text)
-print(f"Summary: {summarized_text}")
-audio_summary = text_reader(text=summary)
-```
-
-Human: I got the following error: "The variable `summary` is not defined."
-
-Assistant: My bad! Let's try this code instead.
-
-```py
-summarized_text = summarizer(text)
-print(f"Summary: {summarized_text}")
-audio_summary = text_reader(text=summarized_text)
-```
+CHAT_MESSAGE_PROMPT = """
+Human: <<task>>
 
-Human: It worked! Can you translate the summary in German?
+Assistant: """
 
-Assistant: I will use the tool `translator` to translate the text in German.
 
-```py
-translated_summary = translator(summarized_text, src_lang="English", tgt_lang="German")
-```
+DEFAULT_PROMPTS_REPO = "huggingface-tools/default-prompts"
+PROMPT_FILES = {"chat": "chat_prompt_template.txt", "run": "run_prompt_template.txt"}
 
-====
-"""
 
+def download_prompt(prompt_or_repo_id, agent_name, mode="run"):
+    """
+    Downloads and caches the prompt from a repo and returns it contents (if necessary)
+    """
+    if prompt_or_repo_id is None:
+        prompt_or_repo_id = DEFAULT_PROMPTS_REPO
 
-# docstyle-ignore
-CHAT_MESSAGE_PROMPT = """
-Human: <<task>>
+    # prompt is considered a repo ID when it does not contain any kind of space
+    if re.search("\\s", prompt_or_repo_id) is not None:
+        return prompt_or_repo_id
 
-Assistant: """
+    prompt_file = cached_file(
+        prompt_or_repo_id, PROMPT_FILES[mode], repo_type="dataset", user_agent={"agent": agent_name}
+    )
+    with open(prompt_file, "r", encoding="utf-8") as f:
+        return f.read()