forked from vllm-project/vllm
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Feature] Add support for Llama 3.1 and 3.2 tool use (vllm-project#8343)
Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
- Loading branch information
1 parent
1b49148
commit 344cd2b
Showing
10 changed files
with
576 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| {{- bos_token }} | ||
| {%- if custom_tools is defined %} | ||
| {%- set tools = custom_tools %} | ||
| {%- endif %} | ||
| {%- if not tools_in_user_message is defined %} | ||
| {#- Llama 3.1 doesn't pass all tests if the tools are in the system prompt #} | ||
| {%- set tools_in_user_message = true %} | ||
| {%- endif %} | ||
| {%- if not date_string is defined %} | ||
| {%- if strftime_now is defined %} | ||
| {%- set date_string = strftime_now("%d %b %Y") %} | ||
| {%- else %} | ||
| {%- set date_string = "26 Jul 2024" %} | ||
| {%- endif %} | ||
| {%- endif %} | ||
| {%- if not tools is defined %} | ||
| {%- set tools = none %} | ||
| {%- endif %} | ||
|
|
||
| {#- This block extracts the system message, so we can slot it into the right place. #} | ||
| {%- if messages[0]['role'] == 'system' %} | ||
| {%- set system_message = messages[0]['content']|trim %} | ||
| {%- set messages = messages[1:] %} | ||
| {%- else %} | ||
| {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %} | ||
| {%- endif %} | ||
|
|
||
| {#- System message #} | ||
| {{- "<|start_header_id|>system<|end_header_id|>\n\n" }} | ||
| {%- if tools is not none %} | ||
| {{- "Environment: ipython\n" }} | ||
| {%- endif %} | ||
| {{- "Cutting Knowledge Date: December 2023\n" }} | ||
| {{- "Today Date: " + date_string + "\n\n" }} | ||
| {%- if tools is not none and not tools_in_user_message %} | ||
| {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} | ||
| {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} | ||
| {{- "Do not use variables.\n\n" }} | ||
| {%- for t in tools %} | ||
| {{- t | tojson(indent=4) }} | ||
| {{- "\n\n" }} | ||
| {%- endfor %} | ||
| {%- endif %} | ||
| {{- system_message }} | ||
| {{- "<|eot_id|>" }} | ||
|
|
||
| {#- Custom tools are passed in a user message with some extra guidance #} | ||
| {%- if tools_in_user_message and not tools is none %} | ||
| {#- Extract the first user message so we can plug it in here #} | ||
| {%- if messages | length != 0 %} | ||
| {%- set first_user_message = messages[0]['content']|trim %} | ||
| {%- set messages = messages[1:] %} | ||
| {%- else %} | ||
| {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} | ||
| {%- endif %} | ||
| {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} | ||
| {{- "Given the following functions, please respond with a JSON for a function call " }} | ||
| {{- "with its proper arguments that best answers the given prompt.\n\n" }} | ||
| {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} | ||
| {{- "Do not use variables.\n\n" }} | ||
| {%- for t in tools %} | ||
| {{- t | tojson(indent=4) }} | ||
| {{- "\n\n" }} | ||
| {%- endfor %} | ||
| {{- first_user_message + "<|eot_id|>"}} | ||
| {%- endif %} | ||
|
|
||
| {%- for message in messages %} | ||
| {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} | ||
| {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} | ||
| {%- elif 'tool_calls' in message %} | ||
| {%- if not message.tool_calls|length == 1 %} | ||
| {{- raise_exception("This model only supports single tool-calls at once!") }} | ||
| {%- endif %} | ||
| {%- set tool_call = message.tool_calls[0].function %} | ||
| {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} | ||
| {{- '{"name": "' + tool_call.name + '", ' }} | ||
| {{- '"parameters": ' }} | ||
| {{- tool_call.arguments | tojson }} | ||
| {{- "}" }} | ||
| {{- "<|eot_id|>" }} | ||
| {%- elif message.role == "tool" or message.role == "ipython" %} | ||
| {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} | ||
| {%- if message.content is mapping %} | ||
| {{- message.content | tojson }} | ||
| {%- else %} | ||
| {{- { "output": message.content } | tojson }} | ||
| {%- endif %} | ||
| {{- "<|eot_id|>" }} | ||
| {%- endif %} | ||
| {%- endfor %} | ||
| {%- if add_generation_prompt %} | ||
| {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} | ||
| {%- endif %} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| {{- bos_token }} | ||
| {%- if custom_tools is defined %} | ||
| {%- set tools = custom_tools %} | ||
| {%- endif %} | ||
| {%- if not tools_in_user_message is defined %} | ||
| {%- set tools_in_user_message = false %} | ||
| {%- endif %} | ||
| {%- if not date_string is defined %} | ||
| {%- if strftime_now is defined %} | ||
| {%- set date_string = strftime_now("%d %b %Y") %} | ||
| {%- else %} | ||
| {%- set date_string = "26 Jul 2024" %} | ||
| {%- endif %} | ||
| {%- endif %} | ||
| {%- if not tools is defined %} | ||
| {%- set tools = none %} | ||
| {%- endif %} | ||
|
|
||
| {#- This block extracts the system message, so we can slot it into the right place. #} | ||
| {%- if messages[0]['role'] == 'system' %} | ||
| {%- set system_message = messages[0]['content']|trim %} | ||
| {%- set messages = messages[1:] %} | ||
| {%- else %} | ||
| {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %} | ||
| {%- endif %} | ||
|
|
||
| {#- System message #} | ||
| {{- "<|start_header_id|>system<|end_header_id|>\n\n" }} | ||
| {%- if tools is not none %} | ||
| {{- "Environment: ipython\n" }} | ||
| {%- endif %} | ||
| {{- "Cutting Knowledge Date: December 2023\n" }} | ||
| {{- "Today Date: " + date_string + "\n\n" }} | ||
| {%- if tools is not none and not tools_in_user_message %} | ||
| {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} | ||
| {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} | ||
| {{- "Do not use variables.\n\n" }} | ||
| {%- for t in tools %} | ||
| {{- t | tojson(indent=4) }} | ||
| {{- "\n\n" }} | ||
| {%- endfor %} | ||
| {%- endif %} | ||
| {{- system_message }} | ||
| {{- "<|eot_id|>" }} | ||
|
|
||
| {#- Custom tools are passed in a user message with some extra guidance #} | ||
| {%- if tools_in_user_message and not tools is none %} | ||
| {#- Extract the first user message so we can plug it in here #} | ||
| {%- if messages | length != 0 %} | ||
| {%- set first_user_message = messages[0]['content']|trim %} | ||
| {%- set messages = messages[1:] %} | ||
| {%- else %} | ||
| {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} | ||
| {%- endif %} | ||
| {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} | ||
| {{- "Given the following functions, please respond with a JSON for a function call " }} | ||
| {{- "with its proper arguments that best answers the given prompt.\n\n" }} | ||
| {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} | ||
| {{- "Do not use variables.\n\n" }} | ||
| {%- for t in tools %} | ||
| {{- t | tojson(indent=4) }} | ||
| {{- "\n\n" }} | ||
| {%- endfor %} | ||
| {{- first_user_message + "<|eot_id|>"}} | ||
| {%- endif %} | ||
|
|
||
| {%- for message in messages %} | ||
| {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} | ||
| {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} | ||
| {%- elif 'tool_calls' in message %} | ||
| {%- if not message.tool_calls|length == 1 %} | ||
| {{- raise_exception("This model only supports single tool-calls at once!") }} | ||
| {%- endif %} | ||
| {%- set tool_call = message.tool_calls[0].function %} | ||
| {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} | ||
| {{- '{"name": "' + tool_call.name + '", ' }} | ||
| {{- '"parameters": ' }} | ||
| {{- tool_call.arguments | tojson }} | ||
| {{- "}" }} | ||
| {{- "<|eot_id|>" }} | ||
| {%- elif message.role == "tool" or message.role == "ipython" %} | ||
| {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} | ||
| {%- if message.content is mapping %} | ||
| {{- message.content | tojson }} | ||
| {%- else %} | ||
| {{- { "output": message.content } | tojson }} | ||
| {%- endif %} | ||
| {{- "<|eot_id|>" }} | ||
| {%- endif %} | ||
| {%- endfor %} | ||
| {%- if add_generation_prompt %} | ||
| {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} | ||
| {%- endif %} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.