Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Model] tool calling support for ibm-granite/granite-20b-functioncalling #8339

Merged
merged 35 commits into from
Oct 29, 2024
Merged
Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
58e468d
initial commit
wseaton Sep 10, 2024
d4cc66b
remove original part of template
wseaton Sep 10, 2024
742704f
clean up debug logging
wseaton Sep 10, 2024
410ff88
update docs; raise not implemented
wseaton Sep 10, 2024
a5e9a1f
fix lints
wseaton Sep 10, 2024
3d28b6d
sort imports
wseaton Sep 10, 2024
74c8cc7
yapf fixes
wseaton Sep 10, 2024
23a4ca3
another format change
wseaton Sep 10, 2024
1659236
update example prompt to be conversational instead of single turn
wseaton Sep 10, 2024
b1e09a8
update docs for template; link paper
wseaton Sep 10, 2024
e82b2a6
Merge remote-tracking branch 'upstream/main' into granite-fc
wseaton Sep 27, 2024
6b0eebb
add granite to test config
wseaton Sep 27, 2024
346d554
fixup json
wseaton Sep 27, 2024
24e49b8
Add stream support for Granite 20b Tool Use
maxdebayser Sep 27, 2024
86dead8
fix docs
maxdebayser Sep 27, 2024
113fbb6
more robust whispace handling
maxdebayser Sep 28, 2024
acecb6d
remove reference to defunct granite parser
wseaton Oct 2, 2024
86e8466
remove old template
wseaton Oct 2, 2024
43c8078
Update tests/tool_use/utils.py to remove dupe
wseaton Oct 7, 2024
6bf4a41
Merge remote-tracking branch 'upstream/main' into granite-fc
wseaton Oct 7, 2024
2e969c7
fix double import
wseaton Oct 7, 2024
e18219c
add completion request arg to abstract method
wseaton Oct 7, 2024
9a0321b
formatting fixes
wseaton Oct 7, 2024
078ab85
import sorts
wseaton Oct 7, 2024
0a031bf
appease yapf
wseaton Oct 7, 2024
c6a6b56
Apply suggestions from code review
wseaton Oct 16, 2024
defed52
remove redudant indents; add type hints to utils
wseaton Oct 17, 2024
5b78cea
Merge branch 'granite-fc' of github.com:wseaton/vllm into granite-fc
wseaton Oct 17, 2024
2d3b8fe
formatting churn
wseaton Oct 17, 2024
fe13b72
Merge branch 'main' into granite-fc
wseaton Oct 21, 2024
84e93bf
change to old style type aliasing
wseaton Oct 21, 2024
ae55760
Merge branch 'granite-fc' of github.com:wseaton/vllm into granite-fc
wseaton Oct 21, 2024
1277f0b
Doc reformat, add back missing line
wseaton Oct 25, 2024
738d003
Temporarily disable the granite20b-fc test task
wseaton Oct 25, 2024
a6e1bf9
Merge branch 'vllm-project:main' into granite-fc
wseaton Oct 28, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix lints
  • Loading branch information
wseaton committed Sep 10, 2024
commit a5e9a1f9cc3224596c5535e40bee89f0ff093469
28 changes: 13 additions & 15 deletions vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py
Original file line number Diff line number Diff line change
@@ -1,23 +1,17 @@
import json
import re
from typing import Dict, List, Sequence, Union
from typing import Sequence, Union

import partial_json_parser
from partial_json_parser.core.options import Allow

from vllm.entrypoints.openai.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
ExtractedToolCallInformation,
FunctionCall,
InitialDeltaToolCall,
ToolCall,
)
from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import ToolParser
from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff
from vllm.logger import init_logger
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
from vllm.transformers_utils.tokenizer import AnyTokenizer

logger = init_logger(__name__)

Expand All @@ -33,7 +27,6 @@ def __init__(self, tokenizer: AnyTokenizer):
def extract_tool_calls(
self, model_output: str
) -> ExtractedToolCallInformation:

if self.tool_start_token not in model_output:
return ExtractedToolCallInformation(
tools_called=False, tool_calls=[], content=model_output
Expand All @@ -43,18 +36,20 @@ def extract_tool_calls(
try:
matches = list(self.tool_call_regex.finditer(model_output))
logger.debug("Found %d tool call matches", len(matches))

raw_function_calls = []

for i, match in enumerate(matches):
# position after the <function_call> tag
start_of_json = match.end()
# end_index == the start of the next function call, if it exists
# end_index == the start of the next function call
# (if exists)
next_function_call_start = (
matches[i + 1].start() if i + 1 < len(matches) else None
)

# extract the full JSON object using bracket matching via start_of_json and optional end index
# extract the full JSON object using bracket matching via
# start_of_json and optional end index
full_json_str = extract_full_json(
model_output, start_of_json, next_function_call_start
)
Expand Down Expand Up @@ -99,12 +94,15 @@ def extract_tool_calls_streaming(
) -> Union[DeltaMessage, None]:
raise NotImplementedError(
"streaming tool call parsing has not yet been implemented"
"for Granite")

"for Granite"
)


def extract_full_json(text, start_index, end_index=None):
"""Extracts the full JSON object from text starting at `start_index` and optionally up to `end_index`."""
"""
Extracts the full JSON object from text starting at `start_index`
and optionally up to `end_index`.
"""
brace_count = 0
json_start = start_index
for i, char in enumerate(text[start_index:end_index], start=start_index):
Expand Down
Loading