Skip to content

parllelized extract_tools call in process_chat rponse #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 1, 2025
Merged

parllelized extract_tools call in process_chat rponse #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions backend/score.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -399,10 +399,11 @@ async def magic_trek_chat_bot(
# test switching the model

## added by ian
mode = "graph_vector_fulltext" ## added by ian
uri = "neo4j+s://1eeea795.databases.neo4j.io"
# mode = "graph_vector_fulltext" ## added by ian
mode = "graph_vector" ## changed by ian from graph_vector_fulltext to graph_vector april 1, 2025
uri = "neo4j+s://939cd2a2.databases.neo4j.io"
userName = "neo4j"
password = "BhuaeBNSCPh_aouTyIyiyMhMeRC55DZHBr1_0vfM2Pg"
password = "4SG1SUwI22yLo6DMNkRZe1ItvGp778LIEFpCE_aOEL0"
database = "neo4j"
# model = "openai_gpt_3.5" # changed from this model by ian friday mar 21, 2025
model = "openai_gpt_4o_mini"
Expand Down Expand Up @@ -442,8 +443,11 @@ async def magic_trek_chat_bot(
logging.info("printing tool calls in magic_trek_chat_bot")
logging.info(result["tool_calls"])
response = create_api_response('Success',data=result)
log_start = time.time()
logging.info("response")
logging.info(response)
log_time = time.time() - log_start
logging.info(f"Logging response took {log_time:.2f} seconds")
return response
except Exception as e:
job_status = "Failed"
Expand Down
42 changes: 31 additions & 11 deletions backend/src/QA_integration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,11 +436,14 @@ def extract_tool_calls_direct(model, messages):
]

logging.info("calling tool extracting with direct OpenAI API")
start_time = time.time()
response = create_chat_completion_sync(
messages=formatted_messages,
model=model,
add_tools=True
)
tool_extraction_time = time.time() - start_time
logging.info(f"Tool extraction took {tool_extraction_time:.2f} seconds")

logging.info(response)
# Extract tool calls from additional_kwargs, handle missing key gracefully
Expand Down Expand Up @@ -508,17 +511,31 @@ def process_chat_response(messages, history, question, model, graph, document_na
try:
llm, doc_retriever, model_version = setup_chat(model, graph, document_names, chat_mode_settings)

tool_calls = None
if(extract_tools):
tool_calls = extract_tool_calls_direct(model, messages)
logging.info("returned tool calls")
logging.info(tool_calls)
# Shared variable to store tool calls, initialized as empty list
tool_calls = []
tool_calls_lock = threading.Lock()

def extract_tools_thread():
nonlocal tool_calls
if extract_tools:
extracted_tools = extract_tool_calls_direct(model, messages)
with tool_calls_lock:
tool_calls = extracted_tools
logging.info("returned tool calls")
logging.info(tool_calls)

# Start tool extraction thread if needed
tool_thread = None
if extract_tools:
tool_thread = threading.Thread(target=extract_tools_thread)
tool_thread.start()

docs,transformed_question = retrieve_documents(doc_retriever, messages)
# Run document retrieval and processing in parallel with tool extraction
docs, transformed_question = retrieve_documents(doc_retriever, messages)

if docs:
logging.info("documents found, process_documents about to be called")
content, result, total_tokens,formatted_docs = process_documents(docs, question, messages, llm, model, chat_mode_settings)
content, result, total_tokens, formatted_docs = process_documents(docs, question, messages, llm, model, chat_mode_settings)
else:
logging.info("No document clause running")
content = "I couldn't find any relevant documents to answer your question."
Expand All @@ -529,17 +546,20 @@ def process_chat_response(messages, history, question, model, graph, document_na
ai_response = AIMessage(content=content)
messages.append(ai_response)

if(history):
if history:
summarization_thread = threading.Thread(target=summarize_and_log, args=(history, messages, llm))
summarization_thread.start()
logging.info("Summarization thread started.")
# summarize_and_log(history, messages, llm)
metric_details = {"question":question,"contexts":formatted_docs,"answer":content}

# Wait for tool extraction to complete if it was started
if tool_thread:
tool_thread.join()

metric_details = {"question": question, "contexts": formatted_docs, "answer": content}
return {
"session_id": "",
"message": content,
"info": {
# "metrics" : metrics,
"sources": result["sources"],
"model": model_version,
"nodedetails": result["nodedetails"],
Expand Down