feat: Add tool calling versions of example agents, handle tool calling agent memory

dsowinski2 · rafalcymerys · commit 8c00889b66ce · 2025-12-08T14:52:03.000+01:00
diff --git a/plugins/enthusiast-agent-catalog-enrichment/pyproject.toml b/plugins/enthusiast-agent-catalog-enrichment/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "enthusiast-agent-catalog-enrichment"
-version = "1.1.2"
+version = "1.2.0"
 description = "Example implementation of a catalog enrichment ReAct agent for Enthusiast"
 authors = [
     {name = "Damian Sowiński",email = "damian.sowinski@upsidelab.io"}
@@ -10,7 +10,8 @@ requires-python = ">=3.10,<4"
 dependencies = [
     "enthusiast-common (>=1.4.0,<2.0.0)",
     "langchain (>=0.3.26,<0.4.0)",
-    "enthusiast_agent_re_act (>=1.2.0)"
+    "enthusiast_agent_re_act (>=1.2.0)",
+    "enthusiast_agent_tool_calling (>=1.0.0)"
 ]
 
 [tool.poetry]
diff --git a/plugins/enthusiast-agent-catalog-enrichment/src/enthusiast_agent_catalog_enrichment/re_act/agent.py b/plugins/enthusiast-agent-catalog-enrichment/src/enthusiast_agent_catalog_enrichment/re_act/agent.py
diff --git a/plugins/enthusiast-agent-catalog-enrichment/src/enthusiast_agent_catalog_enrichment/re_act/config.py b/plugins/enthusiast-agent-catalog-enrichment/src/enthusiast_agent_catalog_enrichment/re_act/config.py
diff --git a/plugins/enthusiast-agent-catalog-enrichment/src/enthusiast_agent_catalog_enrichment/re_act/prompt.py b/plugins/enthusiast-agent-catalog-enrichment/src/enthusiast_agent_catalog_enrichment/re_act/prompt.py
diff --git a/plugins/enthusiast-agent-catalog-enrichment/src/enthusiast_agent_catalog_enrichment/tool_calling/agent.py b/plugins/enthusiast-agent-catalog-enrichment/src/enthusiast_agent_catalog_enrichment/tool_calling/agent.py
@@ -0,0 +1,19 @@
+from enthusiast_agent_tool_calling import BaseToolCallingAgent
+from enthusiast_common.utils import RequiredFieldsModel
+from pydantic import Field, Json
+
+
+class ExtractDataPromptInput(RequiredFieldsModel):
+    output_format: Json = Field(title="Output format", description="Output format of the extracted data")
+
+
+class CatalogEnrichmentToolCallingAgent(BaseToolCallingAgent):
+    PROMPT_INPUT = ExtractDataPromptInput
+    FILE_UPLOAD = True
+
+    def get_answer(self, input_text: str) -> str:
+        agent_executor = self._build_agent_executor()
+        agent_output = agent_executor.invoke(
+            {"input": input_text, "data_format": self.PROMPT_INPUT.output_format}, config=self._build_invoke_config()
+        )
+        return agent_output["output"]
diff --git a/plugins/enthusiast-agent-catalog-enrichment/src/enthusiast_agent_catalog_enrichment/tool_calling/config.py b/plugins/enthusiast-agent-catalog-enrichment/src/enthusiast_agent_catalog_enrichment/tool_calling/config.py
@@ -0,0 +1,23 @@
+from enthusiast_common.config import AgentConfigWithDefaults
+from enthusiast_common.config.prompts import ChatPromptTemplateConfig, Message, MessageRole
+
+from .agent import CatalogEnrichmentToolCallingAgent
+from .prompt import DATA_EXTRACTION_TOOL_CALLING_AGENT_PROMPT
+
+
+def get_config() -> AgentConfigWithDefaults:
+    return AgentConfigWithDefaults(
+        prompt_template=ChatPromptTemplateConfig(
+            messages=[
+                Message(
+                    role=MessageRole.SYSTEM,
+                    content=DATA_EXTRACTION_TOOL_CALLING_AGENT_PROMPT,
+                ),
+                Message(role=MessageRole.PLACEHOLDER, content="{chat_history}"),
+                Message(role=MessageRole.USER, content="{input}"),
+                Message(role=MessageRole.PLACEHOLDER, content="{agent_scratchpad}"),
+            ]
+        ),
+        agent_class=CatalogEnrichmentToolCallingAgent,
+        tools=CatalogEnrichmentToolCallingAgent.TOOLS,
+    )
diff --git a/plugins/enthusiast-agent-catalog-enrichment/src/enthusiast_agent_catalog_enrichment/tool_calling/prompt.py b/plugins/enthusiast-agent-catalog-enrichment/src/enthusiast_agent_catalog_enrichment/tool_calling/prompt.py
@@ -0,0 +1,9 @@
+DATA_EXTRACTION_TOOL_CALLING_AGENT_PROMPT = """
+I want you to help extracting and describing in details attributes for products.
+In case of any missing information carefully collect it one by one.
+In tools specify exactly what are you looking for.
+If there are more variants return all of them.
+You need to return variant data in given shape: {data_format}.
+Always verify your answer
+Return only json
+"""
diff --git a/plugins/enthusiast-agent-ocr-to-order/pyproject.toml b/plugins/enthusiast-agent-ocr-to-order/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "enthusiast-agent-ocr-to-order"
-version = "1.1.2"
+version = "1.2.0"
 description = "Example implementation of a OCR to order ReAct agent for Enthusiast"
 authors = [
     {name = "Damian Sowiński",email = "damian.sowinski@upsidelab.io"}
@@ -10,7 +10,8 @@ requires-python = ">=3.10,<4"
 dependencies = [
     "enthusiast-common (>=1.4.0,<2.0.0)",
     "langchain (>=0.3.26,<0.4.0)",
-    "enthusiast_agent_re_act (>=1.2.0)"
+    "enthusiast_agent_re_act (>=1.2.0)",
+    "enthusiast_agent_tool_calling (>=1.0.0)"
 ]
 
 [tool.poetry]
diff --git a/plugins/enthusiast-agent-ocr-to-order/src/enthusiast_agent_ocr_to_order/re_act/agent.py b/plugins/enthusiast-agent-ocr-to-order/src/enthusiast_agent_ocr_to_order/re_act/agent.py
@@ -3,8 +3,8 @@
 from langchain.agents import AgentExecutor, create_react_agent
 from langchain_core.tools import render_text_description_and_args
 
-from .tools.place_order_tool import OrderPlacementTool
-from .tools.product_search_tool import ProductSearchTool
+from ..tools.place_order_tool import OrderPlacementTool
+from ..tools.product_search_tool import ProductSearchTool
 
 
 class OCROrderReActAgent(BaseReActAgent):
diff --git a/plugins/enthusiast-agent-ocr-to-order/src/enthusiast_agent_ocr_to_order/re_act/config.py b/plugins/enthusiast-agent-ocr-to-order/src/enthusiast_agent_ocr_to_order/re_act/config.py
diff --git a/plugins/enthusiast-agent-ocr-to-order/src/enthusiast_agent_ocr_to_order/re_act/prompt.py b/plugins/enthusiast-agent-ocr-to-order/src/enthusiast_agent_ocr_to_order/re_act/prompt.py
diff --git a/plugins/enthusiast-agent-ocr-to-order/src/enthusiast_agent_ocr_to_order/tool_calling/agent.py b/plugins/enthusiast-agent-ocr-to-order/src/enthusiast_agent_ocr_to_order/tool_calling/agent.py
@@ -0,0 +1,13 @@
+from enthusiast_agent_tool_calling import BaseToolCallingAgent
+from enthusiast_common.config.base import LLMToolConfig
+
+from ..tools.place_order_tool import OrderPlacementTool
+from ..tools.product_search_tool import ProductSearchTool
+
+
+class OCROrderToolCallingAgent(BaseToolCallingAgent):
+    TOOLS = [
+        LLMToolConfig(tool_class=ProductSearchTool),
+        LLMToolConfig(tool_class=OrderPlacementTool),
+    ]
+    FILE_UPLOAD = True
diff --git a/plugins/enthusiast-agent-ocr-to-order/src/enthusiast_agent_ocr_to_order/tool_calling/config.py b/plugins/enthusiast-agent-ocr-to-order/src/enthusiast_agent_ocr_to_order/tool_calling/config.py
@@ -0,0 +1,25 @@
+from enthusiast_common.config import (
+    AgentConfigWithDefaults,
+)
+from enthusiast_common.config.prompts import ChatPromptTemplateConfig, Message, MessageRole
+
+from .agent import OCROrderToolCallingAgent
+from .prompt import OCR_TOOL_CALLING_AGENT_PROMPT
+
+
+def get_config() -> AgentConfigWithDefaults:
+    return AgentConfigWithDefaults(
+        prompt_template=ChatPromptTemplateConfig(
+            messages=[
+                Message(
+                    role=MessageRole.SYSTEM,
+                    content=OCR_TOOL_CALLING_AGENT_PROMPT,
+                ),
+                Message(role=MessageRole.PLACEHOLDER, content="{chat_history}"),
+                Message(role=MessageRole.USER, content="{input}"),
+                Message(role=MessageRole.PLACEHOLDER, content="{agent_scratchpad}"),
+            ]
+        ),
+        agent_class=OCROrderToolCallingAgent,
+        tools=OCROrderToolCallingAgent.TOOLS,
+    )
diff --git a/plugins/enthusiast-agent-ocr-to-order/src/enthusiast_agent_ocr_to_order/tool_calling/prompt.py b/plugins/enthusiast-agent-ocr-to-order/src/enthusiast_agent_ocr_to_order/tool_calling/prompt.py
@@ -0,0 +1,9 @@
+OCR_TOOL_CALLING_AGENT_PROMPT = """
+I want you to help finding products and placing order.
+You need to find matching products and place order for them with corresponding quantity.
+Search for products one by one.
+If some products are missing ask user what to do.
+If you ask user something always give him context.
+Do not came up with anything, use tools if it's possible.
+Always verify your answer
+"""
diff --git a/plugins/enthusiast-agent-product-search/pyproject.toml b/plugins/enthusiast-agent-product-search/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "enthusiast-agent-product-search"
-version = "1.1.0"
+version = "1.2.0"
 description = "Example implementation of a Product Search ReAct agent for Enthusiast"
 authors = [
     {name = "Damian Sowiński",email = "damian.sowinski@upsidelab.io"}
@@ -10,7 +10,8 @@ requires-python = ">=3.10,<4"
 dependencies = [
     "enthusiast-common (>=1.4.0,<2.0.0)",
     "langchain (>=0.3.26,<0.4.0)",
-    "enthusiast_agent_re_act (>=1.2.0)"
+    "enthusiast_agent_re_act (>=1.2.0)",
+    "enthusiast_agent_tool_calling (>=1.0.0)"
 ]
 
 [tool.poetry]
diff --git a/plugins/enthusiast-agent-product-search/src/enthusiast_agent_product_search/re_act/agent.py b/plugins/enthusiast-agent-product-search/src/enthusiast_agent_product_search/re_act/agent.py
@@ -3,7 +3,7 @@
 from langchain.agents import AgentExecutor, create_react_agent
 from langchain_core.tools import render_text_description_and_args
 
-from .tools.product_search_tool import ProductSearchTool
+from ..tools.product_search_tool import ProductSearchTool
 
 
 class ProductSearchReActAgent(BaseReActAgent):
diff --git a/plugins/enthusiast-agent-product-search/src/enthusiast_agent_product_search/re_act/config.py b/plugins/enthusiast-agent-product-search/src/enthusiast_agent_product_search/re_act/config.py
@@ -5,10 +5,10 @@
 )
 from enthusiast_common.config.prompts import PromptTemplateConfig
 
+from ..document_retriever import DocumentRetriever
+from ..product_retriever import ProductRetriever
+from ..product_retriever_prompt import CUSTOM_QUERY_PROMPT_TEMPLATE
 from .agent import ProductSearchReActAgent
-from .document_retriever import DocumentRetriever
-from .product_retriever import ProductRetriever
-from .product_retriever_prompt import CUSTOM_QUERY_PROMPT_TEMPLATE
 from .prompt import PRODUCT_SEARCH_AGENT_PROMPT
 
 
diff --git a/plugins/enthusiast-agent-product-search/src/enthusiast_agent_product_search/re_act/prompt.py b/plugins/enthusiast-agent-product-search/src/enthusiast_agent_product_search/re_act/prompt.py
diff --git a/plugins/enthusiast-agent-product-search/src/enthusiast_agent_product_search/tool_calling/agent.py b/plugins/enthusiast-agent-product-search/src/enthusiast_agent_product_search/tool_calling/agent.py
@@ -0,0 +1,10 @@
+from enthusiast_agent_tool_calling import BaseToolCallingAgent
+from enthusiast_common.config.base import LLMToolConfig
+
+from ..tools.product_search_tool import ProductSearchTool
+
+
+class ProductSearchToolCallingAgent(BaseToolCallingAgent):
+    TOOLS = [
+        LLMToolConfig(tool_class=ProductSearchTool),
+    ]
diff --git a/plugins/enthusiast-agent-product-search/src/enthusiast_agent_product_search/tool_calling/config.py b/plugins/enthusiast-agent-product-search/src/enthusiast_agent_product_search/tool_calling/config.py
@@ -0,0 +1,41 @@
+from enthusiast_common.config import (
+    AgentConfigWithDefaults,
+    RetrieverConfig,
+    RetrieversConfig,
+)
+from enthusiast_common.config.prompts import ChatPromptTemplateConfig, Message, MessageRole
+
+from ..document_retriever import DocumentRetriever
+from ..product_retriever import ProductRetriever
+from ..product_retriever_prompt import CUSTOM_QUERY_PROMPT_TEMPLATE
+from .agent import ProductSearchToolCallingAgent
+from .prompt import PRODUCT_SEARCH_TOOL_CALLING_AGENT_PROMPT
+
+
+def get_config() -> AgentConfigWithDefaults:
+    return AgentConfigWithDefaults(
+        prompt_template=ChatPromptTemplateConfig(
+            messages=[
+                Message(
+                    role=MessageRole.SYSTEM,
+                    content=PRODUCT_SEARCH_TOOL_CALLING_AGENT_PROMPT,
+                ),
+                Message(role=MessageRole.PLACEHOLDER, content="{chat_history}"),
+                Message(role=MessageRole.USER, content="{input}"),
+                Message(role=MessageRole.PLACEHOLDER, content="{agent_scratchpad}"),
+            ]
+        ),
+        agent_class=ProductSearchToolCallingAgent,
+        tools=ProductSearchToolCallingAgent.TOOLS,
+        retrievers=RetrieversConfig(
+            document=RetrieverConfig(retriever_class=DocumentRetriever),
+            product=RetrieverConfig(
+                retriever_class=ProductRetriever,
+                extra_kwargs={
+                    "prompt_template": CUSTOM_QUERY_PROMPT_TEMPLATE,
+                    "max_sample_products": 12,
+                    "number_of_products": 12,
+                },
+            ),
+        ),
+    )
diff --git a/plugins/enthusiast-agent-product-search/src/enthusiast_agent_product_search/tool_calling/prompt.py b/plugins/enthusiast-agent-product-search/src/enthusiast_agent_product_search/tool_calling/prompt.py
@@ -0,0 +1,10 @@
+PRODUCT_SEARCH_TOOL_CALLING_AGENT_PROMPT = """
+I want you to help finding most suitable product using the ReACT (Reasoning and Acting) approach.
+Search for products one by one.
+Filter out products which are not suitable or match less user needs.
+Assume that user is not an expert in this field and try to help.
+Ask follow-up questions to user to find one product.
+If you ask user something always give him context.
+Do not came up with anything, use tools if it's possible.
+Always verify your answer
+"""
diff --git a/plugins/enthusiast-agent-product-search/src/enthusiast_agent_product_search/tools/product_search_tool.py b/plugins/enthusiast-agent-product-search/src/enthusiast_agent_product_search/tools/product_search_tool.py
@@ -26,9 +26,9 @@ def __init__(
         self.llm = llm
         self.injector = injector
 
-    def run(self, full_user_request: str):
+    def run(self, product_description: str):
         product_retriever = self.injector.product_retriever
-        relevant_products = product_retriever.find_products_matching_query(full_user_request)
+        relevant_products = product_retriever.find_products_matching_query(product_description)
         if not relevant_products:
             return "No products found, try to loosen the criteria"
 
diff --git a/plugins/enthusiast-agent-user-manual-search/pyproject.toml b/plugins/enthusiast-agent-user-manual-search/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "enthusiast-agent-user-manual-search"
-version = "1.1.0"
+version = "1.2.0"
 description = "Example implementation of a Customer Support ReAct agent for Enthusiast"
 authors = [
     {name = "Damian Sowiński",email = "damian.sowinski@upsidelab.io"}
@@ -10,7 +10,8 @@ requires-python = ">=3.10,<4"
 dependencies = [
     "enthusiast-common (>=1.4.0,<2.0.0)",
     "langchain (>=0.3.26,<0.4.0)",
-    "enthusiast_agent_re_act (>=1.2.0)"
+    "enthusiast_agent_re_act (>=1.2.0)",
+    "enthusiast_agent_tool_calling (>=1.0.0)"
 ]
 
 [tool.poetry]
diff --git a/plugins/enthusiast-agent-user-manual-search/src/enthusiast_agent_user_manual_search/re_act/agent.py b/plugins/enthusiast-agent-user-manual-search/src/enthusiast_agent_user_manual_search/re_act/agent.py
@@ -3,8 +3,8 @@
 from langchain.agents import AgentExecutor, create_react_agent
 from langchain_core.tools import render_text_description_and_args
 
-from .tools.document_retriver_tool import RetrieveDocumentsTool
-from .tools.solution_verification_tool import SolutionVerificationTool
+from ..tools.document_retriver_tool import RetrieveDocumentsTool
+from ..tools.solution_verification_tool import SolutionVerificationTool
 
 
 class UserManualSearchAgent(BaseReActAgent):
diff --git a/plugins/enthusiast-agent-user-manual-search/src/enthusiast_agent_user_manual_search/re_act/config.py b/plugins/enthusiast-agent-user-manual-search/src/enthusiast_agent_user_manual_search/re_act/config.py
diff --git a/plugins/enthusiast-agent-user-manual-search/src/enthusiast_agent_user_manual_search/re_act/prompt.py b/plugins/enthusiast-agent-user-manual-search/src/enthusiast_agent_user_manual_search/re_act/prompt.py
@@ -1,7 +1,7 @@
 USER_MANUAL_AGENT_PROMPT = """
-You need to help user with their questions/problems with household equipment using the ReACT (Reasoning and Acting) approach.
+You need to help user with their questions/problems using the ReACT (Reasoning and Acting) approach.
 Always verify your answer
-If final answer is based on manuals informations add relevant fragments in format:
+If final answer is based on documents informations add relevant fragments in format:
 '''
 Final Answer: <solution>\nSource of information: <relevant pieces of information from documents with including context to make user easily find in by himself next time and section if available>
 '''
diff --git a/plugins/enthusiast-agent-user-manual-search/src/enthusiast_agent_user_manual_search/tool_calling/agent.py b/plugins/enthusiast-agent-user-manual-search/src/enthusiast_agent_user_manual_search/tool_calling/agent.py
@@ -0,0 +1,9 @@
+from enthusiast_agent_tool_calling import BaseToolCallingAgent
+from enthusiast_common.config.base import LLMToolConfig
+
+from ..tools.document_retriver_tool import RetrieveDocumentsTool
+from ..tools.solution_verification_tool import SolutionVerificationTool
+
+
+class UserManualSearchToolCallingAgent(BaseToolCallingAgent):
+    TOOLS = [LLMToolConfig(tool_class=SolutionVerificationTool), LLMToolConfig(tool_class=RetrieveDocumentsTool)]
diff --git a/plugins/enthusiast-agent-user-manual-search/src/enthusiast_agent_user_manual_search/tool_calling/config.py b/plugins/enthusiast-agent-user-manual-search/src/enthusiast_agent_user_manual_search/tool_calling/config.py
@@ -0,0 +1,23 @@
+from enthusiast_common.config import AgentConfigWithDefaults
+from enthusiast_common.config.prompts import ChatPromptTemplateConfig, Message, MessageRole
+
+from .agent import UserManualSearchToolCallingAgent
+from .prompt import USER_MANUAL_TOOL_CALLING_AGENT_PROMPT
+
+
+def get_config() -> AgentConfigWithDefaults:
+    return AgentConfigWithDefaults(
+        prompt_template=ChatPromptTemplateConfig(
+            messages=[
+                Message(
+                    role=MessageRole.SYSTEM,
+                    content=USER_MANUAL_TOOL_CALLING_AGENT_PROMPT,
+                ),
+                Message(role=MessageRole.PLACEHOLDER, content="{chat_history}"),
+                Message(role=MessageRole.USER, content="{input}"),
+                Message(role=MessageRole.PLACEHOLDER, content="{agent_scratchpad}"),
+            ]
+        ),
+        agent_class=UserManualSearchToolCallingAgent,
+        tools=UserManualSearchToolCallingAgent.TOOLS,
+    )
diff --git a/plugins/enthusiast-agent-user-manual-search/src/enthusiast_agent_user_manual_search/tool_calling/prompt.py b/plugins/enthusiast-agent-user-manual-search/src/enthusiast_agent_user_manual_search/tool_calling/prompt.py
@@ -0,0 +1,8 @@
+USER_MANUAL_TOOL_CALLING_AGENT_PROMPT = """
+You need to help user with their questions/problems.
+Always verify your answer
+If your answer is based on documents informations add relevant fragments in format:
+'''
+<solution>\nSource of information: <relevant pieces of information from documents with including context to make user easily find in by himself next time and section if available>
+'''
+"""
diff --git a/server/agent/core/memory/persist_intermediate_steps_mixin.py b/server/agent/core/memory/persist_intermediate_steps_mixin.py
@@ -1,6 +1,7 @@
 from abc import ABC
 from typing import Any, Dict, cast
 
+from langchain.agents.output_parsers.tools import ToolAgentAction
 from langchain.memory import ConversationBufferMemory
 from langchain_core.messages import AIMessage, BaseMessage, FunctionMessage, HumanMessage
 
@@ -20,9 +21,14 @@ def save_context(self, inputs: Dict[str, Any], outputs: Dict[str, str]) -> None:
 
         if "intermediate_steps" in outputs:
             for agent_action, result in outputs["intermediate_steps"]:
-                intermediate_step_message = BaseMessage(
-                    type=Message.MessageType.INTERMEDIATE_STEP, content=agent_action.messages[0].content
-                )
+                if isinstance(agent_action, ToolAgentAction):
+                    intermediate_step_message = BaseMessage(
+                        type=Message.MessageType.INTERMEDIATE_STEP, content=agent_action.log
+                    )
+                else:
+                    intermediate_step_message = BaseMessage(
+                        type=Message.MessageType.INTERMEDIATE_STEP, content=agent_action.messages[0].content
+                    )
                 self_as_conversation_memory.chat_memory.add_message(intermediate_step_message)
 
                 function_message = FunctionMessage(name=agent_action.tool, content=result)
