feat(Tools): Add Serply Web/Job/Scholar/News Search tool for more opt…

…ions (#5186)

Co-authored-by: teampen <136991215+teampen@users.noreply.github.com>

api/core/tools/provider/builtin/websearch/tools/get_markdown.py

@@ -0,0 +1,51 @@
+from typing import Any, Union
+
+import requests
+
+from core.tools.entities.tool_entities import ToolInvokeMessage
+from core.tools.tool.builtin_tool import BuiltinTool
+
+BASE_URL = "https://api.serply.io/v1/request"
+
+
+class SerplyApi:
+    """
+    SerplyAPI tool provider.
+    """
+
+    def __init__(self, api_key: str) -> None:
+        """Initialize SerplyAPI tool provider."""
+        self.serply_api_key = api_key
+
+    def run(self, url: str, **kwargs: Any) -> str:
+        """Run query through SerplyAPI and parse result."""
+
+        location = kwargs.get("location", "US")
+
+        headers = {
+            "X-API-KEY": self.serply_api_key,
+            "X-User-Agent": kwargs.get("device", "desktop"),
+            "X-Proxy-Location": location,
+            "User-Agent": "Dify",
+        }
+        data = {"url": url, "method": "GET", "response_type": "markdown"}
+        res = requests.post(url, headers=headers, json=data)
+        return res.text
+
+
+class GetMarkdownTool(BuiltinTool):
+    def _invoke(
+        self,
+        user_id: str,
+        tool_parameters: dict[str, Any],
+    ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
+        """
+        Invoke the SerplyApi tool.
+        """
+        url = tool_parameters["url"]
+        location = tool_parameters.get("location", None)
+
+        api_key = self.runtime.credentials["serply_api_key"]
+        result = SerplyApi(api_key).run(url, location=location)
+
+        return self.create_text_message(text=result)

api/core/tools/provider/builtin/websearch/tools/get_markdown.yaml

@@ -0,0 +1,96 @@
+identity:
+  name: get_markdown
+  author: Dify
+  label:
+    en_US: Get Markdown API
+    zh_Hans: Get Markdown API
+description:
+  human:
+    en_US: A tool to perform convert a webpage to markdown to make it easier for LLMs to understand.
+    zh_Hans: 一个将网页转换为 Markdown 的工具，以便模型更容易理解
+  llm: A tool to perform convert a webpage to markdown to make it easier for LLMs to understand.
+parameters:
+  - name: url
+    type: string
+    required: true
+    label:
+      en_US: URL
+      zh_Hans: URL
+    human_description:
+      en_US: URL that you want to grab the content from
+      zh_Hans: 您要从中获取内容的 URL
+    llm_description: Defines the link want to grab content from.
+    form: llm
+  - name: location
+    type: string
+    required: false
+    default: US
+    label:
+      en_US: Location
+      zh_Hans: 询问
+    human_description:
+      en_US: Defines from where you want the search to originate. (For example - New York)
+      zh_Hans: 定义您想要搜索的起始位置。 （例如 - 纽约）
+    llm_description: Defines from where you want the search to originate. (For example - New York)
+    form: form
+    options:
+      - value: AU
+        label:
+          en_US: Australia
+          zh_Hans: 澳大利亚
+          pt_BR: Australia
+      - value: BR
+        label:
+          en_US: Brazil
+          zh_Hans: 巴西
+          pt_BR: Brazil
+      - value: CA
+        label:
+          en_US: Canada
+          zh_Hans: 加拿大
+          pt_BR: Canada
+      - value: DE
+        label:
+          en_US: Germany
+          zh_Hans: 德国
+          pt_BR: Germany
+      - value: FR
+        label:
+          en_US: France
+          zh_Hans: 法国
+          pt_BR: France
+      - value: GB
+        label:
+          en_US: United Kingdom
+          zh_Hans: 英国
+          pt_BR: United Kingdom
+      - value: US
+        label:
+          en_US: United States
+          zh_Hans: 美国
+          pt_BR: United States
+      - value: JP
+        label:
+          en_US: Japan
+          zh_Hans: 日本
+          pt_BR: Japan
+      - value: IN
+        label:
+          en_US: India
+          zh_Hans: 印度
+          pt_BR: India
+      - value: KR
+        label:
+          en_US: Korea
+          zh_Hans: 韩国
+          pt_BR: Korea
+      - value: SG
+        label:
+          en_US: Singapore
+          zh_Hans: 新加坡
+          pt_BR: Singapore
+      - value: SE
+        label:
+          en_US: Sweden
+          zh_Hans: 瑞典
+          pt_BR: Sweden

api/core/tools/provider/builtin/websearch/tools/job_search.py

@@ -0,0 +1,86 @@
+from typing import Any, Union
+from urllib.parse import urlencode
+
+import requests
+
+from core.tools.entities.tool_entities import ToolInvokeMessage
+from core.tools.tool.builtin_tool import BuiltinTool
+
+BASE_URL = "https://api.serply.io/v1/news/"
+
+
+class SerplyApi:
+    """
+    SerplyAPI tool provider.
+    """
+
+    def __init__(self, api_key: str) -> None:
+        """Initialize SerplyAPI tool provider."""
+        self.serply_api_key = api_key
+
+    def run(self, query: str, **kwargs: Any) -> str:
+        """Run query through SerplyAPI and parse result."""
+        params = {"q": query, "hl": kwargs.get("hl", "en"), "gl": kwargs.get("gl", "US"), "num": kwargs.get("num", 10)}
+        location = kwargs.get("location", "US")
+
+        headers = {
+            "X-API-KEY": self.serply_api_key,
+            "X-User-Agent": kwargs.get("device", "desktop"),
+            "X-Proxy-Location": location,
+            "User-Agent": "Dify",
+        }
+
+        url = f"{BASE_URL}{urlencode(params)}"
+        res = requests.get(
+            url,
+            headers=headers,
+        )
+        res = res.json()
+
+        return self.parse_results(res)
+
+    @staticmethod
+    def parse_results(res: dict) -> str:
+        """Process response from Serply Job Search."""
+        jobs = res.get("jobs", [])
+        if not jobs:
+            raise ValueError(f"Got error from Serply: {res}")
+
+        string = []
+        for job in jobs[:10]:
+            try:
+                string.append(
+                    "\n".join([
+                        f"Position: {job['position']}",
+                        f"Employer: {job['employer']}",
+                        f"Location: {job['location']}",
+                        f"Link: {job['link']}",
+                        f"""Highest: {", ".join([h for h in job["highlights"]])}""",
+                        "---",
+                    ])
+                )
+            except KeyError:
+                continue
+
+        content = "\n".join(string)
+        return f"\nJobs results:\n {content}\n"
+
+
+class JobSearchTool(BuiltinTool):
+    def _invoke(
+        self,
+        user_id: str,
+        tool_parameters: dict[str, Any],
+    ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
+        """
+        Invoke the SerplyApi tool.
+        """
+        query = tool_parameters["query"]
+        gl = tool_parameters.get("gl", "us")
+        hl = tool_parameters.get("hl", "en")
+        location = tool_parameters.get("location", None)
+
+        api_key = self.runtime.credentials["serply_api_key"]
+        result = SerplyApi(api_key).run(query, gl=gl, hl=hl, location=location)
+
+        return self.create_text_message(text=result)

api/core/tools/provider/builtin/websearch/tools/job_search.yaml

@@ -0,0 +1,41 @@
+identity:
+  name: job_search
+  author: Dify
+  label:
+    en_US: Job Search API
+    zh_Hans: Job Search API
+description:
+  human:
+    en_US: A tool to retrieve job titles, company names and description from Google Jobs engine.
+    zh_Hans: 一个从 Google 招聘引擎检索职位名称、公司名称和描述的工具。
+  llm: A tool to retrieve job titles, company names and description from Google Jobs engine.
+parameters:
+  - name: query
+    type: string
+    required: true
+    label:
+      en_US: Query
+      zh_Hans: 询问
+    human_description:
+      en_US: Defines the query you want to search.
+      zh_Hans: 定义您要搜索的查询。
+    llm_description: Defines the search query you want to search.
+    form: llm
+  - name: location
+    type: string
+    required: false
+    default: US
+    label:
+      en_US: Location
+      zh_Hans: 询问
+    human_description:
+      en_US: Defines from where you want the search to originate. (For example - New York)
+      zh_Hans: 定义您想要搜索的起始位置。 （例如 - 纽约）
+    llm_description: Defines from where you want the search to originate. (For example - New York)
+    form: form
+    options:
+      - value: US
+        label:
+          en_US: United States
+          zh_Hans: 美国
+          pt_BR: United States

api/core/tools/provider/builtin/websearch/tools/news_search.py

@@ -0,0 +1,88 @@
+from typing import Any, Union
+from urllib.parse import urlencode
+
+import requests
+
+from core.tools.entities.tool_entities import ToolInvokeMessage
+from core.tools.tool.builtin_tool import BuiltinTool
+
+BASE_URL = "https://api.serply.io/v1/news/"
+
+
+class SerplyApi:
+    """
+    SerplyApi tool provider.
+    """
+
+    def __init__(self, api_key: str) -> None:
+        """Initialize SerplyApi tool provider."""
+        self.serply_api_key = api_key
+
+    def run(self, query: str, **kwargs: Any) -> str:
+        """Run query through SerplyApi and parse result."""
+        params = {"q": query, "hl": kwargs.get("hl", "en"), "gl": kwargs.get("gl", "US"), "num": kwargs.get("num", 10)}
+        location = kwargs.get("location", "US")
+
+        headers = {
+            "X-API-KEY": self.serply_api_key,
+            "X-User-Agent": kwargs.get("device", "desktop"),
+            "X-Proxy-Location": location,
+            "User-Agent": "Dify",
+        }
+
+        url = f"{BASE_URL}{urlencode(params)}"
+        res = requests.get(
+            url,
+            headers=headers,
+        )
+        res = res.json()
+
+        return self.parse_results(res)
+
+    @staticmethod
+    def parse_results(res: dict) -> str:
+        """Process response from Serply News Search."""
+        news = res.get("entries", [])
+        if not news:
+            raise ValueError(f"Got error from Serply: {res}")
+
+        string = []
+        for entry in news:
+            try:
+                # follow url
+                r = requests.get(entry["link"])
+                final_link = r.history[-1].headers["Location"]
+                string.append(
+                    "\n".join([
+                        f"Title: {entry['title']}",
+                        f"Link: {final_link}",
+                        f"Source: {entry['source']['title']}",
+                        f"Published: {entry['published']}",
+                        "---",
+                    ])
+                )
+            except KeyError:
+                continue
+
+        content = "\n".join(string)
+        return f"\nNews:\n {content}\n"
+
+
+class NewsSearchTool(BuiltinTool):
+    def _invoke(
+        self,
+        user_id: str,
+        tool_parameters: dict[str, Any],
+    ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
+        """
+        Invoke the SerplyApi tool.
+        """
+        query = tool_parameters["query"]
+        gl = tool_parameters.get("gl", "us")
+        hl = tool_parameters.get("hl", "en")
+        location = tool_parameters.get("location", None)
+
+        api_key = self.runtime.credentials["serply_api_key"]
+        result = SerplyApi(api_key).run(query, gl=gl, hl=hl, location=location)
+
+        return self.create_text_message(text=result)