Merge release/0.8 to master

.github/workflows/publish.yaml

@@ -11,7 +11,7 @@ concurrency:
 
 jobs:
   build-n-publish:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     #if: startsWith(github.event.ref, 'refs/tags')
     steps:
       - uses: actions/checkout@v2

apps/mcp-playground/env.py

@@ -1,16 +1,17 @@
 import os
+
 import json
 
 is_cn_env = os.getenv('MODELSCOPE_ENVIRONMENT') == 'studio'
 
 api_key = os.getenv('MODELSCOPE_API_KEY')
 
 internal_mcp_config = json.loads(
-    os.getenv("INTERNAL_MCP_CONFIG", '{"mcpServers": {}}'))
+    os.getenv('INTERNAL_MCP_CONFIG', '{"mcpServers": {}}'))
 
 # oss
-endpoint = os.getenv("OSS_ENDPOINT")
+endpoint = os.getenv('OSS_ENDPOINT')
 
-region = os.getenv("OSS_REGION")
+region = os.getenv('OSS_REGION')
 
-bucket_name = os.getenv("OSS_BUCKET_NAME")
+bucket_name = os.getenv('OSS_BUCKET_NAME')

apps/mcp-playground/tools/oss.py

@@ -1,24 +1,24 @@
-from http import HTTPStatus
 import uuid
+from http import HTTPStatus
+
 import oss2
+from env import bucket_name, endpoint, region
 from oss2.credentials import EnvironmentVariableCredentialsProvider
-from env import endpoint, bucket_name, region
+
 # OSS_ACCESS_KEY_ID and OSS_ACCESS_KEY_SECRET。
 auth = oss2.ProviderAuthV4(EnvironmentVariableCredentialsProvider())
 
 bucket = oss2.Bucket(auth, endpoint, bucket_name, region=region)
 
 
 def file_path_to_oss_url(file_path: str):
-    if file_path.startswith("http"):
+    if file_path.startswith('http'):
         return file_path
     ext = file_path.split('.')[-1]
     object_name = f'studio-temp/mcp-playground/{uuid.uuid4()}.{ext}'
     response = bucket.put_object_from_file(object_name, file_path)
     file_url = file_path
     if response.status == HTTPStatus.OK:
-        file_url = bucket.sign_url('GET',
-                                   object_name,
-                                   60 * 60,
-                                   slash_safe=True)
+        file_url = bucket.sign_url(
+            'GET', object_name, 60 * 60, slash_safe=True)
     return file_url

apps/mcp-playground/ui_components/add_mcp_server_button.py

@@ -1,45 +1,50 @@
-import modelscope_studio.components.antd as antd
-import modelscope_studio.components.base as ms
+# flake8: noqa: F401
 import gradio as gr
 import json
+import modelscope_studio.components.antd as antd
+import modelscope_studio.components.base as ms
 
 
 def AddMcpServerButton():
-    with antd.Button("添加 MCP Server", type="primary",
-                     size="small") as add_mcp_server_btn:
-        with ms.Slot("icon"):
-            antd.Icon("PlusOutlined")
+    with antd.Button(
+            '添加 MCP Server', type='primary',
+            size='small') as add_mcp_server_btn:
+        with ms.Slot('icon'):
+            antd.Icon('PlusOutlined')
     with antd.Modal(
-            title="添加 MCP Server",
+            title='添加 MCP Server',
             footer=False,
-            styles=dict(footer=dict(display="none"))) as add_mcp_server_modal:
+            styles=dict(footer=dict(display='none'))) as add_mcp_server_modal:
         with antd.Tabs():
-            with antd.Tabs.Item(label="表单添加"):
-                with antd.Form(layout="vertical") as add_mcp_server_form:
-                    with antd.Form.Item(form_name="name",
-                                        label="名称",
-                                        rules=[{
-                                            "required": True
-                                        }]):
-                        antd.Input(placeholder="MCP Server 名称，如 fetch、time 等")
-                    with antd.Form.Item(form_name="url",
-                                        label="SSE 链接",
-                                        rules=[{
-                                            "required": True
-                                        }]):
-                        antd.Input(placeholder="MCP Server SSE 链接")
-                    with antd.Flex(gap="small", justify="end"):
-                        add_mcp_server_modal_cancel_btn = antd.Button("取消")
-                        antd.Button("确定", html_type="submit", type="primary")
-            with antd.Tabs.Item(label="JSON 添加"):
-                with antd.Form(layout="vertical") as add_mcp_server_json_form:
-                    with antd.Form.Item(form_name="json",
-                                        label="JSON",
-                                        rules=[{
-                                            "required":
-                                            True,
-                                            "validator":
-                                            """(_, value) => {
+            with antd.Tabs.Item(label='表单添加'):
+                with antd.Form(layout='vertical') as add_mcp_server_form:
+                    with antd.Form.Item(
+                            form_name='name',
+                            label='名称',
+                            rules=[{
+                                'required': True
+                            }]):
+                        antd.Input(placeholder='MCP Server 名称，如 fetch、time 等')
+                    with antd.Form.Item(
+                            form_name='url',
+                            label='SSE 链接',
+                            rules=[{
+                                'required': True
+                            }]):
+                        antd.Input(placeholder='MCP Server SSE 链接')
+                    with antd.Flex(gap='small', justify='end'):
+                        add_mcp_server_modal_cancel_btn = antd.Button('取消')
+                        antd.Button('确定', html_type='submit', type='primary')
+            with antd.Tabs.Item(label='JSON 添加'):
+                with antd.Form(layout='vertical') as add_mcp_server_json_form:
+                    with antd.Form.Item(
+                            form_name='json',
+                            label='JSON',
+                            rules=[{
+                                'required':
+                                True,
+                                'validator':
+                                """(_, value) => {
                                                 if (!value) {
                                                     return Promise.reject('请输入 JSON 值');
                                                 }
@@ -53,40 +58,43 @@ def AddMcpServerButton():
                                                     return Promise.reject('请输入有效的 JSON 值');
                                                 }
                                             } """
-                                        }]):
+                            }]):
                         antd.Input.Textarea(
                             auto_size=dict(minRows=4, maxRows=8),
                             placeholder=json.dumps(
                                 {
-                                    "mcpServers": {
-                                        "fetch": {
-                                            "type": "sse",
-                                            "url": "mcp server sse url"
+                                    'mcpServers': {
+                                        'fetch': {
+                                            'type': 'sse',
+                                            'url': 'mcp server sse url'
                                         }
                                     }
                                 },
                                 indent=4))
-                    with antd.Flex(gap="small", justify="end"):
+                    with antd.Flex(gap='small', justify='end'):
                         add_mcp_server_modal_json_cancel_btn = antd.Button(
-                            "取消")
-                        antd.Button("确定", html_type="submit", type="primary")
-    add_mcp_server_btn.click(fn=lambda: gr.update(open=True),
-                             outputs=[add_mcp_server_modal],
-                             queue=False)
-    gr.on(triggers=[
-        add_mcp_server_modal_cancel_btn.click,
-        add_mcp_server_modal_json_cancel_btn.click, add_mcp_server_modal.cancel
-    ],
-          queue=False,
-          fn=lambda: gr.update(open=False),
-          outputs=[add_mcp_server_modal])
+                            '取消')
+                        antd.Button('确定', html_type='submit', type='primary')
+    add_mcp_server_btn.click(
+        fn=lambda: gr.update(open=True),
+        outputs=[add_mcp_server_modal],
+        queue=False)
+    gr.on(
+        triggers=[
+            add_mcp_server_modal_cancel_btn.click,
+            add_mcp_server_modal_json_cancel_btn.click,
+            add_mcp_server_modal.cancel
+        ],
+        queue=False,
+        fn=lambda: gr.update(open=False),
+        outputs=[add_mcp_server_modal])
     add_mcp_server_form.finish(
         lambda: (gr.update(value={
-            "name": "",
-            "url": "",
+            'name': '',
+            'url': '',
         }), gr.update(open=False)),
         outputs=[add_mcp_server_form, add_mcp_server_modal])
     add_mcp_server_json_form.finish(
-        lambda: (gr.update(value={"json": ""}), gr.update(open=False)),
+        lambda: (gr.update(value={'json': ''}), gr.update(open=False)),
         outputs=[add_mcp_server_json_form, add_mcp_server_modal])
     return add_mcp_server_form, add_mcp_server_json_form

modelscope_agent/tools/base.py

@@ -7,7 +7,6 @@
 import json
 import json5
 import requests
-
 from modelscope_agent.constants import (BASE64_FILES,
                                         DEFAULT_CODE_INTERPRETER_DIR,
                                         DEFAULT_TOOL_MANAGER_SERVICE_URL,

modelscope_agent/tools/mcp/mcp_client.py

@@ -250,17 +250,17 @@ def generate_config(mcp_servers: Dict[str, Any]) -> Dict[str, Any]:
         for mcp_server in mcp_servers:
 
             # Activate local planner `notebook`
-            if mcp_server == 'notebook':
+            if mcp_server in ['notebook', 'crawl4ai']:
                 mcp_path = os.path.dirname(os.path.abspath(__file__))
-                notebook_mcp_path = os.path.join(mcp_path, 'servers',
-                                                 mcp_server)
-                notebook_mcp_config_path = os.path.join(
-                    notebook_mcp_path, 'config.json')
-                if os.path.exists(notebook_mcp_config_path):
+                buildin_mcp_path = os.path.join(mcp_path, 'servers',
+                                                mcp_server)
+                buildin_mcp_config_path = os.path.join(buildin_mcp_path,
+                                                       'config.json')
+                if os.path.exists(buildin_mcp_config_path):
                     print(
-                        f'Got local planner `notebook`: {notebook_mcp_config_path}'
+                        f'Got local planner `notebook`: {buildin_mcp_config_path}'
                     )
-                    with open(notebook_mcp_config_path, 'r') as f:
+                    with open(buildin_mcp_config_path, 'r') as f:
                         content = json.load(f)
                         mcp_content = content[mcp_server]
                         command = mcp_content['command']
@@ -284,7 +284,7 @@ def generate_config(mcp_servers: Dict[str, Any]) -> Dict[str, Any]:
                         for idx in range(len(args)):
                             if 'server.py' in args[idx]:
                                 args[idx] = os.path.join(
-                                    notebook_mcp_path, 'server.py')
+                                    buildin_mcp_path, 'server.py')
 
                     mcp_servers[mcp_server] = mcp_content
 

modelscope_agent/tools/mcp/servers/crawl4ai/README.md

@@ -0,0 +1,19 @@
+# How-to-use
+
+## Installation
+
+```shell
+pip install -r requirements.txt
+crawl4ai-setup
+crawl4ai-doctor
+```
+
+## Dev
+
+```shell
+fastmcp dev server.py
+```
+
+## Run
+
+Please copy the content of config.json and change the path to your actual local file path

modelscope_agent/tools/mcp/servers/crawl4ai/config.json

@@ -0,0 +1,9 @@
+{
+    "crawl4ai": {
+      "command": "/path/to/fastmcp",
+      "args": [
+        "run",
+        "/path/to/crawl4ai/server.py"
+      ]
+    }
+}

modelscope_agent/tools/mcp/servers/crawl4ai/requirements.txt

@@ -0,0 +1,3 @@
+crawl4ai
+fastmcp
+trafilatura

modelscope_agent/tools/mcp/servers/crawl4ai/server.py

@@ -0,0 +1,73 @@
+import json
+import trafilatura
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+from crawl4ai.browser_manager import BrowserManager
+from fastmcp import FastMCP
+
+mcp = FastMCP('crawl4ai')
+
+
+async def __aexit__(self, exc_type, exc_val, exc_tb):
+    await self.close()
+    # Fix: https://github.com/unclecode/crawl4ai/issues/842
+    BrowserManager._playwright_instance = None
+
+
+AsyncPlaywrightCrawlerStrategy.__aexit__ = __aexit__
+
+
+@mcp.tool(
+    description='A crawl tool to get the content of a website page, '
+    'and simplify the content to pure html content. This tool can be used to get the detail '
+    'information in the url')
+async def crawl_website(website: str) -> str:
+    if not website.startswith('http'):
+        website = 'http://' + website
+    try:
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(url=website, )
+            html = str(result.html)
+            html = trafilatura.extract(
+                html,
+                deduplicate=True,
+                favor_precision=True,
+                include_comments=False,
+                output_format='markdown',
+                with_metadata=True,
+            )
+            if not html:
+                html = 'Cannot crawl this web page, please try another web page instead'
+            if len(html) > 2048:
+                html = html[:2048]
+            output = {'text': html}
+            media_list = []
+            if result.media:
+                for key in result.media:
+                    media_dict = result.media[key]
+                    for idx, row in enumerate(media_dict):
+                        if idx > 20:
+                            break
+                        src = row['src'] or ''
+                        if src and not src.startswith('http'):
+                            src = src.lstrip('/')
+                            src = 'https://' + src
+                        media_list.append({
+                            'type':
+                            key,
+                            'description':
+                            row['alt'][:100] or row['desc'][:100]
+                            or 'No description',
+                            'link':
+                            src,
+                        })
+                output['media'] = media_list
+            return json.dumps(output, ensure_ascii=False)
+    except Exception:
+        import traceback
+        print(traceback.format_exc())
+        return 'Cannot crawl this web page, please try another web page instead'
+
+
+if __name__ == '__main__':
+    mcp.run(transport='stdio')

setup.py

@@ -42,4 +42,7 @@ def get_version():
     install_requires=parse_requirements('requirements.txt'),
     long_description=readme(),
     long_description_content_type='text/markdown',
+    package_data={
+        'modelscope_agent.tools.mcp.servers.notebook': ['config.json'],
+    },
 )