update

Author: bowenliang123

.github/workflows/api-tests.yml

@@ -47,6 +47,9 @@ jobs:
       - name: Install dependencies
         run: poetry install -C api --with dev
 
+      - name: Check dependency list
+        run: poetry run -C api bash dev/pytest/pytest_artifacts.sh
+
       - name: Run Unit tests
         run: poetry run -C api bash dev/pytest/pytest_unit_tests.sh
 

api/poetry.lock

@@ -104,33 +104,31 @@ name = "dify-api"
 package-mode = false
 
 ############################################################
-# Main dependencies
+# [ Main ] Dependency group
 ############################################################
 
 [tool.poetry.dependencies]
 anthropic = "~0.23.1"
 authlib = "1.3.1"
+azure-ai-inference = "~1.0.0b3"
+azure-ai-ml = "~1.19.0"
 azure-identity = "1.16.1"
 azure-storage-blob = "12.13.0"
 beautifulsoup4 = "4.12.2"
 boto3 = "1.35.17"
-sagemaker = "2.231.0"
 bs4 = "~0.0.1"
 cachetools = "~5.3.0"
 celery = "~5.3.6"
 chardet = "~5.1.0"
 cohere = "~5.2.4"
-cos-python-sdk-v5 = "1.9.30"
-esdk-obs-python = "3.24.6.1"
-bce-python-sdk = "~0.9.23"
 dashscope = { version = "~1.17.0", extras = ["tokenizer"] }
 flask = "~3.0.1"
 flask-compress = "~1.14"
 flask-cors = "~4.0.0"
 flask-login = "~0.6.3"
 flask-migrate = "~4.0.5"
 flask-restful = "~0.3.10"
-Flask-SQLAlchemy = "~3.1.1"
+flask-sqlalchemy = "~3.1.1"
 gevent = "~23.9.1"
 gmpy2 = "~2.2.1"
 google-ai-generativelanguage = "0.6.9"
@@ -139,22 +137,22 @@ google-api-python-client = "2.90.0"
 google-auth = "2.29.0"
 google-auth-httplib2 = "0.2.0"
 google-cloud-aiplatform = "1.49.0"
-google-cloud-storage = "2.16.0"
 google-generativeai = "0.8.1"
 googleapis-common-protos = "1.63.0"
 gunicorn = "~22.0.0"
 httpx = { version = "~0.27.0", extras = ["socks"] }
 huggingface-hub = "~0.16.4"
 jieba = "0.42.1"
-langfuse = "^2.48.0"
-langsmith = "^0.1.77"
+langfuse = "~2.48.0"
+langsmith = "~0.1.77"
 mailchimp-transactional = "~1.0.50"
 markdown = "~3.5.1"
-novita-client = "^0.5.7"
+nomic = "~3.1.2"
+novita-client = "~0.5.7"
 numpy = "~1.26.4"
+oci = "~2.133.0"
 openai = "~1.29.0"
 openpyxl = "~3.1.5"
-oss2 = "2.18.5"
 pandas = { version = "~2.2.2", extras = ["performance", "excel"] }
 psycopg2-binary = "~2.9.6"
 pycryptodome = "3.19.1"
@@ -171,14 +169,17 @@ readabilipy = "0.2.0"
 redis = { version = "~5.0.3", extras = ["hiredis"] }
 replicate = "~0.22.0"
 resend = "~0.7.0"
-scikit-learn = "^1.5.1"
+sagemaker = "2.231.0"
+scikit-learn = "~1.5.1"
 sentry-sdk = { version = "~1.44.1", extras = ["flask"] }
 sqlalchemy = "~2.0.29"
 tencentcloud-sdk-python-hunyuan = "~3.0.1158"
 tiktoken = "~0.7.0"
 tokenizers = "~0.15.0"
 transformers = "~4.35.0"
 unstructured = { version = "~0.10.27", extras = ["docx", "epub", "md", "msg", "ppt", "pptx"] }
+validators = "0.21.0"
+volcengine-python-sdk = {extras = ["ark"], version = "~1.0.98"}
 websocket-client = "~1.7.0"
 werkzeug = "~3.0.1"
 xinference-client = "0.15.2"
@@ -187,25 +188,18 @@ zhipuai = "1.0.7"
 # Before adding new dependency, consider place it in alphabet order (a-z) and suitable group.
 
 ############################################################
+# [ Indirect ] Dependency group
 # Related transparent dependencies with pinned version
 # required by main implementations
 ############################################################
-azure-ai-ml = "^1.19.0"
-azure-ai-inference = "^1.0.0b3"
-volcengine-python-sdk = {extras = ["ark"], version = "^1.0.98"}
-oci = "^2.133.0"
-tos = "^2.7.1"
-nomic = "^3.1.2"
-validators = "0.21.0"
-[tool.poetry.group.indriect.dependencies]
+[tool.poetry.group.indirect.dependencies]
 kaleido = "0.2.1"
 rank-bm25 = "~0.2.2"
 safetensors = "~0.4.3"
 
 ############################################################
-# Tool dependencies required by tool implementations
+# [ Tool ] Dependency group
 ############################################################
-
 [tool.poetry.group.tool.dependencies]
 arxiv = "2.1.0"
 cloudscraper = "1.2.71"
@@ -221,10 +215,23 @@ vanna = { version = "0.5.5", extras = ["postgres", "mysql", "clickhouse", "duckd
 wikipedia = "1.4.0"
 yfinance = "~0.2.40"
 nltk = "3.8.1"
+
 ############################################################
-# VDB dependencies required by vector store clients
+# [ Storage ] Dependency group
+# Required for storage clients
 ############################################################
+[tool.poetry.group.storage.dependencies]
+bce-python-sdk = "~0.9.23"
+cos-python-sdk-v5 = "1.9.30"
+esdk-obs-python = "3.24.6.1"
+google-cloud-storage = "2.16.0"
+oss2 = "2.18.5"
+tos = "~2.7.1"
 
+############################################################
+# [ VDB ] Dependency group
+# Required by vector store clients
+############################################################
 [tool.poetry.group.vdb.dependencies]
 alibabacloud_gpdb20160503 = "~3.8.0"
 alibabacloud_tea_openapi = "~0.3.9"
@@ -235,18 +242,17 @@ oracledb = "~2.2.1"
 pgvecto-rs = { version = "~0.2.1", extras = ['sqlalchemy'] }
 pgvector = "0.2.5"
 pymilvus = "~2.4.4"
+qdrant-client = "1.7.3"
 tcvectordb = "1.3.2"
 tidb-vector = "0.0.9"
-qdrant-client = "1.7.3"
 weaviate-client = "~3.21.0"
 
 ############################################################
-# Dev dependencies for running tests
+# [ Dev ] Dependency group
+# Required for development and running tests
 ############################################################
-
 [tool.poetry.group.dev]
 optional = true
-
 [tool.poetry.group.dev.dependencies]
 coverage = "~7.2.4"
 pytest = "~8.3.2"
@@ -255,12 +261,11 @@ pytest-env = "~1.1.3"
 pytest-mock = "~3.14.0"
 
 ############################################################
-# Lint dependencies for code style linting
+# [ Lint ] Dependency group
+# Required for code style linting
 ############################################################
-
 [tool.poetry.group.lint]
 optional = true
-
 [tool.poetry.group.lint.dependencies]
 dotenv-linter = "~0.5.0"
 ruff = "~0.6.9"

api/pyproject.toml

@@ -0,0 +1,21 @@
+import toml
+
+
+def test_group_dependencies_sorted():
+    pyproject_toml = toml.load("api/pyproject.toml")
+    poetry_settings = pyproject_toml.get("tool").get("poetry")
+    groups = ["", "indirect", "dev", "lint", "storage", "vdb"]
+    all_dependency_names: list[str] = []
+    for group_name in groups:
+        dependencies = poetry_settings.get("group").get(group_name) if group_name else poetry_settings
+        section = f"tool.poetry.group.{group_name}.dependencies" if group_name else "tool.poetry.dependencies"
+        dependency_names = list(dependencies.get("dependencies"))
+        all_dependency_names.extend(dependency_names)
+        expected_dependency_names = sorted(set(dependency_names))
+        assert expected_dependency_names == dependency_names, (
+            f"Dependencies in group {group_name} are not sorted. "
+            f"Check and fix [{section}] section in pyproject.toml file"
+        )
+
+    expected_all_names = sorted(set(all_dependency_names))
+    assert expected_all_names == sorted(all_dependency_names), "Duplicated dependencies crossing groups are found"

api/tests/artifact_tests/dependencies/__init__.py

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -x
+
+pytest api/tests/artifact_tests/