Merge branch 'refs/heads/feat/cleaning_ops_trace' into deploy/dev

* refs/heads/feat/cleaning_ops_trace: (23 commits) feat: update migrate file fix: generate name trace info error feat: undo/redo for workflow editor (#3927) fix: populate app configs to system environment variables (#5590) fix: DuckDuckGo image search tool error (#5606) delete the deprecated method (#5612) feat: update langfuse trace structure fix: context icon in chat (#5604) fix: Modify the incorrect configuration name for Google storage (#5595) feat: update issue template (#5592) feat: update workflow trace chore: fix typo in config descriptions (#5585) optimize feat: optimize file_list fix fix _invoke_error_mapping of tongyi tts optimize error msg feat: update suggested question trace manager fix: add support for FILE type in ToolParameterConverter (#5578) Fix/single run panel show parent scrollbar (#5574) ... # Conflicts: # api/controllers/console/datasets/error.py # api/services/workflow_service.py # web/i18n/en-US/workflow.ts # web/i18n/zh-Hans/workflow.ts
langgenius · Jun 27, 2024 · 3ba1c1c · 3ba1c1c
2 parents b4bbd16 + 85ee3a3
commit 3ba1c1c
Show file tree

Hide file tree

Showing 104 changed files with 2,543 additions and 1,080 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -14,6 +14,8 @@ body:
           required: true
         - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
+        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
+          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
 

diff --git a/.github/ISSUE_TEMPLATE/document_issue.yml b/.github/ISSUE_TEMPLATE/document_issue.yml
@@ -12,6 +12,8 @@ body:
           required: true
         - label: I confirm that I am using English to submit report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
+        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
+          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: textarea

diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml
@@ -12,6 +12,8 @@ body:
           required: true
         - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
+        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
+          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: textarea

diff --git a/.github/ISSUE_TEMPLATE/translation_issue.yml b/.github/ISSUE_TEMPLATE/translation_issue.yml
@@ -12,6 +12,8 @@ body:
           required: true
         - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
+        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
+          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: input

diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml
@@ -77,7 +77,6 @@ jobs:
             docker/docker-compose.pgvecto-rs.yaml
             docker/docker-compose.pgvector.yaml
             docker/docker-compose.chroma.yaml
-            docker/docker-compose.oracle.yaml
           services: |
             weaviate
             qdrant
@@ -87,7 +86,6 @@ jobs:
             pgvecto-rs
             pgvector
             chroma
-            oracle
 
       - name: Test Vector Stores
         run: poetry run -C api bash dev/pytest/pytest_vdb.sh
diff --git a/ArrowUturnLeft.svg b/ArrowUturnLeft.svg
diff --git a/api/.env.example b/api/.env.example
@@ -39,7 +39,7 @@ DB_DATABASE=dify
 
 # Storage configuration
 # use for store upload files, private keys...
-# storage type: local, s3, azure-blob
+# storage type: local, s3, azure-blob, google-storage
 STORAGE_TYPE=local
 STORAGE_LOCAL_PATH=storage
 S3_USE_AWS_MANAGED_IAM=false
@@ -63,7 +63,7 @@ ALIYUN_OSS_REGION=your-region
 
 # Google Storage configuration
 GOOGLE_STORAGE_BUCKET_NAME=yout-bucket-name
-GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON=your-google-service-account-json-base64-string
+GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64=your-google-service-account-json-base64-string
 
 # Tencent COS Storage configuration
 TENCENT_COS_BUCKET_NAME=your-bucket-name

diff --git a/api/app.py b/api/app.py
@@ -24,7 +24,6 @@
 from werkzeug.exceptions import Unauthorized
 
 from commands import register_commands
-from config import Config
 
 # DO NOT REMOVE BELOW
 from events import event_handlers
@@ -82,8 +81,17 @@ def create_flask_app_with_configs() -> Flask:
     with configs loaded from .env file
     """
     dify_app = DifyApp(__name__)
-    dify_app.config.from_object(Config())
     dify_app.config.from_mapping(DifyConfig().model_dump())
+
+    # populate configs into system environment variables
+    for key, value in dify_app.config.items():
+        if isinstance(value, str):
+            os.environ[key] = value
+        elif isinstance(value, int | float | bool):
+            os.environ[key] = str(value)
+        elif value is None:
+            os.environ[key] = ''
+
     return dify_app
 
 
@@ -232,7 +240,7 @@ def register_blueprints(app):
 app = create_app()
 celery = app.extensions["celery"]
 
-if app.config['TESTING']:
+if app.config.get('TESTING'):
     print("App is running in TESTING mode")
 
 

diff --git a/api/config.py b/api/config.py
diff --git a/api/configs/deploy/__init__.py b/api/configs/deploy/__init__.py
@@ -5,6 +5,16 @@ class DeploymentConfig(BaseModel):
     """
     Deployment configs
     """
+    APPLICATION_NAME: str = Field(
+        description='application name',
+        default='langgenius/dify',
+    )
+
+    TESTING: bool = Field(
+        description='',
+        default=False,
+    )
+
     EDITION: str = Field(
         description='deployment edition',
         default='SELF_HOSTED',

diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py
@@ -1,6 +1,6 @@
 from typing import Optional
 
-from pydantic import AliasChoices, BaseModel, Field, NonNegativeInt, PositiveInt
+from pydantic import AliasChoices, BaseModel, Field, NonNegativeInt, PositiveInt, computed_field
 
 from configs.feature.hosted_service import HostedServiceConfig
 
@@ -33,7 +33,7 @@ class CodeExecutionSandboxConfig(BaseModel):
     Code Execution Sandbox configs
     """
     CODE_EXECUTION_ENDPOINT: str = Field(
-        description='whether to enable HTTP response compression of gzip',
+        description='endpoint URL of code execution servcie',
         default='http://sandbox:8194',
     )
 
@@ -125,6 +125,28 @@ class HttpConfig(BaseModel):
         default=False,
     )
 
+    inner_CONSOLE_CORS_ALLOW_ORIGINS: str = Field(
+        description='',
+        validation_alias=AliasChoices('CONSOLE_CORS_ALLOW_ORIGINS', 'CONSOLE_WEB_URL'),
+        default='',
+    )
+
+    @computed_field
+    @property
+    def CONSOLE_CORS_ALLOW_ORIGINS(self) -> list[str]:
+        return self.inner_CONSOLE_CORS_ALLOW_ORIGINS.split(',')
+
+    inner_WEB_API_CORS_ALLOW_ORIGINS: Optional[str] = Field(
+        description='',
+        validation_alias=AliasChoices('WEB_API_CORS_ALLOW_ORIGINS'),
+        default='*',
+    )
+
+    @computed_field
+    @property
+    def WEB_API_CORS_ALLOW_ORIGINS(self) -> list[str]:
+        return self.inner_WEB_API_CORS_ALLOW_ORIGINS.split(',')
+
 
 class InnerAPIConfig(BaseModel):
     """

diff --git a/api/configs/middleware/__init__.py b/api/configs/middleware/__init__.py
@@ -2,7 +2,7 @@
 
 from pydantic import BaseModel, Field, NonNegativeInt, PositiveInt, computed_field
 
-from configs.middleware.redis_config import RedisConfig
+from configs.middleware.cache.redis_config import RedisConfig
 from configs.middleware.storage.aliyun_oss_storage_config import AliyunOSSStorageConfig
 from configs.middleware.storage.amazon_s3_storage_config import S3StorageConfig
 from configs.middleware.storage.azure_blob_storage_config import AzureBlobStorageConfig

diff --git a/api/configs/middleware/cache/__init__.py b/api/configs/middleware/cache/__init__.py
diff --git a/api/configs/middleware/redis_config.py → api/configs/middleware/cache/redis_config.py b/api/configs/middleware/redis_config.py → api/configs/middleware/cache/redis_config.py
diff --git a/api/configs/middleware/storage/aliyun_oss_storage_config.py b/api/configs/middleware/storage/aliyun_oss_storage_config.py
@@ -9,31 +9,31 @@ class AliyunOSSStorageConfig(BaseModel):
     """
 
     ALIYUN_OSS_BUCKET_NAME: Optional[str] = Field(
-        description='Aliyun storage ',
+        description='Aliyun OSS bucket name',
         default=None,
     )
 
     ALIYUN_OSS_ACCESS_KEY: Optional[str] = Field(
-        description='Aliyun storage access key',
+        description='Aliyun OSS access key',
         default=None,
     )
 
     ALIYUN_OSS_SECRET_KEY: Optional[str] = Field(
-        description='Aliyun storage secret key',
+        description='Aliyun OSS secret key',
         default=None,
     )
 
     ALIYUN_OSS_ENDPOINT: Optional[str] = Field(
-        description='Aliyun storage endpoint URL',
+        description='Aliyun OSS endpoint URL',
         default=None,
     )
 
     ALIYUN_OSS_REGION: Optional[str] = Field(
-        description='Aliyun storage region',
+        description='Aliyun OSS region',
         default=None,
     )
 
     ALIYUN_OSS_AUTH_VERSION: Optional[str] = Field(
-        description='Aliyun storage authentication version',
+        description='Aliyun OSS authentication version',
         default=None,
     )
diff --git a/api/configs/middleware/storage/azure_blob_storage_config.py b/api/configs/middleware/storage/azure_blob_storage_config.py
@@ -24,6 +24,6 @@ class AzureBlobStorageConfig(BaseModel):
     )
 
     AZURE_BLOB_ACCOUNT_URL: Optional[str] = Field(
-        description='Azure Blob account url',
+        description='Azure Blob account URL',
         default=None,
     )
diff --git a/api/configs/middleware/vdb/milvus_config.py b/api/configs/middleware/vdb/milvus_config.py
@@ -29,11 +29,11 @@ class MilvusConfig(BaseModel):
     )
 
     MILVUS_SECURE: bool = Field(
-        description='wheter to use SSL connection for Milvus',
+        description='whether to use SSL connection for Milvus',
         default=False,
     )
 
     MILVUS_DATABASE: str = Field(
-        description='Milvus database',
+        description='Milvus database, default to `default`',
         default='default',
     )
diff --git a/api/configs/middleware/vdb/tencent_vector_config.py b/api/configs/middleware/vdb/tencent_vector_config.py
@@ -14,12 +14,12 @@ class TencentVectorDBConfig(BaseModel):
     )
 
     TENCENT_VECTOR_DB_API_KEY: Optional[str] = Field(
-        description='Tencent Vector api key',
+        description='Tencent Vector API key',
         default=None,
     )
 
     TENCENT_VECTOR_DB_TIMEOUT: PositiveInt = Field(
-        description='Tencent Vector timeout',
+        description='Tencent Vector timeout in seconds',
         default=30,
     )
 

diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py
@@ -8,7 +8,7 @@
 from controllers.console import api
 from controllers.console.apikey import api_key_fields, api_key_list
 from controllers.console.app.error import ProviderNotInitializeError
-from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError
+from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
@@ -346,6 +346,8 @@ def post(self):
                 "in the Settings -> Model Provider.")
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
+        except Exception as e:
+            raise IndexingEstimateError(str(e))
 
         return response, 200
 

diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py
@@ -20,6 +20,7 @@
     ArchivedDocumentImmutableError,
     DocumentAlreadyFinishedError,
     DocumentIndexingError,
+    IndexingEstimateError,
     InvalidActionError,
     InvalidMetadataError,
 )
@@ -388,6 +389,8 @@ def get(self, dataset_id, document_id):
                         "in the Settings -> Model Provider.")
                 except ProviderTokenNotInitError as ex:
                     raise ProviderNotInitializeError(ex.description)
+                except Exception as e:
+                    raise IndexingEstimateError(str(e))
 
         return response
 
@@ -493,6 +496,8 @@ def get(self, dataset_id, batch):
                     "in the Settings -> Model Provider.")
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
+            except Exception as e:
+                raise IndexingEstimateError(str(e))
         return response
 
 

diff --git a/api/controllers/console/datasets/error.py b/api/controllers/console/datasets/error.py
@@ -84,3 +84,8 @@ class DatasetInUseError(BaseHTTPException):
     description = "The dataset is being used by some apps. Please remove the dataset from the apps before deleting it."
     code = 409
 
+
+class IndexingEstimateError(BaseHTTPException):
+    error_code = 'indexing_estimate_error'
+    description = "Knowledge indexing estimate failed: {message}"
+    code = 500