Merge branch 'langgenius:main' into main

api/.env.example

@@ -1,6 +1,3 @@
-# Server Edition
-EDITION=SELF_HOSTED
-
 # Your App secret key will be used for securely signing the session cookie
 # Make sure you are changing this key for your deployment with a strong key.
 # You can generate a strong key using `openssl rand -base64 42`.
@@ -123,25 +120,6 @@ NOTION_CLIENT_SECRET=you-client-secret
 NOTION_CLIENT_ID=you-client-id
 NOTION_INTERNAL_SECRET=you-internal-secret
 
-# Hosted Model Credentials
-HOSTED_OPENAI_API_KEY=
-HOSTED_OPENAI_API_BASE=
-HOSTED_OPENAI_API_ORGANIZATION=
-HOSTED_OPENAI_TRIAL_ENABLED=false
-HOSTED_OPENAI_QUOTA_LIMIT=200
-HOSTED_OPENAI_PAID_ENABLED=false
-
-HOSTED_AZURE_OPENAI_ENABLED=false
-HOSTED_AZURE_OPENAI_API_KEY=
-HOSTED_AZURE_OPENAI_API_BASE=
-HOSTED_AZURE_OPENAI_QUOTA_LIMIT=200
-
-HOSTED_ANTHROPIC_API_BASE=
-HOSTED_ANTHROPIC_API_KEY=
-HOSTED_ANTHROPIC_TRIAL_ENABLED=false
-HOSTED_ANTHROPIC_QUOTA_LIMIT=600000
-HOSTED_ANTHROPIC_PAID_ENABLED=false
-
 ETL_TYPE=dify
 UNSTRUCTURED_API_URL=
 

api/app.py

@@ -1,28 +1,28 @@
 import os
-import sys
-from logging.handlers import RotatingFileHandler
 
 if not os.environ.get("DEBUG") or os.environ.get("DEBUG").lower() != 'true':
     from gevent import monkey
 
     monkey.patch_all()
-    # if os.environ.get("VECTOR_STORE") == 'milvus':
+
     import grpc.experimental.gevent
 
     grpc.experimental.gevent.init_gevent()
 
 import json
 import logging
+import sys
 import threading
 import time
 import warnings
+from logging.handlers import RotatingFileHandler
 
 from flask import Flask, Response, request
 from flask_cors import CORS
 from werkzeug.exceptions import Unauthorized
 
 from commands import register_commands
-from config import CloudEditionConfig, Config
+from config import Config
 
 # DO NOT REMOVE BELOW
 from events import event_handlers
@@ -75,16 +75,9 @@ class DifyApp(Flask):
 # ----------------------------
 
 
-def create_app(test_config=None) -> Flask:
+def create_app() -> Flask:
     app = DifyApp(__name__)
-
-    if test_config:
-        app.config.from_object(test_config)
-    else:
-        if config_type == "CLOUD":
-            app.config.from_object(CloudEditionConfig())
-        else:
-            app.config.from_object(Config())
+    app.config.from_object(Config())
 
     app.secret_key = app.config['SECRET_KEY']
 
@@ -101,6 +94,7 @@ def create_app(test_config=None) -> Flask:
             ),
             logging.StreamHandler(sys.stdout)
         ]
+
     logging.basicConfig(
         level=app.config.get('LOG_LEVEL'),
         format=app.config.get('LOG_FORMAT'),

api/config.py

@@ -5,6 +5,7 @@
 dotenv.load_dotenv()
 
 DEFAULTS = {
+    'EDITION': 'SELF_HOSTED',
     'DB_USERNAME': 'postgres',
     'DB_PASSWORD': '',
     'DB_HOST': 'localhost',
@@ -106,7 +107,7 @@ def __init__(self):
         # ------------------------
         self.CURRENT_VERSION = "0.6.5"
         self.COMMIT_SHA = get_env('COMMIT_SHA')
-        self.EDITION = "SELF_HOSTED"
+        self.EDITION = get_env('EDITION')
         self.DEPLOY_ENV = get_env('DEPLOY_ENV')
         self.TESTING = False
         self.LOG_LEVEL = get_env('LOG_LEVEL')
@@ -260,7 +261,7 @@ def __init__(self):
         self.SMTP_USE_TLS = get_bool_env('SMTP_USE_TLS')
 
         # ------------------------
-        # Workpace Configurations.
+        # Workspace Configurations.
         # ------------------------
         self.INVITE_EXPIRY_HOURS = int(get_env('INVITE_EXPIRY_HOURS'))
 
@@ -299,6 +300,12 @@ def __init__(self):
         # ------------------------
         # Platform Configurations.
         # ------------------------
+        self.GITHUB_CLIENT_ID = get_env('GITHUB_CLIENT_ID')
+        self.GITHUB_CLIENT_SECRET = get_env('GITHUB_CLIENT_SECRET')
+        self.GOOGLE_CLIENT_ID = get_env('GOOGLE_CLIENT_ID')
+        self.GOOGLE_CLIENT_SECRET = get_env('GOOGLE_CLIENT_SECRET')
+        self.OAUTH_REDIRECT_PATH = get_env('OAUTH_REDIRECT_PATH')
+
         self.HOSTED_OPENAI_API_KEY = get_env('HOSTED_OPENAI_API_KEY')
         self.HOSTED_OPENAI_API_BASE = get_env('HOSTED_OPENAI_API_BASE')
         self.HOSTED_OPENAI_API_ORGANIZATION = get_env('HOSTED_OPENAI_API_ORGANIZATION')
@@ -345,17 +352,3 @@ def __init__(self):
 
         self.KEYWORD_DATA_SOURCE_TYPE = get_env('KEYWORD_DATA_SOURCE_TYPE')
         self.ENTERPRISE_ENABLED = get_bool_env('ENTERPRISE_ENABLED')
-
-
-class CloudEditionConfig(Config):
-
-    def __init__(self):
-        super().__init__()
-
-        self.EDITION = "CLOUD"
-
-        self.GITHUB_CLIENT_ID = get_env('GITHUB_CLIENT_ID')
-        self.GITHUB_CLIENT_SECRET = get_env('GITHUB_CLIENT_SECRET')
-        self.GOOGLE_CLIENT_ID = get_env('GOOGLE_CLIENT_ID')
-        self.GOOGLE_CLIENT_SECRET = get_env('GOOGLE_CLIENT_SECRET')
-        self.OAUTH_REDIRECT_PATH = get_env('OAUTH_REDIRECT_PATH')

api/core/rag/datasource/vdb/vector_base.py

@@ -27,6 +27,12 @@ def text_exists(self, id: str) -> bool:
     def delete_by_ids(self, ids: list[str]) -> None:
         raise NotImplementedError
 
+    def delete_by_document_id(self, document_id: str):
+        raise NotImplementedError
+
+    def get_ids_by_metadata_field(self, key: str, value: str):
+        raise NotImplementedError
+
     @abstractmethod
     def delete_by_metadata_field(self, key: str, value: str) -> None:
         raise NotImplementedError

api/services/billing_service.py

@@ -74,5 +74,5 @@ def is_tenant_owner_or_admin(current_user):
             TenantAccountJoin.account_id == current_user.id
         ).first()
 
-        if TenantAccountRole.is_privileged_role(join.role):
+        if not TenantAccountRole.is_privileged_role(join.role):
             raise ValueError('Only team owner or team admin can perform this action')

api/tests/integration_tests/vdb/milvus/test_milvus.py

@@ -24,6 +24,13 @@ def search_by_full_text(self):
         hits_by_full_text = self.vector.search_by_full_text(query=get_sample_text())
         assert len(hits_by_full_text) == 0
 
+    def delete_document_by_id(self):
+        self.vector.delete_by_document_id(self.dataset_id)
+
+    def get_ids_by_metadata_field(self):
+        ids = self.vector.get_ids_by_metadata_field('document_id', self.dataset_id)
+        assert len(ids) >= 1
+
 
 def test_milvus_vector(setup_mock_redis):
-    TestMilvusVector().run_all_test()
+    TestMilvusVector().run_all_tests()

api/tests/integration_tests/vdb/qdrant/test_qdrant.py

@@ -20,4 +20,4 @@ def __init__(self):
 
 
 def test_qdrant_vector(setup_mock_redis):
-    TestQdrantVector().run_all_test()
+    TestQdrantVector().run_all_tests()

api/tests/integration_tests/vdb/test_vector_store.py

@@ -71,8 +71,39 @@ def search_by_full_text(self):
     def delete_vector(self):
         self.vector.delete()
 
-    def run_all_test(self):
+    def delete_by_ids(self):
+        self.vector.delete_by_ids([self.dataset_id])
+
+    def add_texts(self):
+        self.vector.add_texts(
+            documents=[
+                get_sample_document(str(uuid.uuid4())),
+                get_sample_document(str(uuid.uuid4())),
+            ],
+            embeddings=[
+                get_sample_embedding(),
+                get_sample_embedding(),
+            ],
+        )
+
+    def text_exists(self):
+        self.vector.text_exists(self.dataset_id)
+
+    def delete_document_by_id(self):
+        with pytest.raises(NotImplementedError):
+            self.vector.delete_by_document_id(self.dataset_id)
+
+    def get_ids_by_metadata_field(self):
+        with pytest.raises(NotImplementedError):
+            self.vector.get_ids_by_metadata_field('key', 'value')
+
+    def run_all_tests(self):
         self.create_vector()
         self.search_by_vector()
         self.search_by_full_text()
+        self.text_exists()
+        self.get_ids_by_metadata_field()
+        self.add_texts()
+        self.delete_document_by_id()
+        self.delete_by_ids()
         self.delete_vector()

api/tests/integration_tests/vdb/weaviate/test_weaviate.py

@@ -21,4 +21,4 @@ def __init__(self):
 
 
 def test_weaviate_vector(setup_mock_redis):
-    TestWeaviateVector().run_all_test()
+    TestWeaviateVector().run_all_tests()

docker/docker-compose.yaml

@@ -247,7 +247,6 @@ services:
     image: langgenius/dify-web:0.6.5
     restart: always
     environment:
-      EDITION: SELF_HOSTED
       # The base URL of console application api server, refers to the Console base URL of WEB service if console domain is
       # different from api or web app domain.
       # example: http://cloud.dify.ai

web/.env.example

@@ -1,6 +1,6 @@
 # For production release, change this to PRODUCTION
 NEXT_PUBLIC_DEPLOY_ENV=DEVELOPMENT
-# The deployment edition, SELF_HOSTED or CLOUD
+# The deployment edition, SELF_HOSTED
 NEXT_PUBLIC_EDITION=SELF_HOSTED
 # The base URL of console application, refers to the Console base URL of WEB service if console domain is
 # different from api or web app domain.

web/README.md

@@ -17,7 +17,7 @@ Then, configure the environment variables. Create a file named `.env.local` in t
 ```
 # For production release, change this to PRODUCTION
 NEXT_PUBLIC_DEPLOY_ENV=DEVELOPMENT
-# The deployment edition, SELF_HOSTED or CLOUD
+# The deployment edition, SELF_HOSTED
 NEXT_PUBLIC_EDITION=SELF_HOSTED
 # The base URL of console application, refers to the Console base URL of WEB service if console domain is
 # different from api or web app domain.