ci: fix ci in env

.github/workflows/test-backend.yml

@@ -95,6 +95,7 @@ jobs:
           -e POSTGRES_URL="postgres://postgres:TaskingAI321@db:5432/taskingai" \
           -e REDIS_URL="redis://cache:6379/0" \
           -e OBJECT_STORAGE_TYPE="local" \
+          -e HOST_URL=http://127.0.0.1:8080 \
           -e PATH_TO_VOLUME="./storage" \
           -e PROJECT_ID="taskingai" \
           -e TASKINGAI_INFERENCE_URL=http://inference:8000 \
@@ -107,12 +108,15 @@ jobs:
         working-directory: backend
         env:
           AES_ENCRYPTION_KEY: ${{ env.AES_ENCRYPTION_KEY }}
+          TEST_S3_SECRET_R2: ${{ secrets.TEST_S3_SECRET_R2 }}
         run: |
+          echo "$TEST_S3_SECRET_R2" > docker.env
           docker run --network taskingai-network --rm -d -p 8090:8000 \
+          --env-file docker.env \
           -e PURPOSE=API \
           -e POSTGRES_URL="postgres://postgres:TaskingAI321@db:5432/taskingai" \
           -e REDIS_URL="redis://cache:6379/0" \
-          -e OBJECT_STORAGE_TYPE="local" \
+          -e OBJECT_STORAGE_TYPE="s3" \
           -e PATH_TO_VOLUME="./storage" \
           -e PROJECT_ID="taskingai" \
           -e TASKINGAI_INFERENCE_URL=http://inference:8000 \
@@ -126,7 +130,7 @@ jobs:
           INFERENCE_PROVIDER_CREDENTIALS: ${{ secrets.INFERENCE_PROVIDER_CREDENTIALS }}
           PLUGIN_BUNDLE_CREDENTIALS: ${{ secrets.PLUGIN_BUNDLE_CREDENTIALS }}
         run: |
-          echo "$INFERENCE_PROVIDER_CREDENTIALS" > .env
+          echo "$INFERENCE_PROVIDER_CREDENTIALS" >> .env
           echo "$PLUGIN_BUNDLE_CREDENTIALS" >> .env
           sleep 10
 

backend/app/routes/file/file.py

@@ -4,9 +4,9 @@
 from app.config import CONFIG
 from app.database import boto3_client
 from app.models import UploadFilePurpose
-from app.routes.file.utils import file_purpose_dict
+from app.routes.file.utils import check_ext, check_file_size, file_purpose_dict
 from app.schemas.file import UploadFileResponse
-from tkhelper.error import ErrorCode, raise_http_error, raise_request_validation_error
+from tkhelper.error import ErrorCode, raise_http_error
 from tkhelper.utils import generate_random_id
 
 from ..utils import auth_info_required
@@ -24,20 +24,11 @@ async def api_upload_file(
     file: UploadFile = FastAPIFile(...),
     auth_info: dict = Depends(auth_info_required),
 ):
-    ext = file.filename.split(".")[-1].lower()
-
     purpose_info = file_purpose_dict.get(purpose)
+    ext = check_ext(purpose_info, file.filename.split(".")[-1].lower())
 
     file_size_limit_mb = 15
-
-    if file.size > file_size_limit_mb * 1024 * 1024:
-        raise_request_validation_error("File size is too large.")
-
-    if ext not in purpose_info.allow_file_formats:
-        raise_request_validation_error(
-            f"File format is not supported, supported formats: {', '.join(purpose_info.allow_file_formats.keys())}"
-        )
-    ext = purpose_info.allow_file_formats[ext]
+    check_file_size(file_size_limit_mb, file.size)
 
     random_id = generate_random_id(20)
     file_id = f"{ext}_{purpose_info.id_prefix}{random_id}"

backend/app/routes/file/image.py

@@ -4,10 +4,10 @@
 from app.config import CONFIG
 from app.database import boto3_client
 from app.models import UploadImagePurpose
-from app.routes.file.utils import image_purpose_dict
+from app.routes.file.utils import check_ext, check_file_size, image_purpose_dict
 from app.routes.utils import auth_info_required
 from app.schemas import UploadImageResponse
-from tkhelper.error import ErrorCode, raise_http_error, raise_request_validation_error
+from tkhelper.error import ErrorCode, raise_http_error
 from tkhelper.utils import generate_random_id
 
 router = APIRouter()
@@ -23,22 +23,13 @@ async def upload_image(
     image: UploadFile = FastAPIFile(...),
     auth_info: dict = Depends(auth_info_required),
 ):
-    ext = image.filename.split(".")[-1].lower()
-
     purpose_info = image_purpose_dict.get(purpose)
+    ext = check_ext(purpose_info, image.filename.split(".")[-1].lower())
 
     image_size_limit_mb = 5
+    check_file_size(image_size_limit_mb, image.size)
 
-    if image.size > image_size_limit_mb * 1024 * 1024:
-        raise_request_validation_error("image size is too large.")
-
-    if ext not in purpose_info.allow_file_formats:
-        raise_request_validation_error(
-            f"File format is not supported, supported formats: {', '.join(purpose_info.allow_file_formats.keys())}"
-        )
-    ext = purpose_info.allow_file_formats[ext]
-
-    random_id = generate_random_id(20)
+    random_id = generate_random_id(8)
     file_id = f"{ext}_{purpose_info.id_prefix}{random_id}"
 
     content_bytes = await image.read()

backend/app/routes/file/utils.py

@@ -2,6 +2,7 @@
 from typing import Dict
 
 from app.models.file import UploadFilePurpose, UploadImagePurpose
+from tkhelper.error import raise_request_validation_error
 
 
 class UploadFileModule(str, Enum):
@@ -49,6 +50,19 @@ def __init__(
         "umIM",
         "assistant",
         "user_message_image_size_limit_mb",
-        {"jpg": "jpg", "jpeg": "jpeg", "png": "png"},
+        {"jpg": "jpg", "jpeg": "jpg", "png": "png"},
     ),
 }
+
+
+def check_file_size(limit_mb: int, size: int):
+    if size > limit_mb * 1024 * 1024:
+        raise_request_validation_error("File size is too large.")
+
+
+def check_ext(purpose_info: PurposeInfo, ext: str):
+    if ext not in purpose_info.allow_file_formats:
+        raise_request_validation_error(
+            f"File format is not supported, supported formats: {', '.join(purpose_info.allow_file_formats.keys())}"
+        )
+    return purpose_info.allow_file_formats[ext]

backend/tests/api_services/image/image.py

@@ -3,7 +3,7 @@
 from backend.tests.common.utils import ResponseWrapper, get_headers, get_file_name
 from backend.tests.common.config import CONFIG
 
-IMAGE_BASE_URL = f"{CONFIG.WEB_BASE_URL}/images"
+IMAGE_BASE_URL = f"{CONFIG.BASE_URL}/images"
 
 
 async def upload_image(payload: Dict):

backend/tests/services_tests/assistant/test_upload_image.py

@@ -1,12 +1,13 @@
 import pytest
 import os
 
-from backend.tests.api_services.image.image import upload_image
+from backend.tests.api_services.image.image import upload_image, download_image
 
 
 @pytest.mark.api_test
 class TestUploadImage:
-    upload_image_list = []
+    max_image_data = None
+    normal_image_data = None
     base_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
     image_names = os.listdir(base_path + "/image")
     for image_name in image_names:
@@ -17,15 +18,26 @@ class TestUploadImage:
                 "purpose": "user_message_image",
             }
             upload_image_dict.update({"image": image_path})
-            upload_image_list.append(upload_image_dict)
+            if "5M" in image_name:
+                max_image_data = upload_image_dict
+            else:
+                normal_image_data = upload_image_dict
 
     @pytest.mark.run(order=201)
     @pytest.mark.asyncio
-    @pytest.mark.parametrize("upload_image_data", upload_image_list[:2])
-    async def test_upload_image(self, upload_image_data):
-        res = await upload_image(upload_image_data)
+    async def test_upload_image(self):
+        res = await upload_image(self.normal_image_data)
         assert res.status_code == 200, res.json()
         assert res.json()["status"] == "success"
         url = res.json()["data"]["url"]
         assert url is not None
-        assert os.path.isfile(url)
+        get_res = await download_image(url)
+        assert get_res.status_code == 200, get_res.json()
+
+    @pytest.mark.run(order=201)
+    @pytest.mark.asyncio
+    async def test_upload_max_image(self):
+        res = await upload_image(self.max_image_data)
+        assert res.status_code == 400, res.json()
+        assert res.json()["status"] == "error"
+        assert res.json()["error"]["code"] == "REQUEST_VALIDATION_ERROR"

backend/tests/services_tests/model/test_model.py

@@ -128,7 +128,7 @@ class TestModel:
     @pytest.mark.parametrize("create_model_data", create_model_list)
     async def test_create_model(self, create_model_data):
 
-        create_model_data.pop("host_type")
+        create_model_data.pop("host_type", None)
 
         res = await create_model(create_model_data)
         res_json = res.json()
@@ -240,7 +240,7 @@ async def test_get_model(self):
     @pytest.mark.parametrize("update_model_data", update_model_list)
     async def test_update_model(self, update_model_data):
 
-        update_model_data.pop("host_type")
+        update_model_data.pop("host_type", None)
 
         res = await update_model(CONFIG.chat_completion_model_id, update_model_data)
         res_json = res.json()