update studio fe table UI and updated studio be according to the dataprep refactor

.github/workflows/_e2e-test.yml

@@ -53,6 +53,7 @@ jobs:
           fi
           sleep 5
           sudo apt install ansible -y
+          sed -i 's/value: ${TAG}/value: latest/' manifests/studio-manifest.yaml
           ansible-playbook genai-studio.yml -e "container_registry=${OPEA_IMAGE_REPO}opea" -e "container_tag=${{ inputs.tag }}" -e "mysql_host=$(hostname -I | awk '{print $1}')"
           sleep 5
           kubectl wait --for=condition=ready pod --all --namespace=studio --timeout=300s --field-selector=status.phase!=Succeeded

app-frontend/react/src/components/Conversation/DataSource.tsx

@@ -36,7 +36,7 @@ export default function DataSource({ opened, onClose }: Props) {
     try {
       setTimeout(async () => {
         const response = await client.post(
-          `${DATA_PREP_URL}/get_file`,
+          `${DATA_PREP_URL}/get`,
           {}, // Request body (if needed, replace the empty object with actual data)
           {
             headers: {
@@ -55,7 +55,7 @@ export default function DataSource({ opened, onClose }: Props) {
   const deleteFile = async (id: string) => {
     try {
       await client.post(
-        `${DATA_PREP_URL}/delete_file`,
+        `${DATA_PREP_URL}/delete`,
         { file_path: id }, // Request body (if needed, replace the empty object with actual data)
         {
           headers: {

app-frontend/react/src/redux/Conversation/ConversationSlice.ts

@@ -114,7 +114,7 @@ export const submitDataSourceURL = createAsyncThunkWrapper(
     try {
       const body = new FormData();
       body.append("link_list", JSON.stringify(link_list));
-      const response = await client.post(DATA_PREP_URL, body);
+      const response = await client.post(`${DATA_PREP_URL}/ingest`, body);
       return response.data;
     } catch (error) {
       console.log ("error", error);
@@ -132,7 +132,7 @@ export const uploadFile = createAsyncThunkWrapper("conversation/uploadFile", asy
       message: "uploading File",
       loading: true,
     });
-    const response = await client.post(DATA_PREP_URL, body);
+    const response = await client.post(`${DATA_PREP_URL}/ingest`, body);
     return response.data;
   } catch (error) {
     throw error;

setup-scripts/setup-genai-studio/manifests/studio-manifest.yaml

@@ -326,9 +326,9 @@ spec:
         - name: APP_BACKEND_IMAGE
           value: ${REGISTRY}/app-backend:${TAG}
         - name: REGISTRY
-          value: opea
+          value: ${REGISTRY}
         - name: TAG
-          value: latest
+          value: ${TAG}
         - name: SBX_HTTP_PROXY
           value: ${HTTP_PROXY}
         - name: SBX_NO_PROXY

setup-scripts/setup-genai-studio/playbooks/deploy-studio.yml

@@ -77,7 +77,7 @@
         MYSQL_HOST: "{{ mysql_host }}"
 
     - name: Wait for all pods to be ready in studio namespace
-      shell: kubectl wait --for=condition=ready pod --all --namespace=studio --timeout=180s
+      shell: kubectl wait --for=condition=ready pod --all --namespace=studio --timeout=300s
       register: pod_ready_check
       failed_when: pod_ready_check.rc != 0
       changed_when: false

studio-backend/app/templates/microsvc-composes/data-prep.yaml

@@ -1,19 +1,20 @@
 "{{endpoint}}":
-  image: ${REGISTRY}/dataprep-redis:${TAG}
+  image: ${REGISTRY}/dataprep:${TAG}
   container_name: "{{endpoint}}"
   depends_on:
     "{{redis_vector_store_endpoint}}":
       condition: service_started
     "{{tei_endpoint}}":
       condition: service_healthy
   ports:
-    - 6007:6007
+    - 6007:5000
   environment:
     no_proxy: ${no_proxy}
     http_proxy: ${http_proxy}
     https_proxy: ${https_proxy}
     REDIS_URL: "redis://${public_host_ip}:{{redis_vector_store_port}}"
     REDIS_HOST: "${public_host_ip}"
     INDEX_NAME: "rag-redis"
-    TEI_ENDPOINT: "http://${public_host_ip}:{{tei_port}}"
+    TEI_EMBEDDING_ENDPOINT: "http://${public_host_ip}:{{tei_port}}"
+    DATAPREP_COMPONENT_NAME: "OPEA_DATAPREP_REDIS"
     HUGGINGFACEHUB_API_TOKEN: "{{tei_huggingFaceToken}}"

studio-backend/app/templates/microsvc-manifests/data-prep.yaml

@@ -9,8 +9,8 @@ metadata:
   name: config-{endpoint}
 data:
   HEALTHCHECK_ENDPOINT: "{tei_endpoint}"
-  TEI_ENDPOINT: "http://{tei_endpoint}"
-  EMBED_MODEL: ""
+  TEI_EMBEDDING_ENDPOINT: "http://{tei_endpoint}"
+  DATAPREP_COMPONENT_NAME: "OPEA_DATAPREP_REDIS"
   REDIS_URL: "redis://{redis_vector_store_endpoint}:{redis_vector_store_port}"
   INDEX_NAME: "rag-redis"
   KEY_INDEX_NAME: "file-keys"
@@ -34,7 +34,7 @@ spec:
   type: ClusterIP
   ports:
     - port: 6007
-      targetPort: 6007
+      targetPort: 5000
       protocol: TCP
       name: "{endpoint}"
   selector:
@@ -84,11 +84,11 @@ spec:
             runAsUser: 1000
             seccompProfile:
               type: RuntimeDefault
-          image: "${REGISTRY}/dataprep-redis:${TAG}"
+          image: "${REGISTRY}/dataprep:${TAG}"
           imagePullPolicy: Always
           ports:
             - name: data-prep
-              containerPort: 6007
+              containerPort: 5000
               protocol: TCP
           volumeMounts:
             - mountPath: /tmp

studio-backend/tests/exporter-groundtruth/gt_app-compose.yaml

@@ -95,7 +95,7 @@ services:
       HF_HUB_ENABLE_HF_TRANSFER: 0
     restart: unless-stopped
   prepare-doc-redis-prep-0:
-    image: opea/dataprep-redis:latest
+    image: opea/dataprep:latest
     container_name: prepare-doc-redis-prep-0
     depends_on:
     - redis-vector-store-0

studio-backend/tests/exporter-groundtruth/gt_app-manifest-with-nginx.yaml

@@ -778,7 +778,7 @@ spec:
           runAsUser: 1000
           seccompProfile:
             type: RuntimeDefault
-        image: opea/dataprep-redis:latest
+        image: opea/dataprep:latest
         imagePullPolicy: Always
         ports:
         - name: data-prep

studio-backend/tests/exporter-groundtruth/gt_app-manifest.yaml

@@ -782,7 +782,7 @@ spec:
           runAsUser: 1000
           seccompProfile:
             type: RuntimeDefault
-        image: opea/dataprep-redis:latest
+        image: opea/dataprep:latest
         imagePullPolicy: Always
         ports:
         - name: data-prep

studio-frontend/Dockerfile

@@ -1,17 +1,14 @@
 FROM node:20-alpine
 
 # Install necessary packages
-RUN apk add --no-cache gcompat python3 make g++ \
+RUN apk add --no-cache gcompat python3 make g++ git \
     # Needed for pdfjs-dist
     build-base cairo-dev pango-dev \
     # Install Chromium
     chromium && \
     # Install PNPM globally
     npm install -g pnpm@9.12.3
 
-# Install git
-RUN apk add --no-cache git
-
 # Debug step to verify git installation
 RUN git --version