update logic

.gitignore

@@ -1,11 +1,10 @@
 # Ignore files generated by IDEs
 .idea/
 /web_scrape/data/*
-/rag_core/database/*
-/chat_core/database/*
+/database/*
 /data/abstract/*
 /data/paper/*
-/finetune_retriever/model/*
+/finetune_embedder/models/v1/*
 
 # Ignore Python bytecode
 __pycache__/

app.py

@@ -1,14 +1,11 @@
 from flask import Flask, render_template, request, jsonify, redirect, url_for
 import os
+from parse_data.code.collect_data.parse_data import fetch_papers
+from rag_core.utils.embedding import load_embedding_model, fine_tune_model
+from chatcore.utils import load_chat_history, save_chat_history, generate_response
 
 app = Flask(__name__)
 
-# Assume that the core functions for embedding, parsing, etc., are defined in the project modules.
-# For example:
-# from parse_data.code.collect_data.parse_data import fetch_papers
-# from rag_core.utils.embedding import load_embedding_model, fine_tune_model
-# from chatcore.utils import load_chat_history, save_chat_history
-
 @app.route('/')
 def index():
     return render_template('index.html')
@@ -19,12 +16,12 @@ def chat():
     user_id = request.form['user_id']
     thread_id = request.form['thread_id']
 
-    # Load the chat history and process the input (dummy response for now)
-    # response = generate_response(user_input, user_id, thread_id)
-    response = "This is a dummy response."
+    # Load the chat history and process the input
+    history = load_chat_history(user_id, thread_id)
+    response = generate_response(user_input, history)
 
     # Save chat history
-    # save_chat_history(user_id, thread_id, user_input, response)
+    save_chat_history(user_id, thread_id, user_input, response)
 
     return jsonify({'response': response})
 
@@ -35,31 +32,31 @@ def settings():
         fine_tune_option = request.form.get('fine_tune', False)
 
         # Update the embedding model
-        # load_embedding_model(embedding_model)
+        load_embedding_model(embedding_model)
 
         if fine_tune_option:
             # Fine-tune the model
-            # fine_tune_model()
-            pass
+            fine_tune_model()
 
         return redirect(url_for('settings'))
 
     # Get available models for selection
-    available_models = ["Model1", "Model2", "Model3"]
+    available_models = ["Model1", "Model2", "Model3"]  # Replace with actual model names
     return render_template('settings.html', models=available_models)
 
 @app.route('/fetch_data', methods=['POST'])
 def fetch_data():
     # Fetch new papers from the parse_data folder
-    # fetch_papers()
+    fetch_papers()
     return jsonify({'status': 'success'})
 
 @app.route('/change_user', methods=['POST'])
 def change_user():
     user_id = request.form['user_id']
+    thread_id = request.form['thread_id']
     # Load the user's chat history
-    # history = load_chat_history(user_id)
-    return jsonify({'history': "User's chat history."})
+    history = load_chat_history(user_id, thread_id)
+    return jsonify({'history': history})
 
 if __name__ == '__main__':
     app.run(debug=True)

chat_core/command/chat_core.sh

@@ -4,33 +4,46 @@ ROOT_DIR="."
 # Export PYTHONPATH to include the root directory
 export PYTHONPATH="$ROOT_DIR"
 
+
 python3 chat_core/chatbot.py \
     --question "How can Large Language Models (LLMs) be susceptible to adversarial attacks, and what are the implications of such vulnerabilities?" \
     --compressor_type 1 \
-    --db_path "./rag_core/database/chroma_db" \
-    --user_id "admin" \
+    --db_path "./database/rag_core/chroma_db_custom_embedding" \
+    --user_id "Huy" \
     --conversation_id "1" \
     --k 3 \
     --embedding_type 1
 
 
-python3 chat_core/chatbot.py \
-    --question "How does the “LLM-as-a-Judge” metric align with human judgment in assessing bias in LLM-generated responses?" \
-    --compressor_type 1 \
-    --db_path "./rag_core/database/chroma_db" \
-    --user_id "admin" \
-    --conversation_id "1" \
-    --k 3 \
-    --embedding_type 1
 
-python3 chat_core/chatbot.py \
-    --question "What are the key strengths and weaknesses of different model families in responding to adversarial prompts aimed at eliciting biased outputs?" \
-    --compressor_type 1 \
-    --db_path "./rag_core/database/chroma_db" \
-    --user_id "admin" \
-    --conversation_id "1" \
-    --k 3 \
-    --embedding_type 1
+
+# python3 chat_core/chatbot.py \
+#     --question "How can Large Language Models (LLMs) be susceptible to adversarial attacks, and what are the implications of such vulnerabilities?" \
+#     --compressor_type 1 \
+#     --db_path "./database/rag_core/chroma_db" \
+#     --user_id "admin" \
+#     --conversation_id "1" \
+#     --k 3 \
+#     --embedding_type 1
+
+
+# python3 chat_core/chatbot.py \
+#     --question "How does the “LLM-as-a-Judge” metric align with human judgment in assessing bias in LLM-generated responses?" \
+#     --compressor_type 1 \
+#     --db_path "./database/rag_core/chroma_db" \
+#     --user_id "admin" \
+#     --conversation_id "1" \
+#     --k 3 \
+#     --embedding_type 1
+
+# python3 chat_core/chatbot.py \
+#     --question "What are the key strengths and weaknesses of different model families in responding to adversarial prompts aimed at eliciting biased outputs?" \
+#     --compressor_type 1 \
+#     --db_path "./database/rag_core/chroma_db" \
+#     --user_id "admin" \
+#     --conversation_id "1" \
+#     --k 3 \
+#     --embedding_type 1
 
 
 
@@ -44,7 +57,7 @@ python3 chat_core/chatbot.py \
 # python3 chat_core/chatbot.py \
 #     --question "How can Large Language Models (LLMs) be susceptible to adversarial attacks, and what are the implications of such vulnerabilities?" \
 #     --compressor_type 1 \
-#     --db_path "./rag_core/database/chroma_db_local_embedding" \
+#     --db_path "./database/rag_core/chroma_db_local_embedding" \
 #     --user_id "admin" \
 #     --conversation_id "2" \
 #     --k 3 \
@@ -54,7 +67,7 @@ python3 chat_core/chatbot.py \
 # python3 chat_core/chatbot.py \
 #     --question "How does the “LLM-as-a-Judge” metric align with human judgment in assessing bias in LLM-generated responses?" \
 #     --compressor_type 1 \
-#     --db_path "./rag_core/database/chroma_db_local_embedding" \
+#     --db_path "./database/rag_core/chroma_db_local_embedding" \
 #     --user_id "admin" \
 #     --conversation_id "2" \
 #     --k 3 \
@@ -63,7 +76,7 @@ python3 chat_core/chatbot.py \
 # python3 chat_core/chatbot.py \
 #     --question "What are the key strengths and weaknesses of different model families in responding to adversarial prompts aimed at eliciting biased outputs?" \
 #     --compressor_type 1 \
-#     --db_path "./rag_core/database/chroma_db_local_embedding" \
+#     --db_path "./database/rag_core/chroma_db_local_embedding" \
 #     --user_id "admin" \
 #     --conversation_id "2" \
 #     --k 3 \
@@ -81,7 +94,7 @@ python3 chat_core/chatbot.py \
 # python3 chat_core/chatbot.py \
 #     --question "How can Large Language Models (LLMs) be susceptible to adversarial attacks, and what are the implications of such vulnerabilities?" \
 #     --compressor_type 1 \
-#     --db_path "./rag_core/database/chroma_db_local_embedding" \
+#     --db_path "./database/rag_core/chroma_db_local_embedding" \
 #     --user_id "admin" \
 #     --conversation_id "3" \
 #     --k 3 \
@@ -91,7 +104,7 @@ python3 chat_core/chatbot.py \
 # python3 chat_core/chatbot.py \
 #     --question "How does the “LLM-as-a-Judge” metric align with human judgment in assessing bias in LLM-generated responses?" \
 #     --compressor_type 1 \
-#     --db_path "./rag_core/database/chroma_db_local_embedding" \
+#     --db_path "./database/rag_core/chroma_db_local_embedding" \
 #     --user_id "admin" \
 #     --conversation_id "3" \
 #     --k 3 \
@@ -100,7 +113,7 @@ python3 chat_core/chatbot.py \
 # python3 chat_core/chatbot.py \
 #     --question "What are the key strengths and weaknesses of different model families in responding to adversarial prompts aimed at eliciting biased outputs?" \
 #     --compressor_type 1 \
-#     --db_path "./rag_core/database/chroma_db_local_embedding" \
+#     --db_path "./database/rag_core/chroma_db_local_embedding" \
 #     --user_id "admin" \
 #     --conversation_id "3" \
 #     --k 3 \

chat_core/utils/get_history.py

@@ -6,7 +6,7 @@
 
 def get_session_history(user_id, conversation_id):
     # Define the database path
-    database_path = "chat_core/database/memory.db"
+    database_path = "database/chat_core/memory.db"
     # Extract the directory from the path
     database_dir = os.path.dirname(database_path)