data faiss 3

mrzaizai2k · Jun 24, 2024 · e164ec9 · e164ec9
1 parent 54b83c3
commit e164ec9
Show file tree

Hide file tree

Showing 7 changed files with 184,004 additions and 218 deletions.
diff --git a/config/model_config.yaml b/config/model_config.yaml
@@ -1,9 +1,9 @@
-embedding_model: sentence-transformers/stsb-xlm-r-multilingual
+embedding_model: sentence-transformers/distiluse-base-multilingual-cased-v2
 routing_model: chibao24/model_routing_few_shot
-splitter_chunk_size: 700 # Must check carefully on the embedding model
+splitter_chunk_size: 500 # Must check carefully on the embedding model
 splitter_chunk_overlap: 100
 
-db_faiss_path: data/vectorstores/db_faiss/
+db_faiss_path: data/vectorstores/db_faiss_3/
 
 custom_prompt_template_path: config/custom_prompt_template.txt
 openai_max_tokens: 1024
@@ -19,9 +19,9 @@ k_similar_rerank: 5
 lambda_mult: 0.7
 search_type: mmr
 
-ocr_model: facebook/nougat-small
+ocr_model: facebook/nougat-base
 ocr_model_max_new_tokens: 1024
 
 math_classifier: CrissWang/bert-math
-rerank_model: sentence-transformers/stsb-xlm-r-multilingual
+rerank_model: sentence-transformers/distiluse-base-multilingual-cased-v2