Skip to content

Commit

Permalink
refactor: The search results of the CSV file are not expanded.
Browse files Browse the repository at this point in the history
  • Loading branch information
xixihahaliu committed Feb 20, 2024
1 parent 6e4e125 commit 6b44835
Showing 1 changed file with 10 additions and 2 deletions.
12 changes: 10 additions & 2 deletions qanything_kernel/connector/database/milvus/milvus_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,15 @@ def parse_batch_result(self, batch_result):
"file_name": cand.entity.get('file_name'),
"chunk_id": cand.entity.get('chunk_id')})
new_cands.append(doc)
new_cands = self.expand_cand_docs(new_cands)
# csv文件不做expand_cand_docs
need_expand, not_need_expand = [], []
for doc in new_cands:
if doc.metadata['file_name'].lower().endswith('.csv'):
not_need_expand.append(doc)
else:
need_expand.append(doc)
expand_res = self.expand_cand_docs(need_expand)
new_cands = not_need_expand + expand_res
new_result.append(new_cands)
return new_result

Expand Down Expand Up @@ -194,6 +202,7 @@ def seperate_list(self, ls: List[int]) -> List[List[int]]:

def process_group(self, group):
new_cands = []
# 对每个分组按照chunk_id进行排序
group.sort(key=lambda x: int(x.metadata['chunk_id'].split('_')[-1]))
id_set = set()
file_id = group[0].metadata['file_id']
Expand Down Expand Up @@ -256,7 +265,6 @@ def expand_cand_docs(self, cand_docs):

with ThreadPoolExecutor(max_workers=10) as executor:
futures = []
# 对每个分组按照chunk_id进行排序
for group in m_grouped:
if not group:
continue
Expand Down

0 comments on commit 6b44835

Please sign in to comment.