-
Notifications
You must be signed in to change notification settings - Fork 1
/
research_agent.py
50 lines (39 loc) · 1.57 KB
/
research_agent.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
from serpapi import GoogleSearch
import requests
from bs4 import BeautifulSoup
def scrape_and_save(query, api_key):
output = []
params = {
"engine": "google",
"q": query,
"api_key": api_key
}
search = GoogleSearch(params)
results = search.get_dict()
for result in results.get('organic_results', []):
snippet = result.get('snippet', 'No snippet found')
link = result.get('link', 'No link found')
if link != 'No link found':
response = requests.get(link)
if response.status_code == 200:
soup = BeautifulSoup(response.text, 'html.parser')
paragraphs = soup.find_all('p')
full_text = "\n".join([para.get_text() for para in paragraphs])
output.append(full_text)
else:
print(f"Failed to retrieve the page: {link}")
return output
def edit_answers(output, client):
advanced_answers = {}
for k in output:
for i, text in enumerate(k):
response = client.chat.completions.create(
model="deepseek-chat",
messages=[
{"role": "system", "content": f"Remove dates, addresses, and line breaks, then correct and rephrase the following text while retaining all specific words and names intact: {text}"},
{"role": "user", "content": ""},
],
stream=False)
enhanced_answer = response.choices[0].message.content
advanced_answers[f"Answers_{i+1}"] = enhanced_answer
return advanced_answers