added filter for websearching only official drexel websites

data_collection/data_manager.py

@@ -426,12 +426,14 @@ def is_valid_url(url: str) -> bool:
     is_social_media = any(platform in url for platform in ["reddit", "tiktok", "linkedin", "instagram", "facebook", "twitter", "youtube"])
     return is_http and not has_invalid_extension and not is_social_media
 
+# too slow, deprecating
 def fetch_content_from_urls(urls):
     content = ""
     if type(urls) == str:
         urls = [urls]
     for url in urls:
-        if is_valid_url(url):
+        if is_valid_url(url) and "drexel.edu" in url:
+            print(url)
             try:
                 response = requests.get(url)
                 if response.status_code == 200:
@@ -446,4 +448,4 @@ def duckduckgo_search(query):
     return DDGS().text(query, max_results=3, backend="lite")
 
 if __name__ == "__main__":
-    upload_minors_to_index(r'C:\Users\alexa\Desktop\dragongpt-backend\data_collection\tools\drexel_catalog\data\minors_data.json')
+    pass

server.py

@@ -38,14 +38,17 @@ def improve_rag(RAG, query):
 
     if check_answer.lower() != 'yes':
         urls = parse_urls_from_rag(RAG)
-        RAG += data_manager.fetch_content_from_urls(urls)
+        # RAG += data_manager.fetch_content_from_urls(urls)
         search_results = data_manager.duckduckgo_search(query + " at Drexel University 2024")
+        # print(search_results)
         for result in search_results:
+            # print("fetching info")
+            RAG += result["body"]
             moreinfo = data_manager.fetch_content_from_urls(result["href"])
             if result["href"] not in urls:
                 RAG += moreinfo + result["href"]
-                #print(result["href"])
-                #print(moreinfo)
+                # print(result["href"])
+                # print(moreinfo)
         if len(RAG) > 128000:
             RAG = RAG[:128000]
     return RAG
@@ -73,7 +76,6 @@ def query_llm():
 
         if not query:
             return jsonify({"detail": "Query is required"}), 400
-
         RAG = data_manager.query_from_index(query)
         RAG = improve_rag(RAG, query)
         system_prompt = open(os.path.join("prompts", "system.txt"), 'r').read()