Merge pull request Significant-Gravitas#275 from master312/master

Added fake user-agent headers to browser request.
xunyuw · Apr 6, 2023 · ba615fb · ba615fb
2 parents 73b8121 + 62854af
commit ba615fb
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 2 deletions.
diff --git a/scripts/browse.py b/scripts/browse.py
@@ -6,7 +6,7 @@
 cfg = Config()
 
 def scrape_text(url):
-    response = requests.get(url)
+    response = requests.get(url, headers=cfg.user_agent_header)
 
     # Check if the response contains an HTTP error
     if response.status_code >= 400:
@@ -40,7 +40,7 @@ def format_hyperlinks(hyperlinks):
 
 
 def scrape_links(url):
-    response = requests.get(url)
+    response = requests.get(url, headers=cfg.user_agent_header)
 
     # Check if the response contains an HTTP error
     if response.status_code >= 400:

diff --git a/scripts/config.py b/scripts/config.py
@@ -39,6 +39,11 @@ def __init__(self):
         self.google_api_key = os.getenv("GOOGLE_API_KEY")
         self.custom_search_engine_id = os.getenv("CUSTOM_SEARCH_ENGINE_ID")
 
+        # User agent headers to use when browsing web
+        # Some websites might just completely deny request with an error code if no user agent was found.
+        self.user_agent_header = {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"}
+
+
         # Initialize the OpenAI API client
         openai.api_key = self.openai_api_key