fix crawler

Author: cassc

contract_crawler.py

@@ -26,6 +26,32 @@
                     'Optimization Enabled': 'optimizations',
                     'Other Settings:': 'settings'}
 
+session = {}
+
+def get_session_from_chromedriver(url):
+    driver = uc.Chrome()
+    driver.get(url)
+
+    session = requests.Session()
+    user_agent = driver.execute_script("return navigator.userAgent;")
+    session.headers.update({'User-Agent': user_agent})
+
+    if len(driver.get_cookies()):
+        raise Exception('Should have some cookies here')
+
+    for cookie in driver.get_cookies():
+        session.cookies.set(cookie['name'], cookie['value'])
+
+    print(f'Cookies loaded from {url} {session.cookies}')
+    return session
+
+def load_session(url):
+    global session
+    if not session:
+        session = get_session_from_chromedriver(url)
+    return session
+
+
 def address_from_tr(td: Any) -> str:
     a = td.select_one('a.js-clipboard')
     return a.attrs.get('data-clipboard-text') if (a and a.attrs) else None
@@ -34,7 +60,7 @@ def address_from_tr(td: Any) -> str:
 def parse_page(page: Optional[int]=None, retry=3, retry_delay=5) -> Optional[List[Dict[str, str]]]:
     url = VERIFIED_CONTRACT_URL if page is None else f'{VERIFIED_CONTRACT_URL}/{page}'
     print(f'Crawling {url}')
-    resp = requests.get(url, headers=REQ_HEADER, allow_redirects=False, proxies=proxies)
+    resp = session.get(url, allow_redirects=False)
     if resp.status_code != 200:
         print(f'No results found on page: {page}, http status: {resp.status_code}')
         return None
@@ -131,7 +157,7 @@ def download_source(contract: Dict[str, str], retry=3, retry_delay=5, throw_if_f
     address = contract['Address']
     contract_name = contract['Contract Name']
     url = CONTRACT_SOURCE_URL.format(address)
-    resp = requests.get(url, headers=REQ_HEADER, allow_redirects=False, proxies=proxies)
+    resp = session.get(url, allow_redirects=False)
 
     def maybe_retry(e=None):
         if retry > 0:
@@ -165,25 +191,16 @@ def fetch_all():
 
 def download_url_poly(url, retry=3, retry_delay=5, throw_if_fail=False):
     address = url.split('/')[-1].split('#')[0]
-    driver = uc.Chrome()
-    driver.get(url)
-
-    # fullscreen_btn = driver.find_elements(By.XPATH, '//a[@class="btn btn-xss btn-secondary togglefullscreen"]')
-    # for btn in fullscreen_btn:
-    #     btn.click()
-    #     time.sleep(0.05)
 
-    cookie = driver.get_cookies()[0]
-    for key, value in cookie.items():
-        cookie[key] = str(value)
+    session = get_session_from_chromedriver(url)
 
-    resp = requests.get(url, headers=REQ_HEADER, allow_redirects=True, cookies=cookie, proxies=proxies)
+    resp = session.get(url)
     soup = BeautifulSoup(resp.content, 'lxml')
     parse_source_soup(soup, address)
 
 def download_url(url, retry=3, retry_delay=5, throw_if_fail=False):
     address = url.split('/')[-1].split('#')[0]
-    resp = requests.get(url, headers=REQ_HEADER, allow_redirects=False, proxies=proxies)
+    resp = session.get(url, allow_redirects=False)
 
     if resp.status_code != 200:
         if retry > 0:
@@ -198,7 +215,6 @@ def download_url(url, retry=3, retry_delay=5, throw_if_fail=False):
     soup = BeautifulSoup(resp.content, 'lxml')
     parse_source_soup(soup, address)
 
-
 if __name__ == '__main__':
     ap = argparse.ArgumentParser()
     ap.add_argument("--web", default="etherscan",type=str, help="Choose website, etherscan(default) or bscscan")
@@ -210,6 +226,7 @@ def download_url(url, retry=3, retry_delay=5, throw_if_fail=False):
     ROOT_DIR = f'{OUTPUT_DIR}/contracts'
 
     web = args.web
+
     if web == 'etherscan':
         VERIFIED_CONTRACT_URL = 'https://etherscan.io/contractsVerified'
         CONTRACT_SOURCE_URL   = 'https://etherscan.io/address/{}#code'
@@ -237,6 +254,9 @@ def download_url(url, retry=3, retry_delay=5, throw_if_fail=False):
     print(CONTRACT_SOURCE_URL)
     print(ROOT_DIR)
     url = args.url
+
+    load_session(VERIFIED_CONTRACT_URL)
+
     if url:
         fn(url)
     else: