scraping

Author: furas

__scraping__/forexfactory.com/main.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3 
+
+# date: 2019.12.30
+# https://stackoverflow.com/questions/59535798/python-webscraping-with-beautifulsoup-not-displaying-full-content/59536553#59536553
+
+import requests
+from bs4 import BeautifulSoup
+
+r = requests.get("https://www.forexfactory.com/#detail=108867")
+# page uses JavaScript to redirect page so browser may shows different results.
+
+soup = BeautifulSoup(r.text, 'lxml')
+
+table = soup.find("table", class_="calendar__table")
+
+for row in table.find_all('tr', class_='calendar__row--grey'):
+    
+    currency = row.find("td", class_="currency")
+    #print(currency.prettify()) # before get text
+    currency = currency.get_text(strip=True)
+
+    actual = row.find("td", class_="actual")
+    actual = actual.get_text(strip=True)
+
+    forecast = row.find("td", class_="forecast")
+    forecast = forecast.get_text(strip=True)
+    
+    print(currency, actual, forecast)
+
+

__scraping__/gall.dcinside.com/main.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3 
+
+# date: 2020.01.01
+# https://stackoverflow.com/questions/59551193/i-want-to-download-images-from-python-what-should-i-do/
+
+from selenium import webdriver
+import requests
+
+#path = r"C:\Users\qpslt\Desktop\py\chromedriver_win32\chromedriver.exe"
+#driver = webdriver.Chrome(path)
+driver = webdriver.Firefox()
+
+url = "https://gall.dcinside.com/board/view/?id=baseball_new8&no=10131338&exception_mode=recommend&page=1"
+driver.get(url)
+
+images = driver.find_elements_by_xpath('//div[@class="writing_view_box"]//img')
+
+for i, img in enumerate(images, 1):
+    img_url = img.get_attribute('src')
+    print(i, img_url)
+
+    r = requests.get(img_url, headers={'Referer': url})
+
+    with open("c:/test/{}.jpg".format(i), 'wb') as f:
+        f.write(r.content)
+

__scraping__/msih.bgu.ac.il - requests BS/main.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3 
+
+# date: 2020.01.02
+# ???
+
+import urllib.request
+import bs4 as bs
+
+sauce = urllib.request.urlopen('https://msih.bgu.ac.il/md-program/residency-placements/').read()
+soup = bs.BeautifulSoup(sauce, 'lxml')
+
+headers = soup.find_all('div', class_={'accord-head'})
+grad_yr_list = []
+for header in headers:
+    grad_yr_list.append(header.h2.text[-4:])
+
+rez_classes = soup.find_all('div', class_={'accord-con'})
+
+data_dict = dict(zip(grad_yr_list, rez_classes))
+
+for key, value in data_dict.items():
+    print(type(value), key, value.find('h4').text)
+

__scraping__/seedly.sg/main.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3 
+
+# date: 2020.01.02
+# ???
+
+# count stars 
+
+from selenium import webdriver 
+
+browser = webdriver.Firefox()
+
+url = 'https://seedly.sg/reviews/p2p-lending/funding-societies'
+browser.get(url)
+
+star_count_list = []
+
+rating_column = browser.find_elements_by_xpath('//div[contains(@class,"qr0ren-7 euifNX")]')
+
+for row in rating_column:
+    stars = row.find_elements_by_xpath('.//span[contains(@style,"width:100%")]')
+    star_count_list.append(len(stars))
+
+for i, e in enumerate(star_count_list, 1):
+    print('{}. {}'.format(i, e))

__scraping__/shopee.tw/main.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+# date: 2020.01.02
+# https://stackoverflow.com/questions/59557071/how-can-i-crawl-the-product-items-from-shopee-website/59557626#59557626
+
+# Without `Referer` it doesn't send price
+
+import requests
+
+url = 'https://shopee.tw/api/v2/search_items/?by=pop&limit=30&match_id=1819984&newest=0&order=desc&page_type=shop&shop_categoryids=9271157&version=2'
+
+headers = {
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0',
+    'Referer': 'https://shopee.tw/shop/1819984/search?shopCollection=9271157',
+    'X-Requested-With': 'XMLHttpRequest',
+}    
+
+r = requests.get(url, headers=headers)
+
+data = r.json()
+
+#print(data['items'][0].keys())
+
+for item in data['items']:
+    print('name:', item['name'])
+    print('prince:', item['price'])
+    print('---')