-
Notifications
You must be signed in to change notification settings - Fork 486
/
ECUT_pos_html.py
50 lines (48 loc) · 3.01 KB
/
ECUT_pos_html.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
import requests
import re from bs4
import BeautifulSoup as bs
def crawl_all_main_url(page=10):
# 默认抓取官网前十页招聘信息的url
all_url_list = []
for _ in range(1, page+1):
url = 'http://zjc.ecit.edu.cn/jy/app/newslist.php?BigClassName=%D5%D0%C6%B8%D0%C5%CF%A2&Page={0}'.format(_)
page_html = requests.get(url).text
x_url_reg = re.compile('<a class="t_13px" href="(.*?)"')
x_url = re.findall(x_url_reg, page_html)
main_url = ['http://zjc.ecit.edu.cn/jy/app/{0}'.format(i)
for i in x_url] all_url_list.extend(main_url)
return all_url_list
def get_title(son_url): # 判断该网页是否为校园招聘
html = requests.get(son_url).content.decode('gbk')
explain_text_reg = re.compile('<h1 class="newstitle">(.*?)</h1>')
explain_text = re.findall(explain_text_reg, html)[0]
if ('时间' and '地点') in explain_text:
return True
else: pass
def save_html():
all_url_list = crawl_all_main_url()
for son_url in all_url_list:
if get_title(son_url):
text_html = requests.get(son_url).content.decode('gbk')
domain_url = 'http://zjc.ecit.edu.cn/jy'
img_url_reg = re.compile('border=0 src="\.\.(.*?)"')
child_url = re.findall(img_url_reg, text_html)
if child_url != []:
img_url = domain_url + child_url[0]
re_url = 'src="..{0}"'.format(child_url[0])
end_url = 'src="{0}"'.format(img_url)
end_html = text_html.replace(re_url, end_url)
soup = bs(end_html, 'lxml')
text_div = soup.find_all('div', id='main')[0]
with open('./{0}.html'.format(son_url[-11:]), 'wb') as file:
text_html = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">U职网提供数据咨询服务 <html xmlns="http://www.w3.org/1999/xhtml"> <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> </head> {0} </body>'.format(text_div) file.write(text_html.encode('utf-8'))
else:
with open('./{0}.html'.format(son_url[-11:]), 'wb') as file:
html = requests.get(son_url).content.decode('gbk')
soup = bs(text_html, 'lxml')
text_div = soup.find_all('div', id='main')[0]
text_html = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">U职网提供数据咨询服务 <html xmlns="http://www.w3.org/1999/xhtml"> <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> </head> {0} </body>'.format(text_div)
file.write(text_html.encode('utf-8'))
else: continue
if __name__ == '__main__':
save_html()