code/robot.py at master · wyatt88/code · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
__author__ = 'wyatt'
###~coding: utf-8###

import urllib
import urllib2
import re
import sys
import os

reload(sys)
sys.setdefaultencoding('utf-8')
print sys.getdefaultencoding()
url = "http://www.douban.com/group/haixiuzu/discussion?start="

def get_pic(sublink,filename):
    fn=filename
    sl=sublink
    print "sl= "+sl
    try:
        subresponse= urllib2.urlopen(sl)
    except:
        print "error"
    subhtml=subresponse.read()
#    print subhtml
    pic_links=re.findall("http://img[0-9].douban.com/view/group_topic/large/public/.*\.jpg",subhtml)
    count_pics=len(pic_links)
    if count_pics > 0:
        file_path="/root/pics/"+str(fn)
        if not os.path.isdir(file_path):
            os.mkdir(file_path)


        for i in range(0,count_pics):
             fs=open(file_path+"/"+str(i)+".jpg",'w')
             pic=urllib2.urlopen(pic_links[i])
             pichtml=pic.read()
             fs.write(pichtml)
             fs.close()

for count in range(0,100):
    html = ""
    try:
        c = count*25
        response = urllib2.urlopen(url+str(c))

        html = response.read()
    except:
        print "error"
	break

    # links = re.findall(ur'<a href="http://www.douban.com/group/topic/[\d]*/" title=".*" .*</a>',html)
    links = re.findall(ur'<a.* href="http://www.douban.com/group/topic/[\d]*/".*</a>',html)
    print links
    for link in links:
        chineselink = str(link)
        sublink = re.findall(ur'http://www.douban.com/group/topic/\d+/',chineselink)
        topicurl = sublink[0]
        filename = re.findall(ur'title=".*" ',str(chineselink))
        get_pic(sublink[0],str(filename[0]).split()[0].split('=')[1].replace('"',""))