Update spider.py

Author: Germey

spider.py

@@ -1,12 +1,10 @@
-import os
 import requests
 from urllib.parse import urlencode
+from requests import codes
+import os
 from hashlib import md5
 from multiprocessing.pool import Pool
 
-GROUP_START = 1
-GROUP_END = 5
-
 
 def get_page(offset):
     params = {
@@ -15,50 +13,52 @@ def get_page(offset):
         'keyword': '街拍',
         'autoload': 'true',
         'count': '20',
-        'cur_tab': '3',
-        'from': 'gallery',
+        'cur_tab': '1',
+        'from': 'search_tab'
     }
-    url = 'https://www.toutiao.com/search_content/?' + urlencode(params)
+    base_url = 'https://www.toutiao.com/search_content/?'
+    url = base_url + urlencode(params)
     try:
-        response = requests.get(url)
-        if response.status_code == 200:
-            return response.json()
+        resp = requests.get(url)
+        if codes.ok == resp.status_code:
+            return resp.json()
     except requests.ConnectionError:
         return None
 
 
 def get_images(json):
-    data = json.get('data')
-    if data:
+    if json.get('data'):
+        data = json.get('data')
         for item in data:
-            # print(item)
-            image_list = item.get('image_list')
+            if item.get('cell_type') is not None:
+                continue
             title = item.get('title')
-            # print(image_list)
-            if image_list:
-                for image in image_list:
-                    yield {
-                        'image': image.get('url'),
-                        'title': title
-                    }
+            images = item.get('image_list')
+            for image in images:
+                yield {
+                    'image': 'https:' + image.get('url'),
+                    'title': title
+                }
 
 
 def save_image(item):
-    if not os.path.exists(item.get('title')):
-        os.mkdir(item.get('title'))
+    img_path = 'img' + os.path.sep + item.get('title')
+    if not os.path.exists(img_path):
+        os.makedirs(img_path)
     try:
-        local_image_url = item.get('image')
-        new_image_url = local_image_url.replace('list','large')
-        response = requests.get('http:' + new_image_url)
-        if response.status_code == 200:
-            file_path = '{0}/{1}.{2}'.format(item.get('title'), md5(response.content).hexdigest(), 'jpg')
+        resp = requests.get(item.get('image'))
+        if codes.ok == resp.status_code:
+            file_path = img_path + os.path.sep + '{file_name}.{file_suffix}'.format(
+                file_name=md5(resp.content).hexdigest(),
+                file_suffix='jpg')
             if not os.path.exists(file_path):
-                with open(file_path, 'wb')as f:
-                    f.write(response.content)
+                with open(file_path, 'wb') as f:
+                    f.write(resp.content)
+                print('Downloaded image path is %s' % file_path)
             else:
                 print('Already Downloaded', file_path)
     except requests.ConnectionError:
-        print('Failed to save image')
+        print('Failed to Save Image，item %s' % item)
 
 
 def main(offset):
@@ -68,6 +68,9 @@ def main(offset):
         save_image(item)
 
 
+GROUP_START = 0
+GROUP_END = 7
+
 if __name__ == '__main__':
     pool = Pool()
     groups = ([x * 20 for x in range(GROUP_START, GROUP_END + 1)])