增加查询所有歌单，增加无信息处理

Author: FullStackDeveloper

music163/albums.py

@@ -13,37 +13,48 @@
     'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
 }
 # https://music.163.com/#/artist/album?id=2116
+# https://music.163.com/#/artist/album?id=1177234&limit=12&offset=36
 url = 'https://music.163.com/artist/album'
 
 '''
 IN:
 artist_id 歌手id
 OUT:
 album_infos = [{'album_id': '36875566', 'album_title': 'G.ream', 'album_time': '2017.12.07'},{}]
-
+limit 分页歌曲数
+offset 分页开始
 '''
 
 
 def main_spider(artist_id='8888'):
-    request = urllib.request.Request(url + '?id=' + artist_id, headers=header)
-    response = urllib.request.urlopen(request)
-    soup = BeautifulSoup(response.read().decode('utf-8'), 'html.parser')
-    albums = soup.find('ul', {'class': 'm-cvrlst m-cvrlst-alb4 f-cb'})
+    limit = 12
+    offset = 0
     album_infos = []
-
-    album_basic_infos = albums.find_all('p', {'class': 'dec dec-1 f-thide2 f-pre'})
-    for album_basic_info in album_basic_infos:
-        album_infod = {}
-        album_info = []
-        # print(album_basic_info.find_next_sibling())
-        album_infod['album_id'] = album_basic_info.find_next()['href'].replace('/album?id=', '').strip()
-        album_infod['album_title'] = album_basic_info['title']
-        album_infod['album_time'] = album_basic_info.find_next_sibling().text
-        album_info.append(album_basic_info.find_next()['href'].replace('/album?id=', '').strip())
-        album_info.append(album_basic_info['title'])
-        album_info.append(album_basic_info.find_next_sibling().text)
-        album_infos.append(album_info)
+    while True:
+        request = urllib.request.Request(url + '?id=' + artist_id + '&limit=' + str(limit) + '&offset=' + str(offset),
+                                         headers=header)
+        response = urllib.request.urlopen(request)
+        soup = BeautifulSoup(response.read().decode('utf-8'), 'html.parser')
+        if soup.find('ul', {'class': 'm-cvrlst m-cvrlst-alb4 f-cb'}) is None:
+            return album_infos
+        albums = soup.find('ul', {'class': 'm-cvrlst m-cvrlst-alb4 f-cb'})
+
+        if albums.find_all('p', {'class': 'dec dec-1 f-thide2 f-pre'}) is None:
+            return album_infos
+        album_basic_infos = albums.find_all('p', {'class': 'dec dec-1 f-thide2 f-pre'})
+        for album_basic_info in album_basic_infos:
+            album_infod = {}
+            album_info = []
+            # print(album_basic_info.find_next_sibling())
+            album_infod['album_id'] = album_basic_info.find_next()['href'].replace('/album?id=', '').strip()
+            album_infod['album_title'] = album_basic_info['title']
+            album_infod['album_time'] = album_basic_info.find_next_sibling().text
+            album_info.append(album_basic_info.find_next()['href'].replace('/album?id=', '').strip())
+            album_info.append(album_basic_info['title'])
+            album_info.append(album_basic_info.find_next_sibling().text)
+            album_infos.append(album_info)
+        offset += 12
 
     return album_infos
 
-#main_spider()
+# print(main_spider('1177234'))

music163/artists.py

@@ -41,13 +41,13 @@ def main_spider(id='1001', initial='-1'):
         else:
             artist_extend_info = artist_basic_info.find_next_sibling()['href'].replace('/user/home?id=', '').strip()
         artist_infod['artist_id'] = artist_basic_info['href'].replace('/artist?id=', '').strip()
-        artist_infod['artist_name'] = artist_basic_info.text
+        artist_infod['artist_name'] = artist_basic_info.text[0:25]
         artist_infod['artist_extend_id'] = artist_extend_info
         artist_info.append(artist_basic_info['href'].replace('/artist?id=', '').strip())
-        artist_info.append(artist_basic_info.text)
+        artist_info.append(artist_basic_info.text[0:25])
         artist_info.append(artist_extend_info)
 
         artist_infos.append(artist_info)
     return artist_infos
 
-#main_spider('1001', '65')
+#print(main_spider('1001', '65'))

music163/comments.py

@@ -40,6 +40,8 @@ def main_spider(music_id='553543014'):
     # response = urllib.request.urlopen(request)
     # print(response.read().decode('utf-8'))
     r = requests.post(url=url + music_id + '?csrf_token=', data=comments_data1, headers=header)
+    if r.text is None:
+        return None
     comments_obj = json.loads(r.text)
     comment_infos = []
     for comment_obj in comments_obj['hotComments']:
@@ -79,4 +81,4 @@ def main_spider(music_id='553543014'):
 
     return comment_infos
 
-# main_spider('553543014')
+# print(main_spider('553543014'))

music163/musics.py

@@ -27,6 +27,8 @@ def main_spider(album_id='74999481'):
     request = urllib.request.Request(url + '?id=' + album_id, headers=header)
     response = urllib.request.urlopen(request)
     soup = BeautifulSoup(response.read().decode('utf-8'), 'html.parser')
+    if soup.find('div', {'id': 'song-list-pre-cache'}).find('ul', {'class': 'f-hide'}) is None:
+        return None
     musics = soup.find('div', {'id': 'song-list-pre-cache'}).find('ul', {'class': 'f-hide'})
     music_basic_infos = musics.find_all('li')
     music_infos = []

music163/mysql.py

@@ -67,14 +67,14 @@ def create_tables(create_type='all'):
     artist_infos_sql = '''
       CREATE TABLE artist_infos (
         artist_id varchar(20), 
-        artist_name varchar(40),
+        artist_name varchar(50),
         artist_extend_id varchar(20)
         )ENGINE=innodb DEFAULT CHARSET=utf8; 
         '''
     album_infos_sql = '''
       CREATE TABLE album_infos (
         album_id varchar(20), 
-        album_title varchar(100),
+        album_title varchar(200),
         album_time varchar(20)
         )ENGINE=innodb DEFAULT CHARSET=utf8; 
         '''
@@ -88,19 +88,19 @@ def create_tables(create_type='all'):
       CREATE TABLE comment_infos (
         comment_id varchar(30), 
         user_id varchar(30),
-        user_nickname varchar(100),
+        user_nickname varchar(200),
         liked_count varchar(50),
-        content varchar(1000),
+        content varchar(2000),
         is_hot varchar(10)
         )ENGINE=innodb DEFAULT CHARSET=utf8; 
     '''
-    if create_type == 'artists':
+    if create_type == 'artist_infos':
         cursor.execute(artist_infos_sql)
-    elif create_type == 'albums':
+    elif create_type == 'album_infos':
         cursor.execute(album_infos_sql)
-    elif create_type == 'musics':
+    elif create_type == 'music_infos':
         cursor.execute(music_infos_sql)
-    elif create_type == 'comments':
+    elif create_type == 'comment_infos':
         cursor.execute(comment_infos_sql)
     else:
         cursor.execute(artist_infos_sql)
@@ -109,9 +109,21 @@ def create_tables(create_type='all'):
         cursor.execute(comment_infos_sql)
 
 
-def drop_tables(table_name):
-    sql = 'drop table ' + table_name
-    cursor.execute(sql)
+def drop_tables(drop_type='all'):
+    sql = 'drop table '
+    if drop_type == 'artist_infos':
+        cursor.execute(sql+drop_type)
+    elif drop_type == 'album_infos':
+        cursor.execute(sql+drop_type)
+    elif drop_type == 'music_infos':
+        cursor.execute(sql+drop_type)
+    elif drop_type == 'comment_infos':
+        cursor.execute(sql+drop_type)
+    else:
+        cursor.execute(sql+'artist_infos')
+        cursor.execute(sql+'album_infos')
+        cursor.execute(sql+'music_infos')
+        cursor.execute(sql+'comment_infos')
 
 
 def close():