update new feature

Signed-off-by: ericyuanhui <285521263@qq.com>

Author: ericyuanhui

Language/python/crawl_web/crawl_web.py

@@ -1,11 +1,13 @@
 
 import urllib.request
 import chardet
-import builtwith
 import ssl
 import pymysql
+import re
+import matplotlib.pyplot as plt
 import time
 
+from pyecharts import Bar
 from pyquery import PyQuery as pq
 
 # 通过指定的字符集对页面进行解码(不是每个网站都将字符集设置为utf-8)
@@ -22,7 +24,6 @@ def decode_page(page_bytes, charset='utf-8'):
 # 获取页面的HTML代码(通过递归实现指定次数的重试操作)
 def get_page_html(seed_url, header_url, retry_times=3, charset='utf-8'):
     page_html = None
-    page_bytes = None
     try:
         page_bytes = urllib.request.urlopen(urllib.request.Request(seed_url, headers = header_url)).read()
         page_html = decode_page(page_bytes, charset)
@@ -34,7 +35,6 @@ def get_page_html(seed_url, header_url, retry_times=3, charset='utf-8'):
 
 # 获得页面的编码格式
 def get_page_encode(seed_url, header_url):
-    page_bytes = None
     page_encode = None
     try:
         page_bytes = urllib.request.urlopen(urllib.request.Request(seed_url, headers=header_url)).read()
@@ -54,18 +54,21 @@ def file_store(file_path, page_bytes):
 
 # analyze data with pyquery
 def collect_data(page_html, start_index):
-    html_query = None
     html_query = pq(page_html)
     query_index = start_index
     while (1):
         href_list = html_query('a').eq(query_index).attr('href')
         book_name = pq(html_query('a').eq(query_index)).find('.hanghang-list-name').text()
+        re_book_name = re.sub(r'\'+', ' ', book_name)
+        final_book_name = re.sub(r'\"+', ' ', re_book_name)
         book_download_num = pq(html_query('a').eq(query_index)).find('.hanghang-list-num').text()
         book_author = pq(html_query('a').eq(query_index)).find('.hanghang-list-zuozhe').text()
+        re_book_author = re.sub(r'\'+', ' ', book_author)
+        final_book_author = re.sub(r'\"+', ' ', re_book_author)
         if book_name:
             query_index = query_index + 1
             print ("book_name: %s ,book num: %s ,book_author: %s, book link: %s" %(book_name, book_download_num, book_author, href_list))
-            store_data(book_name, book_author, book_download_num, href_list)
+            store_data(final_book_name, final_book_author, book_download_num, href_list)
         else:
             break
 
@@ -110,6 +113,69 @@ def get_whole_page_url(header_url, header):
             break
     return list_url
 
+# select mysql data to chart
+def select_data_from_mysql():
+    select_db = pymysql.connect(host="localhost", user="root", password="123456", database="testdb", charset="utf8")
+    select_cursor = select_db.cursor()
+    select_sql = "SELECT * FROM ireadlist WHERE downloadcount > 6000"
+    try:
+        # 执行sql语句
+        select_cursor.execute(select_sql)
+        select_results = select_cursor.fetchall()
+
+        #for select_list in select_results:
+        #    print ("select name: %s, select count: %d" % (select_list[0], int(select_list[2])))
+    except:
+        print ("select error msg")
+    select_db.close()
+    return select_results
+
+# draw data matplot
+def draw_data_matplot(select_results):
+    list_name = []
+    list_count = []
+    for select_list in select_results:
+        list_name.append(select_list[0])
+        list_count.append(int(select_list[2]))
+        #print ("select name: %s, select count: %d" % (select_list[0], int(select_list[2])))
+    #plt.plot(list_count, 'bs')
+    quick_sort(list_count, 0, len(list_count)-1)
+    for i in list_count:
+        print ("quick sort: ", i)
+    plt.plot(list_count, 'bs')
+    plt.show()
+
+# draw data echart
+def draw_data_echart(select_results):
+    list_name = []
+    list_count = []
+    for select_list in select_results:
+        list_name.append(select_list[0])
+        list_count.append(int(select_list[2]))
+        #print ("select name: %s, select count: %d" % (select_list[0], int(select_list[2])))
+    bar = Bar("read weekly", "download count")
+    bar.use_theme('light')
+    bar.add("book download count", list_name, list_count, is_more_utils = True, is_label_show = True, is_datazoom_show = True)
+    bar.render("downloadcount.html")
+
+def sub_sort(array,low,high):
+    key = array[low]
+    while low < high:
+        while low < high and array[high] >= key:
+            high -= 1
+        while low < high and array[high] < key:
+            array[low] = array[high]
+            low += 1
+            array[high] = array[low]
+    array[low] = key
+    return low
+
+def quick_sort(array,low,high):
+    if low < high:
+        key_index = sub_sort(array,low,high)
+        quick_sort(array,low,key_index)
+        quick_sort(array,key_index+1,high)
+
 
 if __name__ == "__main__":
     #url = 'https://www.671cf.com/htm/index.htm'
@@ -135,17 +201,27 @@ def get_whole_page_url(header_url, header):
         page_html = get_page_html(url_unit, header, 3, page_encode)
         # get html data
         collect_data(page_html, queue_num)
-    '''
+
+    results = select_data_from_mysql()
+    #draw_data_matplot(results)
+    draw_data_echart(results)
 
     # test mysql update
-    test_url = 'http://www.ireadweek.com/index.php/index/16.html'
+    #test_url = 'http://www.ireadweek.com/index.php/index/16.html'
+    '''test_url = 'http://www.ireadweek.com/index.php/index/168.html'
     page_encode = get_page_encode(test_url, header)
 
     page_html = get_page_html(test_url, header, 3, page_encode)
 
     collect_data(page_html, 9)
     '''
 
+    '''plt.plot([1, 2, 3, 4])
+    plt.ylabel('some numbers')
+    plt.show()
+    '''
+
+