reamd me

README.MD

@@ -20,11 +20,12 @@
 * fund/fund_share_update.py 上交所，深交所 基金场内基金份额监控
 * fund/fund_share_monitor.py 上交所，深交所 基金基金份额查询，规模变动
 * fund/fund_info_spider.py 集思录基金,腾讯证券基金折价率，溢价率 爬虫
-* etf_info.py 市场指数基金的持仓股监控
+* fund/etf_info.py 市场指数基金的持仓股监控
+* fund/ttjj.py 天天基金数据获取
 
 
 ### 已有：
-* black_list_sql.py 记录A股市场上所有有黑历史的股票名单，并存入数据库
+* datahub/black_list_sql.py 记录A股市场上所有有黑历史的股票名单，并存入数据库
 * big_deal.py 监控每天A股市场上的大单交易
 * bond_monitor 可转债监控
 * ceiling_break.py 新股一直板开板后多少天能够重新回到开板价格

fund/danjuan_fund.py

@@ -34,7 +34,7 @@ def mongo_init(self):
         try:
             self.main_doc.ensure_index('plan_code', unique=True)
         except Exception as e:
-            print(e)
+            self.logger.error(e)
 
     @property
     def headers(self):
@@ -55,22 +55,16 @@ def save_data(self, data_list):
             try:
                 self.main_doc.insert_one(item)
             except Exception as e:
-                pass
-            else:
-                pass
+                self.logger.error(e)
+
 
     def get_plan_code(self):
-        for page in range(1, 50):
+        for page in range(1, 50): # 暂定 50页，实际数据很少
             content = self.crawl(page)
             return_data = self.parse(content)
             self.save_data(return_data)
             time.sleep(1)
-            print('page ', page)
 
-    def run(self):
-        self.get_plan_code() # 获取plan code 并入库
-        self.get_detail() # 获取具体持仓
-        self.plan_detail()  # 方案的持有信息，收益等
 
     @property
     def code_list(self):
@@ -81,8 +75,7 @@ def update_data(self, condition, data):
         try:
             self.main_doc.update_one(condition, {'$set': data})
         except Exception as e:
-            # pass
-            print(e)
+            self.logger.error(e)
         else:
             pass
 
@@ -97,7 +90,7 @@ def plan_detail(self):
                 detail_info = self.post_process(detail_info)
                 self.update_data({'plan_code': code}, detail_info)
             else:
-                print('code {} is empty'.format(code))
+                self.logger.error('code {} is empty'.format(code))
 
     def post_process(self, detail_info):
         '''
@@ -127,7 +120,13 @@ def get_detail(self):
                 holdings = content.get('data').get('items')
                 self.update_data({'plan_code': code}, {"holding": holdings})
             else:
-                print('code {} is empty'.format(code))
+                self.logger.error('code {} is empty'.format(code))
+
+
+    def run(self):
+        self.get_plan_code() # 获取plan code 并入库
+        self.get_detail() # 获取具体持仓
+        self.plan_detail()  # 方案的持有信息，收益等
 
 
 if __name__ == '__main__':

fund/danjuan_fund_data_analysis.py

@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+# @Time : 2021/4/20 12:54
+# @File : danjuan_fund_data_analysis.py
+# @Author : Rocky C@www.30daydo.com
+# 蛋卷数据分析
+import datetime
+import sys
+from collections import defaultdict
+sys.path.append('..')
+from configure.settings import DBSelector
+from common.BaseService import BaseService
+import pandas as pd
+
+
+WEEK_DAY = -7 # 上一周的价格
+
+class DanjuanAnalyser(BaseService):
+
+    def __init__(self):
+        super(DanjuanAnalyser, self).__init__('../log/Danjuan_analysis.log')
+
+
+    def select_collection(self,current_date):
+        self.db = DBSelector().mongo(location_type='qq')
+        doc = self.db['db_danjuan'][f'danjuan_fund_{current_date}']
+        return doc
+
+    def get_top_plan(self,collection,top=10):
+        fund_dict = {}
+        for item in collection.find({},{'holding':1}):
+            plan_holding = item.get('holding',[]) # list
+            for hold in plan_holding:
+                name = hold['fd_name']
+                fund_dict.setdefault(name,0)
+                fund_dict[name]+=1
+        fund_dict=list(sorted(fund_dict.items(),key=lambda x:x[1],reverse=True))[:top]
+        return fund_dict
+
+
+    def get_top_plan_percent(self,collection,top=10):
+        fund_dict = {}
+        for item in collection.find({},{'holding':1}):
+            plan_holding = item.get('holding',[]) # list
+            for hold in plan_holding:
+                name = hold['fd_name']
+                percent =hold['percent']
+                fund_dict.setdefault(name,0)
+                fund_dict[name]+=percent
+        fund_dict=list(sorted(fund_dict.items(),key=lambda x:x[1],reverse=True))[:top]
+        return fund_dict
+
+    def start(self):
+
+        today=datetime.datetime.now()
+        last_week = today + datetime.timedelta(days=WEEK_DAY)
+        last_week_str = last_week.strftime('%Y-%m-%d')
+        # 因为没有执行上周的数据，用历史数据替代
+        last_week_str = '2021-03-18'
+
+        today_doc = self.select_collection(self.today)
+        last_week_doc = self.select_collection(last_week_str)
+
+        # 持有个数
+        # fund_dict = self.get_top_plan(today_doc,20)
+        # self.pretty(fund_dict,self.today)
+
+        # old_fund_dict = self.get_top_plan(last_week_doc,20)
+        # self.pretty(old_fund_dict,last_week_str)
+        # diff_set = self.new_fund(fund_dict,old_fund_dict)
+        # print(diff_set)
+
+        # 按持有比例
+        # new_fund_percent = self.get_top_plan_percent(today_doc,20)
+        # old_fund_percent = self.get_top_plan_percent(last_week_doc,20)
+        #
+        # self.pretty(new_fund_percent,self.today,'percent')
+        # self.pretty(old_fund_percent,last_week_str,'percnet')
+
+        # 清仓
+        clean_fund = self.clear_warehouse_fund(today_doc,200)
+        self.simple_display(clean_fund)
+
+    def simple_display(self,data):
+        for i in data:
+            print(i)
+
+    def pretty(self,fund_dict,date,kind):
+        df = pd.DataFrame(fund_dict,columns=['fund','holding_num'])
+        print(df.head(100))
+        df.to_excel(f'{date}-{kind}.xlsx')
+
+    def new_fund(self,new_fund_dict,old_fund_dict):
+        new_fund_list = list(map(lambda x: x[0], new_fund_dict))
+        old_fund_list = list(map(lambda x: x[0], old_fund_dict))
+        diff_set= set(old_fund_list)-set(new_fund_list)
+        return diff_set
+
+    def clear_warehouse_fund(self,collection,top):
+        '''
+        清仓的基金
+        '''
+        fund_dict = {}
+        for item in collection.find({},{'holding':1}):
+            plan_holding = item.get('holding',[]) # list
+            for hold in plan_holding:
+                name = hold['fd_name']
+                percent =hold['percent']
+
+                if percent>0:
+                    continue
+
+                fund_dict.setdefault(name,0)
+                fund_dict[name]+=1
+        fund_dict=list(sorted(fund_dict.items(),key=lambda x:x[1],reverse=True))[:top]
+        return fund_dict
+
+
+def main():
+    app = DanjuanAnalyser()
+    app.start()
+
+if __name__ == '__main__':
+    main()