add f2pool, viabtc.

coinincome/items.py

@@ -15,5 +15,6 @@ class CoinincomeItem(scrapy.Item):
     income_coin = scrapy.Field()
     next_income_coin = scrapy.Field()
     income_hashrate_unit = scrapy.Field()
+    income_hashrate_unit_num = scrapy.Field()
     request_url = scrapy.Field()
     pass

coinincome/pipelines.py

@@ -32,13 +32,16 @@ def process_item(self, item, spider):
             self.db_pool.runInteraction(self.insert_item, item)
 
     def insert_item(self, cursor, item):
-        insert_sql = """INSERT INTO pool_coin_incomes(pool_name, coin, income_coin, income_hashrate_unit, updated_at)
-                             VALUES (%s, %s, %s, %s, now()) ON DUPLICATE KEY UPDATE pool_name = %s, coin = %s, 
-                             income_coin = %s, income_hashrate_unit = %s, updated_at = now();"""
+        insert_sql = """INSERT INTO pool_coin_incomes(pool_name, coin, income_coin, income_hashrate_unit, 
+                            income_hashrate_unit_num, updated_at) VALUES 
+                            (%s, %s, %s, %s, %s, now()) ON DUPLICATE KEY UPDATE pool_name = %s, coin = %s, 
+                             income_coin = %s, income_hashrate_unit = %s, income_hashrate_unit_num = %s, 
+                             updated_at = now();"""
         params = (item['pool_name'].encode('utf-8'), item['coin'].encode('utf-8'),
                   item['income_coin'], item['income_hashrate_unit'].encode('utf-8'),
+                  item['income_hashrate_unit_num'],
                   item['pool_name'].encode('utf-8'), item['coin'].encode('utf-8'),
-                  item['income_coin'], item['income_hashrate_unit'].encode('utf-8'))
+                  item['income_coin'], item['income_hashrate_unit'].encode('utf-8'), item['income_hashrate_unit_num'])
         cursor.execute(insert_sql, params)
         sql = "SELECT * FROM pool_coin_income_history WHERE pool_name = %s and coin = %s order by " \
               "created_at desc limit 1;"

coinincome/spiders/antpool.py

@@ -2,10 +2,18 @@
 import json
 
 import scrapy
-
 from coinincome.items import CoinincomeItem
 
 
+def calculate_unit(share):
+    units = ["", "K", "M", "G", "T", "P", "E", "Z", "Y"]
+    index = 0
+    while share >= 1000:
+        share = share / 1000
+        index = index + 1
+    return units[index]
+
+
 class AntpoolSpider(scrapy.Spider):
     name = 'antpool'
     allowed_domains = ['antpool.com']
@@ -23,16 +31,9 @@ def parse(self, response):
                 # pps 收益 = 1t * 86400 / D / Coefficient * blockReward
                 item['income_coin'] = coin['calculateUnit'] * 86400 / coin['networkDiff'] * coin['blockReward'] / coin[
                     'coinCoefficient']
-                item['income_hashrate_unit'] = self.calculate_unit(coin['calculateUnit'])
+                item['income_hashrate_unit'] = calculate_unit(coin['calculateUnit'])
+                item['income_hashrate_unit_num'] = coin['calculateUnit']
                 item['next_income_coin'] = 0
                 item['pool_name'] = self.name
                 item['request_url'] = response.url
                 yield item
-
-    def calculate_unit(self, share):
-        units = ["", "K", "M", "G", "T", "P", "E", "Z", "Y"]
-        index = 0
-        while share >= 1000:
-            share = share / 1000
-            index = index + 1
-        return units[index]

coinincome/spiders/btccom.py

@@ -7,6 +7,7 @@
 # https://pool.btc.com/v1/coins-income
 # Main
 from coinincome.items import CoinincomeItem
+from coinincome.utils import calculate_unit
 
 
 class BtccomSpider(scrapy.Spider):
@@ -26,6 +27,8 @@ def parse(self, response):
                 item['coin'] = coin.lower()
                 item['income_coin'] = response_body['data'][coin]['income_coin']
                 item['income_hashrate_unit'] = response_body['data'][coin]['income_hashrate_unit']
+                item['income_hashrate_unit_num'] = calculate_unit(
+                    response_body['data'][coin]['income_hashrate_unit'])
                 item['next_income_coin'] = response_body['data'][coin]['next_income_coin']
                 item['pool_name'] = self.name
                 item['request_url'] = response.url

coinincome/spiders/f2pool.py

@@ -1,13 +1,29 @@
 # -*- coding: utf-8 -*-
+import json
+
 import scrapy
 
+from coinincome.items import CoinincomeItem
+from coinincome.utils import calculate_unit
+
 
 class F2poolSpider(scrapy.Spider):
     name = 'f2pool'
     allowed_domains = ['f2pool.com']
-    start_urls = ['https://f2pool.com/']
+    start_urls = ['https://www.f2pool.com/']
 
     def parse(self, response):
-        for project in response.xpath('//div/div/section/div/div/table/tbody/a[@class="btn-calculator"]'):
-            if project.extract() != '#':
-                print('https://www.f2pool.com%s' % project.extract())
+
+        for coin_config in response.xpath(
+                '//tr[contains(@class, "row-common")]/td/a[contains(@class, "btn-calculator")]/@data-config'):
+            if coin_config.extract() != '':
+                coin = json.loads(coin_config.extract())
+                item = CoinincomeItem()
+                item['coin'] = coin['currency'].lower()
+                item['income_coin'] = float(coin['estimatedProfit'])
+                item['income_hashrate_unit'] = coin['scale']
+                item['income_hashrate_unit_num'] = calculate_unit(coin['scale'])
+                item['next_income_coin'] = 0
+                item['pool_name'] = self.name
+                item['request_url'] = response.url
+                yield item

coinincome/spiders/huobipool.py

@@ -4,6 +4,7 @@
 import scrapy
 
 from coinincome.items import CoinincomeItem
+from coinincome.utils import calculate_unit
 
 
 class HuobipoolSpider(scrapy.Spider):
@@ -19,6 +20,7 @@ def parse(self, response):
                 item['coin'] = coin['currency'].lower()
                 item['income_coin'] = float(coin['profit'].split()[0])
                 item['income_hashrate_unit'] = coin['profit'].split('/')[1]
+                item['income_hashrate_unit_num'] = calculate_unit(coin['profit'].split('/')[1])
                 item['next_income_coin'] = 0
                 item['pool_name'] = self.name
                 item['request_url'] = response.url

coinincome/spiders/poolin.py

@@ -6,6 +6,7 @@
 # https://api-prod.poolin.com/api/public/v2/basedata/coins/block_stats
 ###
 from coinincome.items import CoinincomeItem
+from coinincome.utils import calculate_unit
 
 
 class PoolinSpider(scrapy.Spider):
@@ -19,8 +20,9 @@ def parse(self, response):
             for coin in response_body['data']:
                 item = CoinincomeItem()
                 item['coin'] = coin.lower()
-                item['income_coin'] = response_body['data'][coin]['rewards_per_unit']
+                item['income_coin'] = float(response_body['data'][coin]['rewards_per_unit'])
                 item['income_hashrate_unit'] = response_body['data'][coin]['reward_unit']
+                item['income_hashrate_unit_num'] = calculate_unit(response_body['data'][coin]['reward_unit'])
                 item['next_income_coin'] = 0
                 item['pool_name'] = self.name
                 item['request_url'] = response.url

coinincome/spiders/sparkpool.py

@@ -19,6 +19,7 @@ def parse(self, response):
                 item['coin'] = coin['currency'].lower()
                 item['income_coin'] = coin['income']
                 item['income_hashrate_unit'] = coin['incomeHashrate']
+                item['income_hashrate_unit_num'] = coin['incomeHashrate']
                 item['next_income_coin'] = 0
                 item['pool_name'] = self.name
                 item['request_url'] = response.url

coinincome/spiders/viabtc.py

@@ -1,11 +1,32 @@
 # -*- coding: utf-8 -*-
 import scrapy
 
+from coinincome.items import CoinincomeItem
+from coinincome.utils import calculate_unit
+
 
 class ViabtcSpider(scrapy.Spider):
     name = 'viabtc'
     allowed_domains = ['viabtc.com']
-    start_urls = ['http://pool.viabtc.com/']
+    start_urls = ['https://pool.viabtc.com/']
 
     def parse(self, response):
-        pass
+        # coinname = response.xpath('//div/div[@style="max-width:160px;"]/text()')
+        coinname = response.xpath('//div/div[@class="m-l-10"]/text()')
+        for coin in coinname.extract():
+            if coin.startswith("¥"):
+                continue
+            income_str_list = coin.split()
+            income_hashrate_unit = income_str_list[0][0]
+            standard_units = ["", "K", "M", "G", "T", "P", "E", "Z", "Y"]
+            if income_hashrate_unit not in standard_units:
+                income_hashrate_unit = ""
+            item = CoinincomeItem()
+            item['coin'] = income_str_list[3].lower()
+            item['income_coin'] = float(income_str_list[2])
+            item['income_hashrate_unit'] = income_hashrate_unit
+            item['income_hashrate_unit_num'] = calculate_unit(income_str_list[0][0])
+            item['next_income_coin'] = 0
+            item['pool_name'] = self.name
+            item['request_url'] = response.url
+            yield item

coinincome/utils.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+
+def calculate_unit(unit):
+    units = ["", "K", "M", "G", "T", "P", "E", "Z", "Y"]
+    index = 0
+    while True:
+        if unit == units[index]:
+            break
+        index = index + 1
+    return pow(1000, index)