add scrapinghub and run.sh.

coinincome/pipelines.py

@@ -28,7 +28,8 @@ def from_settings(cls, settings):
         return cls(db_pool)
 
     def process_item(self, item, spider):
-        self.db_pool.runInteraction(self.insert_item, item)
+        if self.db_pool is not None:
+            self.db_pool.runInteraction(self.insert_item, item)
 
     def insert_item(self, cursor, item):
         insert_sql = """INSERT INTO pool_coin_incomes(pool_name, coin, income_coin, income_hashrate_unit, updated_at)
@@ -47,8 +48,15 @@ def insert_item(self, cursor, item):
             if math.isclose(row['income_coin'], item['income_coin'], rel_tol=1e-6):
                 return
             else:
-                print("diff:coin:%s:prev:%s:now:%s" % (item['coin'].encode('utf-8'), row['income_coin'],
-                                                       item['income_coin']))
+                if row['request_url'] != item['request_url']:
+                    text = "diff:coin:%s:prev:%s:time:%s:url:'%s':now:%s:url:'%s'" % (
+                        item['coin'], row['income_coin'], row['created_at'], row['request_url'], item['income_coin'],
+                        item['request_url'])
+                    print(text)
+                else:
+                    text = "diff:coin:%s:prev:%s:url:'%s':now:%s" % (
+                        item['coin'], row['income_coin'], row['request_url'], item['income_coin'])
+                    print(text)
         insert_history_sql = """INSERT INTO pool_coin_income_history(pool_name, coin, request_url, income_coin, 
         income_hashrate_unit, created_at)  VALUES (%s, %s, %s, %s, %s, now());"""
         cursor.execute(insert_history_sql, (item['pool_name'].encode('utf-8'), item['coin'].encode('utf-8'),

coinincome/settings.py

@@ -19,7 +19,7 @@
 USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36'
 
 # Obey robots.txt rules
-ROBOTSTXT_OBEY = True
+ROBOTSTXT_OBEY = False
 
 # Configure maximum concurrent requests performed by Scrapy (default: 16)
 # CONCURRENT_REQUESTS = 32
@@ -89,7 +89,6 @@
 # HTTPCACHE_IGNORE_HTTP_CODES = []
 # HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
 
-
 MYSQL_HOST = "127.0.0.1"
 MYSQL_DBNAME = "pool_coin_income"
 MYSQL_USER = "root"

coinincome/spiders/antpool.py

@@ -1,11 +1,38 @@
 # -*- coding: utf-8 -*-
+import json
+
 import scrapy
 
+from coinincome.items import CoinincomeItem
+
 
 class AntpoolSpider(scrapy.Spider):
     name = 'antpool'
     allowed_domains = ['antpool.com']
-    start_urls = ['http://antpool.com/']
+    start_urls = [
+        'https://v3.antpool.com/auth/v3/index/coinList',
+        'https://v3.antpool.com/lab/poolcoins'
+    ]
 
     def parse(self, response):
-        pass
+        response_body = json.loads(response.body)
+        if response_body['code'] == "000000":
+            for coin in response_body['data']['items']:
+                item = CoinincomeItem()
+                item['coin'] = coin['coinType'].lower()
+                # pps 收益 = 1t * 86400 / D / Coefficient * blockReward
+                item['income_coin'] = coin['calculateUnit'] * 86400 / coin['networkDiff'] * coin['blockReward'] / coin[
+                    'coinCoefficient']
+                item['income_hashrate_unit'] = self.calculate_unit(coin['calculateUnit'])
+                item['next_income_coin'] = 0
+                item['pool_name'] = self.name
+                item['request_url'] = response.url
+                yield item
+
+    def calculate_unit(self, share):
+        units = ["", "K", "M", "G", "T", "P", "E", "Z", "Y"]
+        index = 0
+        while share >= 1000:
+            share = share / 1000
+            index = index + 1
+        return units[index]

coinincome/spiders/f2pool.py

@@ -5,7 +5,9 @@
 class F2poolSpider(scrapy.Spider):
     name = 'f2pool'
     allowed_domains = ['f2pool.com']
-    start_urls = ['http://f2pool.com/']
+    start_urls = ['https://f2pool.com/']
 
     def parse(self, response):
-        pass
+        for project in response.xpath('//div/div/section/div/div/table/tbody/a[@class="btn-calculator"]'):
+            if project.extract() != '#':
+                print('https://www.f2pool.com%s' % project.extract())

coinincome/spiders/huobipool.py

@@ -17,8 +17,8 @@ def parse(self, response):
             for coin in response_body['data']:
                 item = CoinincomeItem()
                 item['coin'] = coin['currency'].lower()
-                item['income_coin'] = coin['profit']
-                item['income_hashrate_unit'] = coin['profit']
+                item['income_coin'] = float(coin['profit'].split()[0])
+                item['income_hashrate_unit'] = coin['profit'].split('/')[1]
                 item['next_income_coin'] = 0
                 item['pool_name'] = self.name
                 item['request_url'] = response.url

requirements.txt

@@ -0,0 +1 @@
+pymysql

run.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+cd coinincome
+scrapy runspider spiders/btccom.py
+scrapy runspider spiders/poolin.py
+scrapy runspider spiders/huobipool.py
+scrapy runspider spiders/antpool.py
+scrapy runspider spiders/sparkpool.py

scrapinghub.yml

@@ -0,0 +1,3 @@
+project: 409348
+requirements:
+  file: requirements.txt

setup.py

@@ -0,0 +1,10 @@
+# Automatically created by: shub deploy
+
+from setuptools import setup, find_packages
+
+setup(
+    name         = 'project',
+    version      = '1.0',
+    packages     = find_packages(),
+    entry_points = {'scrapy': ['settings = coinincome.settings']},
+)