Merge branch 'master'

Dockerfile

@@ -1,7 +1,9 @@
 FROM python:3.6-alpine
 WORKDIR /app
 COPY . .
-RUN pip install -r requirements.txt  -i https://pypi.douban.com/simple
-# RUN pip install -r requirements.txt  -i
+# RUN pip install -r requirements.txt  -i https://pypi.douban.com/simple
+RUN apk add --no-cache libxml2-dev libxslt-dev gcc musl-dev && \
+pip install -r requirements.txt && \
+apk del gcc musl-dev libxml2-dev
 VOLUME ["/app/proxypool/crawlers/private"]
 CMD ["supervisord", "-c", "supervisord.conf"]

docker-compose.yml

@@ -4,7 +4,7 @@ services:
     image: redis:alpine
     container_name: redis4proxypool
     ports:
-      - "6378:6379"
+      - "6374:6379"
     # restart: always
   proxypool:
     build: .

examples/usage2.py

@@ -17,7 +17,7 @@ def getChinaIP(ip='127.0.0.1'):
     reader = geolite2.reader()
     ip_info = reader.get(ip)
     geolite2.close()
-    # print(ip_info)
+    print(ip_info)
     return True if ip_info['country']['iso_code'] == 'CN' else False
 
 
@@ -32,8 +32,8 @@ def run(self):
         pure_ip_address = self.proxyip.split(':')[0]
         # 验证IP归属
         if not getChinaIP(pure_ip_address):
-            pass
-            # raise ValueError('不是有效IP')
+            # pass
+            raise ValueError('不是有效IP')
         # 
         start = time.time()
         # 消除关闭证书验证的警告
@@ -88,8 +88,8 @@ def run(self):
     # apiUrl = "http://127.0.0.1:5555/all"
     apiUrl = "http://127.0.0.1:5555/random"
     # 要抓取的目标网站地址
-    # targetUrl = "http://bb.cf08tp.cn/Home/index.php?m=Index&a=vote&vid=335688&id=2676&tp="
-    targetUrl = 'http://www.so.com'
+    targetUrl = "http://bb.cf08tp.cn/Home/index.php?m=Index&a=vote&vid=335688&id=2676&tp="
+    # targetUrl = 'http://bb.cf08tp.cn/Home/index.php?m=Index&a=vote&vid=335608&id=2676&tp='
     fetchSecond = 5
     # 开始自动获取IP
     GetIpThread(fetchSecond).start()

proxypool/processors/tester.py

@@ -85,8 +85,8 @@ def run(self):
                 break
 
 def run_tester():
-    host = '111.246.42.52'
-    port = '8888'
+    host = '96.113.165.182'
+    port = '3128'
     tasks = [tester.test(Proxy(host=host, port=port))]
     tester.loop.run_until_complete(asyncio.wait(tasks))
 

requirements.txt

@@ -8,6 +8,6 @@ loguru==0.5.3
 pyquery==1.4.3
 supervisor==4.2.1
 redis==3.5.3
-lxml==4.6.2
+lxml==4.6.3
 fake_headers==1.0.2
 maxminddb_geolite2==2018.703