feat: 集成 Freeproxy 提供的代理并提供单独获取匿名代理的方法

Dockerfile

@@ -1,5 +1,13 @@
 FROM python:3.6-alpine
 
+# 设置标签
+LABEL name="proxy-pool" \
+      version="1.0.0" \
+      project="ci" \
+      platform="linux/amd64,linux/arm64" \
+      maintainer="andy" \
+      description="Proxy Pool Service"
+
 MAINTAINER jhao104 <j_hao104@163.com>
 
 WORKDIR /app

api/proxyApi.py

@@ -46,8 +46,9 @@ def force_type(cls, response, environ=None):
     {"url": "/pop", "params": "", "desc": "get and delete a proxy"},
     {"url": "/delete", "params": "proxy: 'e.g. 127.0.0.1:8080'", "desc": "delete an unable proxy"},
     {"url": "/all", "params": "type: ''https'|''", "desc": "get all proxy from proxy pool"},
-    {"url": "/count", "params": "", "desc": "return proxy count"}
-    # 'refresh': 'refresh proxy pool',
+    {"url": "/count", "params": "", "desc": "return proxy count"},
+    {"url": "/get_anonymous", "params": "type: ''https'|''", "desc": "get an anonymous proxy"},
+    {"url": "/all_anonymous", "params": "type: ''https'|''", "desc": "get all anonymous proxy"},
 ]
 
 
@@ -93,14 +94,36 @@ def delete():
 @app.route('/count/')
 def getCount():
     proxies = proxy_handler.getAll()
+    anonymous_proxies = proxy_handler.getAllAnonymous()
     http_type_dict = {}
     source_dict = {}
     for proxy in proxies:
         http_type = 'https' if proxy.https else 'http'
         http_type_dict[http_type] = http_type_dict.get(http_type, 0) + 1
         for source in proxy.source.split('/'):
             source_dict[source] = source_dict.get(source, 0) + 1
-    return {"http_type": http_type_dict, "source": source_dict, "count": len(proxies)}
+    return {
+        "http_type": http_type_dict,
+        "source": source_dict,
+        "count": len(proxies),
+        "anonymous_count": len(anonymous_proxies)
+    }
+
+
+@app.route('/get_anonymous/')
+def get_anonymous():
+    """获取一个匿名代理"""
+    https = request.args.get("type", "").lower() == 'https'
+    proxy = proxy_handler.getAnonymous(https)
+    return proxy.to_dict if proxy else {"code": 0, "src": "no anonymous proxy"}
+
+
+@app.route('/all_anonymous/')
+def get_all_anonymous():
+    """获取所有匿名代理"""
+    https = request.args.get("type", "").lower() == 'https'
+    proxies = proxy_handler.getAllAnonymous(https)
+    return jsonify([_.to_dict for _ in proxies])
 
 
 def runFlask():
@@ -118,7 +141,7 @@ def __init__(self, app, options=None):
 
             def load_config(self):
                 _config = dict([(key, value) for key, value in iteritems(self.options)
-                                if key in self.cfg.settings and value is not None])
+                               if key in self.cfg.settings and value is not None])
                 for key, value in iteritems(_config):
                     self.cfg.set(key.lower(), value)
 

db/dbClient.py

@@ -116,5 +116,23 @@ def changeTable(self, name):
     def getCount(self):
         return self.client.getCount()
 
+    def changeAnonymousTable(self, name):
+        self.client.changeAnonymousTable(name)
+
+    def putAnonymous(self, key, **kwargs):
+        return self.client.putAnonymous(key, **kwargs)
+
+    def getAnonymous(self, https, **kwargs):
+        return self.client.getAnonymous(https, **kwargs)
+
+    def getAllAnonymous(self, https):
+        return self.client.getAllAnonymous(https)
+
+    def deleteAnonymous(self, key, **kwargs):
+        return self.client.deleteAnonymous(key, **kwargs)
+
+    def getAnonymousCount(self):
+        return self.client.getAnonymousCount()
+
     def test(self):
         return self.client.test()

db/redisClient.py

@@ -138,6 +138,73 @@ def changeTable(self, name):
         """
         self.name = name
 
+    def changeAnonymousTable(self, name):
+        """
+        切换匿名代理表
+        :param name:
+        :return:
+        """
+        self.anonymous_name = name
+
+    def putAnonymous(self, proxy_obj):
+        """
+        将匿名代理放入 anonymous hash
+        :param proxy_obj: Proxy obj
+        :return:
+        """
+        if not hasattr(self, 'anonymous_name') or not self.anonymous_name:
+            return 0
+        return self.__conn.hset(self.anonymous_name, proxy_obj.proxy, proxy_obj.to_json)
+
+    def getAnonymous(self, https):
+        """
+        返回一个匿名代理
+        :return:
+        """
+        if not hasattr(self, 'anonymous_name') or not self.anonymous_name:
+            return None
+        if https:
+            items = self.__conn.hvals(self.anonymous_name)
+            proxies = list(filter(lambda x: json.loads(x).get("https"), items))
+            return choice(proxies) if proxies else None
+        else:
+            proxies = self.__conn.hkeys(self.anonymous_name)
+            proxy = choice(proxies) if proxies else None
+            return self.__conn.hget(self.anonymous_name, proxy) if proxy else None
+
+    def getAllAnonymous(self, https):
+        """
+        返回所有匿名代理
+        :return:
+        """
+        if not hasattr(self, 'anonymous_name') or not self.anonymous_name:
+            return []
+        items = self.__conn.hvals(self.anonymous_name)
+        if https:
+            return list(filter(lambda x: json.loads(x).get("https"), items))
+        else:
+            return items
+
+    def deleteAnonymous(self, proxy_str):
+        """
+        删除匿名代理
+        :param proxy_str:
+        :return:
+        """
+        if not hasattr(self, 'anonymous_name') or not self.anonymous_name:
+            return 0
+        return self.__conn.hdel(self.anonymous_name, proxy_str)
+
+    def getAnonymousCount(self):
+        """
+        返回匿名代理数量
+        :return:
+        """
+        if not hasattr(self, 'anonymous_name') or not self.anonymous_name:
+            return {'total': 0, 'https': 0}
+        proxies = self.getAllAnonymous(https=False)
+        return {'total': len(proxies), 'https': len(list(filter(lambda x: json.loads(x).get("https"), proxies)))}
+
     def test(self):
         log = LogHandler('redis_client')
         try:

docker-compose.yml

@@ -1,14 +1,10 @@
 version: '2'
 services:
   proxy_pool:
-    build: .
-    container_name: proxy_pool
+    image: harbor.vocoor.com/ci/proxy-pool:latest
+    container_name: proxy-pool
     ports:
       - "5010:5010"
-    links:
-      - proxy_redis
     environment:
-      DB_CONN: "redis://@proxy_redis:6379/0"
-  proxy_redis:
-    image: "redis"
-    container_name: proxy_redis
+      DB_CONN: "redis://:lilishop@192.168.2.130:6379/10"
+    restart: unless-stopped