update proxy detector validate

lcstore · Feb 10, 2015 · bbec461 · bbec461
1 parent 093a2b0
commit bbec461
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 17 deletions.
diff --git a/iscript-core/iscript-core-client/src/test/java/com/lezo/iscript/yeam/ConfigParserTest.java b/iscript-core/iscript-core-client/src/test/java/com/lezo/iscript/yeam/ConfigParserTest.java
@@ -69,7 +69,7 @@ public void testConfig() throws Exception {
 		parser = new ConfigTmallProduct();
 		parser = new ConfigProxyChecker();
 		parser = new ConfigProxyDetector();
-		parser = new ConfigProxySeedHandler();
+//		parser = new ConfigProxySeedHandler();
 //		parser = new ConfigProxyCollector();
 //		parser = new ConfigJdPromotion();
 //		parser = new ConfigTmallBrandList();
@@ -147,22 +147,21 @@ public void testConfig() throws Exception {
 		task.put("port", 3128);
 
 		// http
-		task.put("ip", "111.1.36.130");
+		task.put("ip", "117.177.243.35");
 		task.put("port", 80);
-		task.put("ip", "180.168.217.86");
-		task.put("proxyType", 8081);
-		// socket
-		task.put("ip", "103.246.161.194");
-		task.put("port", 1080);
-		task.put("proxyType", 2);
-
-		task.put("ip", "222.87.129.218");
-		task.put("port", 83);
 		task.put("proxyType", 1);
-
-		task.put("url", "http://www.proxy.com.ru/list_%s.html");
-		task.put("seedId", "1");
-		task.put("CreateUrlsFun", "var oUrlArr = [];var maxCount=50;for(var i=1;i<=maxCount;i++){oUrlArr.push(java.lang.String.format(args.url,''+i));}return JSON.stringify(oUrlArr);");
+		// socket
+//		task.put("ip", "103.246.161.194");
+//		task.put("port", 1080);
+//		task.put("proxyType", 2);
+//
+//		task.put("ip", "222.87.129.218");
+//		task.put("port", 83);
+//		task.put("proxyType", 1);
+//
+//		task.put("url", "http://www.proxy.com.ru/list_%s.html");
+//		task.put("seedId", "1");
+//		task.put("CreateUrlsFun", "var oUrlArr = [];var maxCount=50;for(var i=1;i<=maxCount;i++){oUrlArr.push(java.lang.String.format(args.url,''+i));}return JSON.stringify(oUrlArr);");
 		String returnObject = parser.doParse(task);
 		System.out.println("result:" + returnObject);
 	}

diff --git a/...e/iscript-core-client/src/test/java/com/lezo/iscript/yeam/config/ConfigProxyDetector.java b/...e/iscript-core-client/src/test/java/com/lezo/iscript/yeam/config/ConfigProxyDetector.java
@@ -15,6 +15,9 @@
 import org.apache.http.util.EntityUtils;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.json.JSONObject;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.select.Elements;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -154,7 +157,9 @@ private void fillStatus(ProxyDetectDto tBean, HttpResponse res, String domain) t
 			limitRemarkLen(tBean);
 			return;
 		}
-		if (html != null && html.indexOf(domain) > 0) {
+		Document dom = Jsoup.parse(html);
+		Elements footEls = dom.select("a[href=http://www.baidu.com/duty/]");
+		if (!footEls.isEmpty()) {
 			tBean.setVerifyStatus(1);
 			tBean.setStatus(1);
 			return;

diff --git a/...ler/src/test/java/com/lezo/iscript/service/crawler/service/impl/ProxyServiceImplTest.java b/...ler/src/test/java/com/lezo/iscript/service/crawler/service/impl/ProxyServiceImplTest.java
@@ -29,7 +29,7 @@ public void testBatchInsertIfAbsent() throws Exception {
 		ProxyDetectServiceImpl proxyDetectService = new ProxyDetectServiceImpl();
 		proxyDetectService.setProxyDetectDao(proxyDetectDao);
 
-		Long fromId = 0L;
+		Long fromId = 199578L;
 		Integer limit = 10000;
 		int sum = 0;
 		long startMills = System.currentTimeMillis();