etherscan.io refactor code

Author: carlosxiao

.gitignore

@@ -1,4 +1,5 @@
 /target/
+*.log
 !.mvn/wrapper/maven-wrapper.jar
 
 ### STS ###

pom.xml

@@ -101,6 +101,17 @@
 			<artifactId>velocity</artifactId>
 			<version>1.7</version>
 		</dependency>
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-pool2</artifactId>
+			<version>2.4.2</version>
+		</dependency>
+
+		<dependency>
+			<groupId>com.fasterxml.jackson.core</groupId>
+			<artifactId>jackson-databind</artifactId>
+			<version>2.8.0</version>
+		</dependency>
 	</dependencies>
 
 	<build>

src/main/java/com/cc/etherscan/io/mapper/EtherContractMapper.java

@@ -2,9 +2,11 @@
 
 import com.baomidou.mybatisplus.mapper.BaseMapper;
 import com.cc.etherscan.io.entity.EtherContract;
+import org.springframework.stereotype.Repository;
 
 /**
  * @author carlosxiao
  */
+@Repository
 public interface EtherContractMapper extends BaseMapper<EtherContract> {
 }

src/main/java/com/cc/etherscan/io/pipeline/EthereumPipeline.java

@@ -3,8 +3,11 @@
 import com.cc.etherscan.io.common.Constants;
 import com.cc.etherscan.io.entity.EtherContract;
 import com.cc.etherscan.io.mapper.EtherContractMapper;
+import lombok.extern.slf4j.Slf4j;
+import org.jsoup.helper.StringUtil;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Component;
+import org.springframework.util.StringUtils;
 import us.codecraft.webmagic.ResultItems;
 import us.codecraft.webmagic.Task;
 import us.codecraft.webmagic.pipeline.Pipeline;
@@ -15,6 +18,7 @@
  * @author carlosxiao
  */
 @Component
+@Slf4j
 public class EthereumPipeline implements Pipeline {
 
     @Resource
@@ -35,6 +39,10 @@ public void process(ResultItems resultItems, Task task) {
         String sourceCode = resultItems.get("sourceCode");
 
         String rAddress = (String) redisTemplate.opsForHash().get(String.format(Constants.REDIS_ETHER_EUM_KEY, address), "address");
+        if (StringUtils.isEmpty(rAddress) || StringUtil.isBlank(address)) {
+            log.error("address or rAddress is null: address: {}, aAddress: {}, RequestUrl: {}", address, rAddress, resultItems.getRequest().getUrl());
+            return;
+        }
         if (!address.equals(rAddress)) {
             return;
         }
@@ -52,6 +60,10 @@ public void process(ResultItems resultItems, Task task) {
         contract.setContractName(contractName);
         contract.setDateVerified(dateVerified);
         contract.setAccessUrl(accessUrl);
-        etherContractMapper.insert(contract);
+        try {
+            etherContractMapper.insert(contract);
+        } catch (Exception e) {
+            log.error("insert failure: {}", e);
+        }
     }
 }

src/main/java/com/cc/etherscan/io/processor/EthereumContractProcessor.java

@@ -3,6 +3,7 @@
 import com.cc.etherscan.io.common.Constants;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.util.StringUtils;
 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.Site;
 import us.codecraft.webmagic.processor.PageProcessor;
@@ -41,8 +42,12 @@ public void process(Page page) {
                 String contractName = page.getHtml().xpath("/html/body/div[1]/div[5]/div[3]/div/div/div/table/tbody/tr[" + i + "]/td[2]/text()").get();
                 String dateVerified = page.getHtml().xpath("/html/body/div[1]/div[5]/div[3]/div/div/div/table/tbody/tr[" + i + "]/td[7]/text()").get();
                 String url = "https://etherscan.io/address/" + address + "#code";
+                if (StringUtils.isEmpty(address)) {
+                    log.error("cannot resolve address, RequestUrl: {}", page.getRequest().getUrl());
+                    continue;
+                }
                 if (redisTemplate.hasKey(String.format(REDIS_ETHER_EUM_KEY, address))) {
-                    return;
+                    continue;
                 }
                 page.addTargetRequest(url);
                 Map<String, String> map = new HashMap<>();

src/main/java/com/cc/etherscan/io/schedule/EtherEumSchedule.java

@@ -22,20 +22,20 @@ public class EtherEumSchedule {
     @Resource
     private EthereumPipeline ethereumPipeline;
 
-    int pageSize = 100;
+    int pageSize = 10;
 
-    int totalPage = 390;
+    int totalPage = 3898;
 
     @PostConstruct
     public void start() {
         for (int i = 1; i<= totalPage; i++) {
             Spider.create(new EthereumContractProcessor(redisTemplate))
                     .addUrl("https://etherscan.io/contractsVerified/" + i + "?ps=" + pageSize)
                     .addPipeline(ethereumPipeline)
-                    .thread(5)
+                    .thread(8)
                     .run();
             try {
-                Thread.sleep(10000);
+                Thread.sleep(4000);
             } catch (InterruptedException e) {
                 e.printStackTrace();
             }

src/main/resources/application-dev.yml

@@ -0,0 +1,9 @@
+spring:
+  redis:
+    host: 127.0.0.1
+    port: 6379
+    password: 123456
+  datasource:
+    url: jdbc:mysql://127.0.0.1:3306/esc?useUnicode=true&characterEncoding=utf8
+    username: root
+    password: root

src/main/resources/application-test.yml

@@ -0,0 +1,9 @@
+spring:
+  redis:
+    sentinel:
+      nodes: 192.168.13.50:20190,192.168.13.50:20191,192.168.13.50:20192
+      master: mymaster
+  datasource:
+    url: jdbc:mysql://192.168.13.60:3306/esc?useUnicode=true&characterEncoding=utf8
+    username: root
+    password: root

src/main/resources/application.yml

@@ -1,19 +1,14 @@
 spring:
-  redis:
-    host: 127.0.0.1
-    port: 6379
-    password: 123456
   datasource:
     driverClassName: com.mysql.jdbc.Driver
-    url: jdbc:mysql://127.0.0.1:3306/esc?useUnicode=true&characterEncoding=utf8
-    username: root
-    password: root
     type: com.zaxxer.hikari.HikariDataSource
     hikari:
       minimum-idle: 10
       maximum-pool-size: 200
       pool-name: ESC-POOL
       connection-test-query: SELECT 1 FROM DUAL
+  profiles:
+    active: test
 liquibase:
   change-log: classpath:db-changelog.xml
 mybatis-plus:

src/main/resources/changelog/ether-contract-changelog.xml

@@ -17,7 +17,7 @@
             <column name="transactions" type="varchar(20)" remarks=""/>
             <column name="create_address" type="varchar(128)" remarks=""/>
             <column name="txn" type="varchar(128)" remarks=""/>
-            <column name="source_code" type="text" remarks="源代码"/>
+            <column name="source_code" type="longblob" remarks="源代码"/>
         </createTable>
         <createIndex tableName="ether_contract" unique="true" indexName="INDEX_ETHER_CONTRACT_ADDRESS">
             <column name="address" />

src/main/resources/logback.xml

@@ -24,7 +24,7 @@
 　　</root>
 
     <!--myibatis log configure-->
-    <logger name="com.apache.ibatis" level="ERROR"/>
+    <logger name="com.apache.ibatis" level="DEBUG"/>
     <logger name="java.sql.Connection" level="DEBUG"/>
     <logger name="java.sql.Statement" level="DEBUG"/>
     <logger name="java.sql.PreparedStatement" level="DEBUG"/>