Merge remote-tracking branch 'origin/master'

Author: carlosxiao

src/main/java/com/cc/etherscan/io/processor/EthereumContractProcessor.java

@@ -1,6 +1,10 @@
 package com.cc.etherscan.io.processor;
 
+import com.baomidou.mybatisplus.mapper.EntityWrapper;
+import com.baomidou.mybatisplus.mapper.Wrapper;
 import com.cc.etherscan.io.common.Constants;
+import com.cc.etherscan.io.entity.EtherContract;
+import com.cc.etherscan.io.mapper.EtherContractMapper;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.util.StringUtils;
@@ -26,8 +30,11 @@ public class EthereumContractProcessor implements PageProcessor {
 
     private RedisTemplate redisTemplate;
 
-    public EthereumContractProcessor(RedisTemplate template) {
+    private EtherContractMapper etherContractMapper;
+
+    public EthereumContractProcessor(RedisTemplate template, EtherContractMapper etherContractMapper) {
         this.redisTemplate = template;
+        this.etherContractMapper = etherContractMapper;
     }
 
 
@@ -49,7 +56,14 @@ public void process(Page page) {
                     continue;
                 }
                 if (redisTemplate.hasKey(String.format(REDIS_ETHER_EUM_KEY, address))) {
-                    continue;
+                    EtherContract queryEntity = new EtherContract();
+                    queryEntity.setAddress(address);
+                    Wrapper<EtherContract> wrapper = new EntityWrapper<>();
+                    wrapper.eq("address", address);
+                    Integer count = etherContractMapper.selectCount(wrapper);
+                    if (count > 0) {
+                        continue;
+                    }
                 }
                 page.addTargetRequest(url);
                 Map<String, String> map = new HashMap<>();

src/main/java/com/cc/etherscan/io/schedule/EtherEumSchedule.java

@@ -1,5 +1,6 @@
 package com.cc.etherscan.io.schedule;
 
+import com.cc.etherscan.io.mapper.EtherContractMapper;
 import com.cc.etherscan.io.pipeline.EthereumPipeline;
 import com.cc.etherscan.io.processor.EthereumContractProcessor;
 import org.springframework.beans.factory.annotation.Value;
@@ -23,6 +24,9 @@ public class EtherEumSchedule {
     @Resource
     private EthereumPipeline ethereumPipeline;
 
+    @Resource
+    private EtherContractMapper etherContractMapper;
+
     @Value("${etherscan.pageSize}")
     private int pageSize = 0;
 
@@ -41,7 +45,7 @@ public class EtherEumSchedule {
     @Scheduled(cron="0 0/2 * * * ?")
     public void start() {
         for (int i = startPage; i<= totalPage; i++) {
-            Spider.create(new EthereumContractProcessor(redisTemplate))
+            Spider.create(new EthereumContractProcessor(redisTemplate, etherContractMapper))
                     .addUrl("https://etherscan.io/contractsVerified/" + i + "?ps=" + pageSize)
                     .addPipeline(ethereumPipeline)
                     .thread(threadCount)

src/main/resources/application.yml

@@ -9,8 +9,8 @@ spring:
       connection-test-query: SELECT 1 FROM DUAL
   profiles:
     active: test
-#liquibase:
-  #change-log: classpath:db-changelog.xml
+liquibase:
+  change-log: classpath:db-changelog.xml
 mybatis-plus:
   mapper-locations: classpath*:sqlmap/*Mapper.xml
   typeAliasesPackage: com.cc.etherscan.io.entity

src/main/resources/db-changelog.xml

@@ -7,7 +7,7 @@
     <property name="now" value="sysdate" dbms="mysql"/>
 
     <!--原则上，这里只留下以项目模块为单位的include, 名字建议采用模块全名-->
-    <include file="classpath:changelog/ether-contract-changelog.xml"/>
+    <!--<include file="classpath:changelog/ether-contract-changelog.xml"/>-->
 
 
 </databaseChangeLog>

src/main/resources/logback.xml

@@ -16,7 +16,7 @@
             <maxFileSize>5MB</maxFileSize>
         </triggeringPolicy>
         <encoder>
-            <pattern>%-4relative [%thread] %-5level %logger{35} - %msg%n</pattern>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
         </encoder>
     </appender>
     <root level="INFO">