From e7f500d4adf5c12a5bf15947496576fe23852133 Mon Sep 17 00:00:00 2001 From: lzw2006 Date: Tue, 23 Oct 2018 10:40:42 +0800 Subject: [PATCH] =?UTF-8?q?#527=20=E5=88=9D=E6=AC=A1=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- saturn-console-api/pom.xml | 20 +++++ .../console/mybatis/entity/DomainCount.java | 55 ++++++++++++ .../repository/DomainCountRepository.java | 45 ++++++++++ .../statistics/StatisticsPersistence.java | 33 +++++--- .../resources/mapper/DomainCountMapper.xml | 44 ++++++++++ .../repository/DomainCountRepositoryTest.java | 84 +++++++++++++++++++ .../src/test/resources/applicationContext.xml | 49 +++++++++++ .../src/test/resources/global.sql | 1 + .../src/test/resources/schema.sql | 11 +++ 9 files changed, 332 insertions(+), 10 deletions(-) create mode 100644 saturn-console-api/src/main/java/com/vip/saturn/job/console/mybatis/entity/DomainCount.java create mode 100644 saturn-console-api/src/main/java/com/vip/saturn/job/console/mybatis/repository/DomainCountRepository.java create mode 100644 saturn-console-api/src/main/resources/mapper/DomainCountMapper.xml create mode 100644 saturn-console-api/src/test/java/com/vip/saturn/job/console/com/vip/saturn/job/console/mybatis/repository/DomainCountRepositoryTest.java create mode 100644 saturn-console-api/src/test/resources/applicationContext.xml create mode 100644 saturn-console-api/src/test/resources/global.sql create mode 100644 saturn-console-api/src/test/resources/schema.sql diff --git a/saturn-console-api/pom.xml b/saturn-console-api/pom.xml index 521c65541..1b1bf5c57 100644 --- a/saturn-console-api/pom.xml +++ b/saturn-console-api/pom.xml @@ -166,10 +166,30 @@ org.mybatis mybatis + + org.mybatis + mybatis-spring + test + org.mybatis.generator mybatis-generator-core + + org.springframework + spring-jdbc + test + + + com.h2database + h2 + test + + + mysql + mysql-connector-java + test + diff --git a/saturn-console-api/src/main/java/com/vip/saturn/job/console/mybatis/entity/DomainCount.java b/saturn-console-api/src/main/java/com/vip/saturn/job/console/mybatis/entity/DomainCount.java new file mode 100644 index 000000000..c57ed99c1 --- /dev/null +++ b/saturn-console-api/src/main/java/com/vip/saturn/job/console/mybatis/entity/DomainCount.java @@ -0,0 +1,55 @@ +package com.vip.saturn.job.console.mybatis.entity; + +import java.util.Date; + +/** + * @author Ray Leung + */ +public class DomainCount { + + private int id; + private String zkCluster; + private int successCount; + private int failCount; + private Date recordDate; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public int getSuccessCount() { + return successCount; + } + + public void setSuccessCount(int successCount) { + this.successCount = successCount; + } + + public int getFailCount() { + return failCount; + } + + public void setFailCount(int failCount) { + this.failCount = failCount; + } + + public String getZkCluster() { + return zkCluster; + } + + public void setZkCluster(String zkCluster) { + this.zkCluster = zkCluster; + } + + public Date getRecordDate() { + return recordDate; + } + + public void setRecordDate(Date recordDate) { + this.recordDate = recordDate; + } +} diff --git a/saturn-console-api/src/main/java/com/vip/saturn/job/console/mybatis/repository/DomainCountRepository.java b/saturn-console-api/src/main/java/com/vip/saturn/job/console/mybatis/repository/DomainCountRepository.java new file mode 100644 index 000000000..aede16cf6 --- /dev/null +++ b/saturn-console-api/src/main/java/com/vip/saturn/job/console/mybatis/repository/DomainCountRepository.java @@ -0,0 +1,45 @@ +package com.vip.saturn.job.console.mybatis.repository; + +import com.vip.saturn.job.console.mybatis.entity.DomainCount; +import org.apache.ibatis.annotations.Param; +import org.springframework.stereotype.Repository; + +import java.util.Date; +import java.util.List; + +/** + * @author Ray Leung + */ +@Repository +public interface DomainCountRepository { + + /** + * 创建或更新全域统计信息 + * @param zkCluster + * @param recordDate + * @param successCount + * @param failCount + * @return + */ + int createOrUpdateDomainCount(@Param("zkCluster") String zkCluster, @Param("recordDate") Date recordDate, + @Param("successCount") Integer successCount, @Param("failCount") Integer failCount); + + /** + * 通过zkCluster和recordDate查询全域信息 + * @param zkCluster + * @param recordDate + * @return + */ + DomainCount selectByZkClusterAndRecordDate(@Param("zkCluster") String zkCluster, + @Param("recordDate") Date recordDate); + + /** + * 按时间范围返回全域信息 + * @param zkCluster + * @param startDate + * @param endDate + * @return + */ + List selectByZkClusterAndFromStartDateToEndDate(@Param("zkCluster") String zkCluster, + @Param("startDate") Date startDate, @Param("endDate") Date endDate); +} diff --git a/saturn-console-api/src/main/java/com/vip/saturn/job/console/service/impl/statistics/StatisticsPersistence.java b/saturn-console-api/src/main/java/com/vip/saturn/job/console/service/impl/statistics/StatisticsPersistence.java index 9992f6c91..f106a66f4 100644 --- a/saturn-console-api/src/main/java/com/vip/saturn/job/console/service/impl/statistics/StatisticsPersistence.java +++ b/saturn-console-api/src/main/java/com/vip/saturn/job/console/service/impl/statistics/StatisticsPersistence.java @@ -32,8 +32,12 @@ public void persist(StatisticsModel statisticsModel, ZkCluster zkCluster) { // 全域当天处理总数，失败总数 saveOrUpdateDomainProcessCount( new ZkStatistics(statisticsModel.getZkClusterDailyCountAnalyzer().getTotalCount(), - statisticsModel.getZkClusterDailyCountAnalyzer().getErrorCount()), - zkCluster.getZkAddr()); + statisticsModel.getZkClusterDailyCountAnalyzer().getErrorCount()), zkCluster.getZkAddr()); + + saveOrUpdateDomainProcessCountHistory( + new ZkStatistics(statisticsModel.getZkClusterDailyCountAnalyzer().getTotalCount(), + statisticsModel.getZkClusterDailyCountAnalyzer().getErrorCount()), zkCluster.getZkAddr()); + // 失败率Top10的域列表 saveOrUpdateTop10FailDomain(statisticsModel.getDomainStatisticsAnalyzer().getDomainList(), @@ -93,6 +97,15 @@ public void persist(StatisticsModel statisticsModel, ZkCluster zkCluster) { saveOrUpdateJobCount(jobList.size(), zkCluster.getZkAddr()); } + private void saveOrUpdateDomainProcessCountHistory(ZkStatistics zkStatistics, String zkAddr) { + + int successCount = zkStatistics.getCount(); + int failCount = zkStatistics.getError(); + +// saturnStatisticsService. + + } + private void saveOrUpdateJobCount(int jobCount, String zkAddr) { try { String jobCountString = JSON.toJSONString(jobCount); @@ -174,8 +187,8 @@ private void saveOrUpdateJobRankDistribution(List jobList, String private void saveOrUpdateTop10FailExecutor(List executorList, String zkAddr) { try { executorList = DashboardServiceHelper.sortExecutorByFailureRate(executorList); - List top10FailExecutor = executorList.subList(0, - executorList.size() > 9 ? 10 : executorList.size()); + List top10FailExecutor = executorList + .subList(0, executorList.size() > 9 ? 10 : executorList.size()); String top10FailExecutorJsonString = JSON.toJSONString(top10FailExecutor); SaturnStatistics top10FailExecutorFromDB = saturnStatisticsService .findStatisticsByNameAndZkList(StatisticsTableKeyConstant.TOP_10_FAIL_EXECUTOR, zkAddr); @@ -196,8 +209,8 @@ private void saveOrUpdateTop10FailExecutor(List executorList private void saveOrUpdateTop10FailDomain(List domainList, String zkAddr) { try { domainList = DashboardServiceHelper.sortDomainByAllTimeFailureRate(domainList); - List top10FailDomainList = domainList.subList(0, - domainList.size() > 9 ? 10 : domainList.size()); + List top10FailDomainList = domainList + .subList(0, domainList.size() > 9 ? 10 : domainList.size()); String top10FailDomainJsonString = JSON.toJSONString(top10FailDomainList); SaturnStatistics top10FailDomainFromDB = saturnStatisticsService .findStatisticsByNameAndZkList(StatisticsTableKeyConstant.TOP_10_FAIL_DOMAIN, zkAddr); @@ -217,8 +230,8 @@ private void saveOrUpdateTop10FailDomain(List domainList, Stri private void saveOrUpdateTop10UnstableDomain(List domainList, String zkAddr) { try { domainList = DashboardServiceHelper.sortDomainByShardingCount(domainList); - List top10UnstableDomain = domainList.subList(0, - domainList.size() > 9 ? 10 : domainList.size()); + List top10UnstableDomain = domainList + .subList(0, domainList.size() > 9 ? 10 : domainList.size()); String top10UnstableDomainJsonString = JSON.toJSONString(top10UnstableDomain); SaturnStatistics top10UnstableDomainFromDB = saturnStatisticsService .findStatisticsByNameAndZkList(StatisticsTableKeyConstant.TOP_10_UNSTABLE_DOMAIN, zkAddr); @@ -298,8 +311,8 @@ private void saveOrUpdateTop10LoadJob(List jobList, String zkAddr private void saveOrUpdateTop10LoadExecutor(List executorList, String zkAddr) { try { executorList = DashboardServiceHelper.sortExecutorByLoadLevel(executorList); - List top10LoadExecutor = executorList.subList(0, - executorList.size() > 9 ? 10 : executorList.size()); + List top10LoadExecutor = executorList + .subList(0, executorList.size() > 9 ? 10 : executorList.size()); String top10LoadExecutorJsonString = JSON.toJSONString(top10LoadExecutor); SaturnStatistics top10LoadExecutorFromDB = saturnStatisticsService .findStatisticsByNameAndZkList(StatisticsTableKeyConstant.TOP_10_LOAD_EXECUTOR, zkAddr); diff --git a/saturn-console-api/src/main/resources/mapper/DomainCountMapper.xml b/saturn-console-api/src/main/resources/mapper/DomainCountMapper.xml new file mode 100644 index 000000000..161857a0d --- /dev/null +++ b/saturn-console-api/src/main/resources/mapper/DomainCountMapper.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + id, zk_cluster, success_count, fail_count, record_date + + + + + + + + insert into saturn_domain_process_history( + zk_cluster, success_count, fail_count, record_date + ) values ( + #{zkCluster}, #{successCount}, #{failCount}, #{recordDate} + ) on duplicate key update + success_count = #{successCount}, + fail_count = #{failCount} + + + \ No newline at end of file diff --git a/saturn-console-api/src/test/java/com/vip/saturn/job/console/com/vip/saturn/job/console/mybatis/repository/DomainCountRepositoryTest.java b/saturn-console-api/src/test/java/com/vip/saturn/job/console/com/vip/saturn/job/console/mybatis/repository/DomainCountRepositoryTest.java new file mode 100644 index 000000000..4facb0cdd --- /dev/null +++ b/saturn-console-api/src/test/java/com/vip/saturn/job/console/com/vip/saturn/job/console/mybatis/repository/DomainCountRepositoryTest.java @@ -0,0 +1,84 @@ +package com.vip.saturn.job.console.com.vip.saturn.job.console.mybatis.repository; + +import com.vip.saturn.job.console.mybatis.entity.DomainCount; +import com.vip.saturn.job.console.mybatis.repository.DomainCountRepository; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabase; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.transaction.annotation.Transactional; + +import java.util.Calendar; +import java.util.Date; +import java.util.List; + +/** + * @author Ray Leung + */ +@Ignore +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(locations = "classpath:applicationContext.xml") +public class DomainCountRepositoryTest { + + @Autowired + private DomainCountRepository domainCountRepository; + + private static EmbeddedDatabase embeddedDatabase; + + + @BeforeClass + public static void before() { + EmbeddedDatabaseBuilder embeddedDatabaseBuilder = new EmbeddedDatabaseBuilder(); + embeddedDatabaseBuilder.setType(EmbeddedDatabaseType.H2).addScript("classpath:global.sql") + .addScript("classpath:schema.sql"); + embeddedDatabase = embeddedDatabaseBuilder.build(); + } + + @Test + @Transactional + public void testCreateDomainCount() { + domainCountRepository.createOrUpdateDomainCount("testCluster", new Date(), 0, 0); + DomainCount domainCount = domainCountRepository.selectByZkClusterAndRecordDate("testCluster", new Date()); + Assert.assertEquals(0, domainCount.getSuccessCount()); + Assert.assertEquals(0, domainCount.getFailCount()); + } + + @Test + @Transactional + public void testUpdateDomainCount() { + Date dateNow = new Date(); + domainCountRepository.createOrUpdateDomainCount("testCluster", dateNow, 0, 0); + DomainCount domainCount = domainCountRepository.selectByZkClusterAndRecordDate("testCluster", new Date()); + Assert.assertEquals(0, domainCount.getSuccessCount()); + Assert.assertEquals(0, domainCount.getFailCount()); + domainCountRepository.createOrUpdateDomainCount("testCluster", dateNow, 10, 10); + domainCount = domainCountRepository.selectByZkClusterAndRecordDate("testCluster", new Date()); + Assert.assertEquals(10, domainCount.getSuccessCount()); + Assert.assertEquals(10, domainCount.getFailCount()); + } + + @Test + @Transactional + public void testSelectByZkClusterAndFromStartDateToEndDate() { + + domainCountRepository.createOrUpdateDomainCount("testCluster", new Date(), 0, 0); + Calendar d1 = Calendar.getInstance(); + d1.add(Calendar.DATE, -1); + Calendar d2 = Calendar.getInstance(); + d2.add(Calendar.DATE, -2); + domainCountRepository.createOrUpdateDomainCount("testCluster", d1.getTime(), 0, 0); + domainCountRepository.createOrUpdateDomainCount("testCluster", d2.getTime(), 0, 0); + + List domainCounts = domainCountRepository + .selectByZkClusterAndFromStartDateToEndDate("testCluster", d2.getTime(), d1.getTime()); + Assert.assertEquals(2, domainCounts.size()); + } + +} diff --git a/saturn-console-api/src/test/resources/applicationContext.xml b/saturn-console-api/src/test/resources/applicationContext.xml new file mode 100644 index 000000000..55150c0ef --- /dev/null +++ b/saturn-console-api/src/test/resources/applicationContext.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/saturn-console-api/src/test/resources/global.sql b/saturn-console-api/src/test/resources/global.sql new file mode 100644 index 000000000..04d24add6 --- /dev/null +++ b/saturn-console-api/src/test/resources/global.sql @@ -0,0 +1 @@ +SET MODE MySQL; \ No newline at end of file diff --git a/saturn-console-api/src/test/resources/schema.sql b/saturn-console-api/src/test/resources/schema.sql new file mode 100644 index 000000000..e95c901c0 --- /dev/null +++ b/saturn-console-api/src/test/resources/schema.sql @@ -0,0 +1,11 @@ +create table saturn_domain_process_history +( + id int auto_increment + primary key, + zk_cluster varchar(256) null , + success_count int null , + fail_count int null , + record_date varchar(256) null , + constraint saturn_domain_process_history_zk_cluster_record_date_uindex + unique (zk_cluster, record_date) +); \ No newline at end of file