Skip to content

Commit

Permalink
edp963#1898 1. fix some issues
Browse files Browse the repository at this point in the history
  • Loading branch information
@xxxllluuu
xxxllluuu committed Aug 21, 2020
1 parent eb74031 commit cb8700b
Show file tree
Hide file tree
Showing 15 changed files with 374 additions and 212 deletions.
36 changes: 28 additions & 8 deletions config/application.yml.example
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,22 @@ jwtToken:

##your datasouce config
source:
initial-size: 2
initial-size: 1
min-idle: 1
max-wait: 6000
max-wait: 30000
max-active: 10
break-after-acquire-failure: true
connection-error-retry-attempts: 0
query-timeout: 600000
validationQueryTimeout: 30000
connection-error-retry-attempts: 1
time-between-eviction-runs-millis: 2000
min-evictable-idle-time-millis: 600000
max-evictable-idle-time-millis: 900000
test-while-idle: true
test-on-borrow: false
test-on-return: false
validation-query: select 1
validation-query-timeout: 10
keep-alive: false
filters: stat

enable-query-log: false
result-limit: 1000000
Expand All @@ -69,10 +77,22 @@ spring:
username: root
password: root
driver-class-name: com.mysql.jdbc.Driver
initial-size: 2
initial-size: 1
min-idle: 1
max-wait: 60000
max-wait: 30000
max-active: 10
break-after-acquire-failure: true
connection-error-retry-attempts: 1
time-between-eviction-runs-millis: 2000
min-evictable-idle-time-millis: 600000
max-evictable-idle-time-millis: 900000
test-while-idle: true
test-on-borrow: false
test-on-return: false
validation-query: select 1
validation-query-timeout: 10
keep-alive: false
filters: stat

## redis config
## please choose either of the two ways
Expand Down Expand Up @@ -156,7 +176,7 @@ screenshot:
timeout_second: 600
phantomjs_path: $your_phantomjs_path$
chromedriver_path: $your_chromedriver_path$
remote_webdriver_url:$your_remote_webdriver_url$
remote_webdriver_url: $your_remote_webdriver_url$

data-auth-center:
channels:
Expand Down
131 changes: 84 additions & 47 deletions server/src/main/java/edp/core/common/jdbc/JdbcDataSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,25 +19,14 @@

package edp.core.common.jdbc;

import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_CONNECTIONPROPERTIES;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_INITIALSIZE;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_MAXACTIVE;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_MAXWAIT;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_MINEVICTABLEIDLETIMEMILLIS;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_MINIDLE;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_PASSWORD;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_TESTONBORROW;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_TESTONRETURN;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_TESTWHILEIDLE;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_TIMEBETWEENEVICTIONRUNSMILLIS;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_URL;
import static com.alibaba.druid.pool.DruidDataSourceFactory.PROP_USERNAME;
import static com.alibaba.druid.pool.DruidDataSourceFactory.*;
import static edp.core.consts.Consts.JDBC_DATASOURCE_DEFAULT_VERSION;

import java.io.File;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
Expand Down Expand Up @@ -79,17 +68,27 @@ public DruidDataSource getDataSource(JdbcSourceInfo jdbcSourceInfo) throws Sourc
if (druidDataSource != null && !druidDataSource.isClosed()) {
return druidDataSource;
}

Lock lock = getDataSourceLock(key);

try {
if (!lock.tryLock(5L, TimeUnit.SECONDS)) {
throw new SourceException("Unable to get driver instance for jdbcUrl: " + jdbcUrl);
}
}
catch (InterruptedException e) {
throw new SourceException("Unable to get driver instance for jdbcUrl: " + jdbcUrl);
}

try {
if (!lock.tryLock(60L, TimeUnit.SECONDS)) {
druidDataSource = dataSourceMap.get(key);
if (druidDataSource != null && !druidDataSource.isClosed()) {
return druidDataSource;
}
throw new SourceException("Unable to get datasource for jdbcUrl: " + jdbcUrl);
}
}
catch (InterruptedException e) {
throw new SourceException("Unable to get datasource for jdbcUrl: " + jdbcUrl);
}

druidDataSource = dataSourceMap.get(key);
if (druidDataSource != null && !druidDataSource.isClosed()) {
lock.unlock();
return druidDataSource;
}

druidDataSource = dataSourceMap.get(key);
if (druidDataSource != null && !druidDataSource.isClosed()) {
Expand All @@ -112,9 +111,15 @@ public DruidDataSource getDataSource(JdbcSourceInfo jdbcSourceInfo) throws Sourc
properties.setProperty(PROP_MAXWAIT, String.valueOf(maxWait));
properties.setProperty(PROP_TIMEBETWEENEVICTIONRUNSMILLIS, String.valueOf(timeBetweenEvictionRunsMillis));
properties.setProperty(PROP_MINEVICTABLEIDLETIMEMILLIS, String.valueOf(minEvictableIdleTimeMillis));
properties.setProperty(PROP_TESTWHILEIDLE, String.valueOf(false));
properties.setProperty(PROP_MAXOPENPREPAREDSTATEMENTS, String.valueOf(maxEvictableIdleTimeMillis));
properties.setProperty(PROP_TESTWHILEIDLE, String.valueOf(testWhileIdle));
properties.setProperty(PROP_TESTONBORROW, String.valueOf(testOnBorrow));
properties.setProperty(PROP_TESTONRETURN, String.valueOf(testOnReturn));
properties.setProperty(PROP_VALIDATIONQUERY, validationQuery);
properties.setProperty(PROP_VALIDATIONQUERY_TIMEOUT, String.valueOf(validationQueryTimeout));
properties.setProperty(PROP_REMOVEABANDONED, "true");
properties.setProperty(PROP_REMOVEABANDONEDTIMEOUT, "3900");
properties.setProperty(PROP_LOGABANDONED, "true");
properties.put(PROP_CONNECTIONPROPERTIES, "client.transport.ignore_cluster_name=true");

if (!CollectionUtils.isEmpty(jdbcSourceInfo.getProperties())) {
Expand All @@ -138,56 +143,69 @@ public DruidDataSource getDataSource(JdbcSourceInfo jdbcSourceInfo) throws Sourc
@Autowired
private ESDataSource esDataSource;

@Value("${spring.datasource.type}")
protected String type;

@Value("${source.max-active:10}")
@Value("${source.max-active:8}")
@Getter
protected int maxActive;

@Value("${source.initial-size:1}")
@Value("${source.initial-size:0}")
@Getter
protected int initialSize;

@Value("${source.min-idle:3}")
@Value("${source.min-idle:1}")
@Getter
protected int minIdle;

@Value("${source.max-wait:30000}")
@Value("${source.max-wait:60000}")
@Getter
protected long maxWait;

@Value("${spring.datasource.time-between-eviction-runs-millis}")
@Value("${source.time-between-eviction-runs-millis}")
@Getter
protected long timeBetweenEvictionRunsMillis;

@Value("${spring.datasource.min-evictable-idle-time-millis}")
@Value("${source.min-evictable-idle-time-millis}")
@Getter
protected long minEvictableIdleTimeMillis;

@Value("${spring.datasource.test-while-idle}")
@Value("${source.max-evictable-idle-time-millis}")
@Getter
protected long maxEvictableIdleTimeMillis;

@Value("${source.test-while-idle}")
@Getter
protected boolean testWhileIdle;

@Value("${spring.datasource.test-on-borrow}")
@Value("${source.test-on-borrow}")
@Getter
protected boolean testOnBorrow;

@Value("${spring.datasource.test-on-return}")
@Value("${source.test-on-return}")
@Getter
protected boolean testOnReturn;

@Value("${source.break-after-acquire-failure:true}")
@Getter
protected boolean breakAfterAcquireFailure;

@Value("${source.connection-error-retry-attempts:0}")
@Value("${source.connection-error-retry-attempts:1}")
@Getter
protected int connectionErrorRetryAttempts;

@Value("${source.query-timeout:600000}")
@Value("${source.keep-alive:false}")
@Getter
protected int queryTimeout;
protected boolean keepAlive;

@Value("${source.validation-query-timeout:5}")
@Getter
protected int validationQueryTimeout;

@Value("${source.validation-query}")
@Getter
protected String validationQuery;

@Value("${source.filters}")
@Getter
protected String filters;

private static volatile Map<String, DruidDataSource> dataSourceMap = new ConcurrentHashMap<>();
private static volatile Map<String, Lock> dataSourceLockMap = new ConcurrentHashMap<>();
Expand Down Expand Up @@ -223,7 +241,7 @@ public void removeDatasource(JdbcSourceInfo jdbcSourceInfo) {
String key = getDataSourceKey(jdbcSourceInfo);

Lock lock = getDataSourceLock(key);

if (!lock.tryLock()) {
return;
}
Expand Down Expand Up @@ -262,16 +280,21 @@ public DruidDataSource getDataSource(JdbcSourceInfo jdbcSourceInfo) throws Sourc
Lock lock = getDataSourceLock(key);

try {
if (!lock.tryLock(5L, TimeUnit.SECONDS)) {
throw new SourceException("Unable to get driver instance for jdbcUrl: " + jdbcUrl);
if (!lock.tryLock(30L, TimeUnit.SECONDS)) {
druidDataSource = dataSourceMap.get(key);
if (druidDataSource != null && !druidDataSource.isClosed()) {
return druidDataSource;
}
throw new SourceException("Unable to get datasource for jdbcUrl: " + jdbcUrl);
}
}
catch (InterruptedException e) {
throw new SourceException("Unable to get driver instance for jdbcUrl: " + jdbcUrl);
throw new SourceException("Unable to get datasource for jdbcUrl: " + jdbcUrl);
}

druidDataSource = dataSourceMap.get(key);
if (druidDataSource != null && !druidDataSource.isClosed()) {
lock.unlock();
return druidDataSource;
}

Expand Down Expand Up @@ -310,24 +333,38 @@ public DruidDataSource getDataSource(JdbcSourceInfo jdbcSourceInfo) throws Sourc
druidDataSource.setMaxWait(maxWait);
druidDataSource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis);
druidDataSource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis);
druidDataSource.setTestWhileIdle(false);
druidDataSource.setMaxEvictableIdleTimeMillis(maxEvictableIdleTimeMillis);
druidDataSource.setTestWhileIdle(testWhileIdle);
druidDataSource.setTestOnBorrow(testOnBorrow);
druidDataSource.setTestOnReturn(testOnReturn);
druidDataSource.setConnectionErrorRetryAttempts(connectionErrorRetryAttempts);
druidDataSource.setBreakAfterAcquireFailure(breakAfterAcquireFailure);

String driverName=druidDataSource.getDriverClassName();
if(driverName.indexOf("sqlserver")!=-1) {
druidDataSource.setKeepAlive(keepAlive);
druidDataSource.setValidationQueryTimeout(validationQueryTimeout);
druidDataSource.setValidationQuery(validationQuery);
druidDataSource.setRemoveAbandoned(true);
druidDataSource.setRemoveAbandonedTimeout(3600 + 5 * 60);
druidDataSource.setLogAbandoned(true);

// default validation query
String driverName = druidDataSource.getDriverClassName();
if (driverName.indexOf("sqlserver") != -1 || driverName.indexOf("mysql") != -1
|| driverName.indexOf("h2") != -1 || driverName.indexOf("moonbox") != -1) {
druidDataSource.setValidationQuery("select 1");
}

if (driverName.indexOf("oracle") != -1) {
druidDataSource.setValidationQuery("select 1 from dual");
}

if (!CollectionUtils.isEmpty(jdbcSourceInfo.getProperties())) {
Properties properties = new Properties();
jdbcSourceInfo.getProperties().forEach(dict -> properties.setProperty(dict.getKey(), dict.getValue()));
druidDataSource.setConnectProperties(properties);
}

try {
druidDataSource.setFilters(filters);
druidDataSource.init();
} catch (Exception e) {
log.error("Exception during pool initialization", e);
Expand Down
8 changes: 8 additions & 0 deletions server/src/main/java/edp/core/config/DruidConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ public class DruidConfig {
@Value("${spring.datasource.min-evictable-idle-time-millis}")
private int minEvictableIdleTimeMillis;

@Value("${spring.datasource.max-evictable-idle-time-millis}")
private int maxEvictableIdleTimeMillis;

@Value("${spring.datasource.test-while-idle}")
private boolean testWhileIdle;

Expand All @@ -93,6 +96,9 @@ public class DruidConfig {
@Value("${spring.datasource.validation-query}")
private String validationQuery;

@Value("${spring.datasource.validation-query-timeout}")
private int validationQueryTime;

/**
* druid监控
*
Expand Down Expand Up @@ -139,12 +145,14 @@ public DruidDataSource druidDataSource() {
druidDataSource.setMaxWait(maxWait);
druidDataSource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis);
druidDataSource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis);
druidDataSource.setMaxEvictableIdleTimeMillis(maxEvictableIdleTimeMillis);
druidDataSource.setTestWhileIdle(testWhileIdle);
druidDataSource.setTestOnBorrow(testOnBorrow);
druidDataSource.setTestOnReturn(testOnReturn);
druidDataSource.setBreakAfterAcquireFailure(breakAfterAcquireFailure);
druidDataSource.setConnectionErrorRetryAttempts(connectionErrorRetryAttempts);
druidDataSource.setValidationQuery(validationQuery);
druidDataSource.setValidationQueryTimeout(validationQueryTime);

try {
druidDataSource.setFilters(filters);
Expand Down
26 changes: 26 additions & 0 deletions server/src/main/java/edp/core/utils/DateUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,4 +314,30 @@ public static java.sql.Date toSqlDate(Long timeLongInMicros) {
return toSqlDate(new Date(timeLongInMicros));
}

public static Date dateFormat(String date, String dateFormat) {
if (date == null) {
return null;
}
SimpleDateFormat format = new SimpleDateFormat(dateFormat);
try {
return format.parse(date);
} catch (Exception ex) {

}
return null;
}

public static String dateFormat(Date date, String dateFormat) {
if (date == null) {
return "";
}
SimpleDateFormat format = new SimpleDateFormat(dateFormat);
try {
return format.format(date);
} catch (Exception ex) {

}
return "";
}

}
10 changes: 10 additions & 0 deletions server/src/main/java/edp/core/utils/FileUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.FileCopyUtils;
import org.springframework.web.multipart.MultipartFile;

import javax.imageio.ImageIO;
Expand Down Expand Up @@ -367,4 +368,13 @@ public static void closeCloseable(Closeable c) {
}
}
}

public static int copy(File in, File out) {
try {
return FileCopyUtils.copy(in, out);
} catch (IOException e) {
e.printStackTrace();
}
return -1;
}
}
Loading

0 comments on commit cb8700b

Please sign in to comment.