fix: get count and execute sql add 'with fragment' support

server/src/main/java/edp/core/consts/Consts.java

@@ -152,7 +152,7 @@ public class Consts {
     public static final String REG_SENSITIVE_SQL = "drop\\s|alter\\s|grant\\s|insert\\s|replace\\s|delete\\s|truncate\\s|update\\s|remove\\s";
     public static final Pattern PATTERN_SENSITIVE_SQL = Pattern.compile(REG_SENSITIVE_SQL);
 
-    private static final String REG_WITH_SQL_FRAGMENT = "((?i)WITH[\\s\\S]+(?i)AS\\s*\\([\\s\\S]+\\))\\s*(?i)SELECT";
+    private static final String REG_WITH_SQL_FRAGMENT = "((?i)WITH[\\s\\S]+(?i)AS?\\s*\\([\\s\\S]+\\))\\s*(?i)SELECT";
     public static final Pattern WITH_SQL_FRAGMENT = Pattern.compile(REG_WITH_SQL_FRAGMENT);
 
     /**

server/src/main/java/edp/core/utils/SqlUtils.java

@@ -29,6 +29,7 @@
 import edp.core.model.*;
 import edp.davinci.core.enums.LogNameEnum;
 import edp.davinci.core.enums.SqlColumnEnum;
+import edp.davinci.core.utils.SqlParseUtils;
 import lombok.extern.slf4j.Slf4j;
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.expression.Alias;
@@ -296,14 +297,15 @@ private Map<String, Object> getResultObjectMap(Set<String> excludeColumns, Resul
     }
 
     public static String getCountSql(String sql) {
+        String countSql = String.format(Consts.QUERY_COUNT_SQL, sql);
         try {
             Select select = (Select) CCJSqlParserUtil.parse(sql);
             PlainSelect plainSelect = (PlainSelect) select.getSelectBody();
             plainSelect.setOrderByElements(null);
-            return String.format(QUERY_COUNT_SQL, select.toString());
+            countSql = String.format(QUERY_COUNT_SQL, select.toString());
         } catch (JSQLParserException e) {
         }
-        return String.format(Consts.QUERY_COUNT_SQL, sql);
+        return SqlParseUtils.rebuildSqlWithFragment(countSql);
     }
 
 
@@ -328,6 +330,7 @@ public static Set<String> getQueryFromsAndJoins(String sql) {
                 columnPrefixExtractor(columnPrefixs, plainSelect);
             }
         } catch (JSQLParserException e) {
+            log.warn("Get table name or alias Error: {}", e.getCause().getMessage());
         }
         return columnPrefixs;
     }

server/src/main/java/edp/davinci/core/utils/SqlParseUtils.java

@@ -36,6 +36,7 @@
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
 import net.sf.jsqlparser.statement.select.PlainSelect;
 import net.sf.jsqlparser.statement.select.Select;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import org.stringtemplate.v4.ST;
 
@@ -61,6 +62,7 @@ public class SqlParseUtils {
     private static final String QUERY_WHERE_TRUE = "1=1";
     private static final String QUERY_WHERE_FALSE = "1=0";
 
+    @Autowired
     private DacChannelUtil dacChannelUtil;
 
     /**
@@ -93,7 +95,7 @@ public SqlEntity parseSql(String sqlStr, List<SqlVariable> variables, String sql
 
         // 解析参数
         if (!CollectionUtils.isEmpty(variables)) {
-            ExecutorService executorService = Executors.newFixedThreadPool(8);
+            ExecutorService executorService = Executors.newFixedThreadPool(4);
             try {
                 CountDownLatch countDownLatch = new CountDownLatch(variables.size());
                 List<Future> futures = new ArrayList<>(variables.size());
@@ -140,7 +142,6 @@ public SqlEntity parseSql(String sqlStr, List<SqlVariable> variables, String sql
 
 
     public List<String> getAuthVarValue(SqlVariable variable, String email) {
-
         SqlVariableChannel channel = variable.getChannel();
         if (null == channel) {
             return SqlVariableValueTypeEnum.getValues(variable.getValueType(), variable.getDefaultValues(),
@@ -241,7 +242,6 @@ public List<String> getSqls(String sql, boolean isQuery) {
         if (split.length > 0) {
             list = new ArrayList<>();
             for (String sqlStr : split) {
-                sqlStr = rebuildSqlWithFragment(sqlStr.trim());
                 boolean select = sqlStr.toLowerCase().startsWith(SELECT) || sqlStr.toLowerCase().startsWith(WITH);
                 if (isQuery) {
                     if (select) {
@@ -257,18 +257,20 @@ public List<String> getSqls(String sql, boolean isQuery) {
         return list;
     }
 
-    private static String rebuildSqlWithFragment(String sql) {
+    public static String rebuildSqlWithFragment(String sql) {
         if (!sql.toLowerCase().startsWith(WITH)) {
             Matcher matcher = WITH_SQL_FRAGMENT.matcher(sql);
             if (matcher.find()) {
                 String withFragment = matcher.group();
-                if (withFragment.length() > 6) {
-                    int lastSelectIndex = withFragment.length() - 6;
-                    sql = sql.replace(withFragment, withFragment.substring(lastSelectIndex));
-                    withFragment = withFragment.substring(0, lastSelectIndex);
+                if (!StringUtils.isEmpty(withFragment)) {
+                    if (withFragment.length() > 6) {
+                        int lastSelectIndex = withFragment.length() - 6;
+                        sql = sql.replace(withFragment, withFragment.substring(lastSelectIndex));
+                        withFragment = withFragment.substring(0, lastSelectIndex);
+                    }
+                    sql = withFragment + SPACE + sql;
+                    sql = sql.replaceAll(SPACE + "{2,}", SPACE);
                 }
-                sql = withFragment + SPACE + sql;
-                sql = sql.replaceAll(SPACE + "{2,}", SPACE);
             }
         }
         return sql;

server/src/main/java/edp/davinci/service/excel/SheetWorker.java

@@ -25,6 +25,7 @@
 import edp.core.utils.CollectionUtils;
 import edp.core.utils.SqlUtils;
 import edp.davinci.core.enums.ActionEnum;
+import edp.davinci.core.utils.SqlParseUtils;
 import edp.davinci.dto.cronJobDto.MsgMailExcel;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.jdbc.core.JdbcTemplate;
@@ -67,6 +68,7 @@ public T call() throws Exception {
             template.setFetchSize(500);
 
             String sql = context.getQuerySql().get(context.getQuerySql().size() - 1);
+            sql = SqlParseUtils.rebuildSqlWithFragment(sql);
             Set<String> queryFromsAndJoins = SqlUtils.getQueryFromsAndJoins(sql);
             if (context.getCustomLogger() != null) {
                 context.getCustomLogger().info("Task ({}) -- {} start query", context.getTaskKey(), context.getName());
@@ -138,6 +140,7 @@ private void buildQueryColumn(JdbcTemplate template) {
         template.setMaxRows(1);
         String sql = context.getQuerySql().get(context.getQuerySql().size() - 1);
         sql = String.format(QUERY_META_SQL, sql);
+        sql = SqlParseUtils.rebuildSqlWithFragment(sql);
         Set<String> queryFromsAndJoins = SqlUtils.getQueryFromsAndJoins(sql);
         template.query(sql, rs -> {
             ResultSetMetaData metaData = rs.getMetaData();

server/src/main/java/edp/davinci/service/impl/ViewServiceImpl.java

@@ -437,6 +437,7 @@ public PaginateWithQueryColumns executeSql(ViewExecuteSql executeSql, User user)
                     }
                     if (!CollectionUtils.isEmpty(querySqlList)) {
                         for (String sql : querySqlList) {
+                            sql = SqlParseUtils.rebuildSqlWithFragment(sql);
                             paginateWithQueryColumns = sqlUtils.syncQuery4Paginate(sql, null, null, null, executeSql.getLimit(), null);
                         }
                     }
@@ -618,7 +619,7 @@ public PaginateWithQueryColumns getResultDataList(boolean isMaintainer, ViewWith
 
                     for (String sql : querySqlList) {
                         paginate = sqlUtils.syncQuery4Paginate(
-                                sql,
+                                SqlParseUtils.rebuildSqlWithFragment(sql),
                                 executeParam.getPageNo(),
                                 executeParam.getPageSize(),
                                 executeParam.getTotalCount(),
@@ -717,7 +718,7 @@ public List<Map<String, Object>> getDistinctValueData(boolean isMaintainer, View
                 }
                 List<Map<String, Object>> list = null;
                 for (String sql : querySqlList) {
-                    list = sqlUtils.query4List(sql, -1);
+                    list = sqlUtils.query4List(SqlParseUtils.rebuildSqlWithFragment(sql), -1);
                 }
 
                 if (null != param.getCache() && param.getCache() && param.getExpired().longValue() > 0L) {