[test][enhance][refactor] support suite block to specify multiple gro…

…up, suppo… (apache#8792)

support suite block to specify multiple groups.
TestAction support compare result to iterator, local file and http stream.
support print teamcity service message.
abandon the logical: generate groovy file for sql file
support 3 levels parrallel: script file, suite block, thread action
support specify JAVA_OPTS for boot shell
avoid jvm metaspace oom
use -d to run the suite in some directories, instead of -g. and -g is used to specify groups

regression-test/conf/logback.xml

@@ -21,9 +21,18 @@ under the License.
     <timestamp key="LOG_TIME" datePattern="yyyyMMdd.HHmmss"/>
 
     <appender name="stdoutAppender" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
-            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %level [%thread] \(%F:%L\) - %msg%n</pattern>
-        </encoder>
+        <if condition='property("stdoutAppenderType").equalsIgnoreCase("teamcity")'>
+            <then>
+                <encoder class="org.apache.doris.regression.logger.TeamcityServiceMessageEncoder">
+                    <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %level [%thread] \(%F:%L\) - %msg%n</pattern>
+                </encoder>
+            </then>
+            <else>
+                <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+                    <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %level [%thread] \(%F:%L\) - %msg%n</pattern>
+                </encoder>
+            </else>
+        </if>
     </appender>
 
     <appender name="rollingFileAppender" class="ch.qos.logback.core.rolling.RollingFileAppender">

regression-test/conf/regression-conf.groovy

@@ -38,5 +38,7 @@ dataPath = "${DORIS_HOME}/regression-test/data"
 testGroups = ""
 // empty suite will test all suite
 testSuites = ""
+// empty directories will test all directories
+testDirectories = ""
 
 customConf1 = "test_custom_conf_value"

regression-test/data/demo/test_action.csv

@@ -0,0 +1,4 @@
+1
+2
+3
+4

regression-test/data/demo/test_sql_file.out

@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !test_sql_file --
+200
+100
+300
+

regression-test/data/demo/test_sql_file_order.out

@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !test_sql_file_order --
+100
+200
+300
+

regression-test/data/empty_table/sql/avg_decimal.out

@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !avg_decimal --
+\N
+

regression-test/data/join/sql/agg_output_as_right_tale_left_outer_order.out

@@ -0,0 +1,5 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !agg_output_as_right_tale_left_outer_order --
+1	1
+2	2
+

regression-test/framework/pom.xml

@@ -130,6 +130,24 @@ under the License.
                         </configuration>
                     </execution>
                 </executions>
+                <configuration>
+                    <shadedArtifactAttached>false</shadedArtifactAttached>
+                    <artifactSet>
+                        <includes>
+                            <include>*:*</include>
+                        </includes>
+                    </artifactSet>
+                    <filters>
+                        <filter>
+                            <artifact>*:*</artifact>
+                            <excludes>
+                                <exclude>META-INF/*.SF</exclude>
+                                <exclude>META-INF/*.DSA</exclude>
+                                <exclude>META-INF/*.RSA</exclude>
+                            </excludes>
+                        </filter>
+                    </filters>
+                </configuration>
             </plugin>
         </plugins>
         <pluginManagement>
@@ -215,5 +233,10 @@ under the License.
             <artifactId>guava</artifactId>
             <version>31.0.1-jre</version>
         </dependency>
+        <dependency>
+            <groupId>org.codehaus.janino</groupId>
+            <artifactId>janino</artifactId>
+            <version>3.1.6</version>
+        </dependency>
     </dependencies>
 </project>

regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy

@@ -27,6 +27,7 @@ import org.apache.doris.regression.util.JdbcUtils
 
 import java.sql.Connection
 import java.sql.DriverManager
+import java.util.function.Predicate
 
 import static org.apache.doris.regression.ConfigOptions.*
 
@@ -47,6 +48,7 @@ class Config {
 
     public String testGroups
     public String testSuites
+    public String testDirectories
     public boolean generateOutputFile
     public boolean forceGenerateOutputFile
     public boolean randomOrder
@@ -55,8 +57,10 @@ class Config {
 
     public Set<String> suiteWildcard = new HashSet<>()
     public Set<String> groups = new HashSet<>()
+    public Set<String> directories = new HashSet<>()
     public InetSocketAddress feHttpInetSocketAddress
     public Integer parallel
+    public Integer suiteParallel
     public Integer actionParallel
     public Integer times
     public boolean withOutLoadData
@@ -65,7 +69,7 @@ class Config {
 
     Config(String defaultDb, String jdbcUrl, String jdbcUser, String jdbcPassword,
            String feHttpAddress, String feHttpUser, String feHttpPassword,
-           String suitePath, String dataPath, String testGroups, String testSuites) {
+           String suitePath, String dataPath, String testGroups, String testSuites, String testDirectories) {
         this.defaultDb = defaultDb
         this.jdbcUrl = jdbcUrl
         this.jdbcUser = jdbcUser
@@ -77,6 +81,7 @@ class Config {
         this.dataPath = dataPath
         this.testGroups = testGroups
         this.testSuites = testSuites
+        this.testDirectories = testDirectories
     }
 
     static Config fromCommandLine(CommandLine cmd) {
@@ -106,6 +111,11 @@ class Config {
                 .collect({g -> g.trim()})
                 .findAll({g -> g != null && g.length() > 0})
                 .toSet()
+        config.directories = cmd.getOptionValue(directoriesOpt, config.testDirectories)
+                .split(",")
+                .collect({d -> d.trim()})
+                .findAll({d -> d != null && d.length() > 0})
+                .toSet()
 
         config.feHttpAddress = cmd.getOptionValue(feHttpAddressOpt, config.feHttpAddress)
         try {
@@ -116,7 +126,7 @@ class Config {
             throw new IllegalStateException("Can not parse stream load address: ${config.feHttpAddress}", t)
         }
 
-        config.defaultDb = cmd.getOptionValue(jdbcOpt, config.defaultDb)
+        config.defaultDb = cmd.getOptionValue(defaultDbOpt, config.defaultDb)
         config.jdbcUrl = cmd.getOptionValue(jdbcOpt, config.jdbcUrl)
         config.jdbcUser = cmd.getOptionValue(userOpt, config.jdbcUser)
         config.jdbcPassword = cmd.getOptionValue(passwordOpt, config.jdbcPassword)
@@ -125,6 +135,7 @@ class Config {
         config.generateOutputFile = cmd.hasOption(genOutOpt)
         config.forceGenerateOutputFile = cmd.hasOption(forceGenOutOpt)
         config.parallel = Integer.parseInt(cmd.getOptionValue(parallelOpt, "1"))
+        config.suiteParallel = Integer.parseInt(cmd.getOptionValue(suiteParallelOpt, "1"))
         config.actionParallel = Integer.parseInt(cmd.getOptionValue(actionParallelOpt, "10"))
         config.times = Integer.parseInt(cmd.getOptionValue(timesOpt, "1"))
         config.randomOrder = cmd.hasOption(randomOrderOpt)
@@ -151,7 +162,8 @@ class Config {
             configToString(obj.suitePath),
             configToString(obj.dataPath),
             configToString(obj.testGroups),
-            configToString(obj.testSuites)
+            configToString(obj.testSuites),
+            configToString(obj.testDirectories)
         )
 
         def declareFileNames = config.getClass()
@@ -218,6 +230,11 @@ class Config {
             log.info("Set testGroups to '${config.testGroups}' because not specify.".toString())
         }
 
+        if (config.testDirectories == null) {
+            config.testDirectories = ""
+            log.info("Set testDirectories to empty because not specify.".toString())
+        }
+
         if (config.testSuites == null) {
             config.testSuites = ""
             log.info("Set testSuites to empty because not specify.".toString())
@@ -228,6 +245,11 @@ class Config {
             log.info("Set parallel to 1 because not specify.".toString())
         }
 
+        if (config.suiteParallel == null) {
+            config.suiteParallel = 1
+            log.info("Set suiteParallel to 1 because not specify.".toString())
+        }
+
         if (config.actionParallel == null) {
             config.actionParallel = 10
             log.info("Set actionParallel to 10 because not specify.".toString())
@@ -265,6 +287,25 @@ class Config {
         return DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword)
     }
 
+    Predicate<String> getDirectoryFilter() {
+        return (Predicate<String>) { String directoryName ->
+            if (directories.isEmpty()) {
+                return true
+            }
+
+            String relativePath = new File(suitePath).relativePath(new File(directoryName))
+            List<String> allLevelPaths = new ArrayList<>()
+            String parentPath = ""
+            for (String pathName : relativePath.split(File.separator)) {
+                String currentPath = parentPath + pathName
+                allLevelPaths.add(currentPath)
+                parentPath = currentPath + File.separator
+            }
+
+            return allLevelPaths.any {directories.contains(it) }
+        }
+    }
+
     private void buildUrlWithDefaultDb() {
         String urlWithDb = jdbcUrl
         String urlWithoutSchema = jdbcUrl.substring(jdbcUrl.indexOf("://") + 3)

regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy

@@ -39,10 +39,12 @@ class ConfigOptions {
     static Option dataOpt
     static Option suiteOpt
     static Option groupsOpt
+    static Option directoriesOpt
     static Option confOpt
     static Option genOutOpt
     static Option forceGenOutOpt
     static Option parallelOpt
+    static Option suiteParallelOpt
     static Option actionParallelOpt
     static Option randomOrderOpt
     static Option timesOpt
@@ -130,6 +132,15 @@ class ConfigOptions {
                 .longOpt("groups")
                 .desc("the suite group to be test")
                 .build()
+        directoriesOpt = Option.builder("d")
+                .argName("directories")
+                .required(false)
+                .hasArg(true)
+                .optionalArg(true)
+                .type(String.class)
+                .longOpt("directories")
+                .desc("only the use cases in these directories can be executed")
+                .build()
         feHttpAddressOpt = Option.builder("ha")
                 .argName("address")
                 .required(false)
@@ -179,7 +190,15 @@ class ConfigOptions {
                 .optionalArg(true)
                 .type(String.class)
                 .longOpt("parallel")
-                .desc("the num of threads running test")
+                .desc("the num of threads running scripts")
+                .build()
+        suiteParallelOpt = Option.builder("suiteParallel")
+                .argName("parallel")
+                .required(false)
+                .hasArg(true)
+                .type(String.class)
+                .longOpt("suiteParallel")
+                .desc("the num of threads running for suites")
                 .build()
         actionParallelOpt = Option.builder("actionParallel")
                 .argName("parallel")
@@ -221,15 +240,18 @@ class ConfigOptions {
                 .addOption(confOpt)
                 .addOption(suiteOpt)
                 .addOption(groupsOpt)
+                .addOption(directoriesOpt)
                 .addOption(feHttpAddressOpt)
                 .addOption(feHttpUserOpt)
                 .addOption(feHttpPasswordOpt)
                 .addOption(genOutOpt)
                 .addOption(confFileOpt)
                 .addOption(forceGenOutOpt)
                 .addOption(parallelOpt)
+                .addOption(suiteParallelOpt)
                 .addOption(actionParallelOpt)
                 .addOption(randomOrderOpt)
+                .addOption(timesOpt)
                 .addOption(withOutLoadDataOpt)
 
         CommandLine cmd = new DefaultParser().parse(options, args, true)