[improvement](regression-test) support exclude suite/group/directory (a…

…pache#9096)

regression testing framework support skip some suite/group/directory

docs/zh-CN/developer-guide/regression-testing.md

@@ -109,6 +109,13 @@ testSuites = ""
 // 默认会加载的用例目录, 可以通过run-regression-test.sh --run -d来动态指定和覆盖
 testDirectories = ""
 
+// 排除这些组的用例，可通过run-regression-test.sh --run -xg来动态指定和覆盖
+excludeGroups = ""
+// 排除这些suite，可通过run-regression-test.sh --run -xs来动态指定和覆盖
+excludeSuites = ""
+// 排除这些目录，可通过run-regression-test.sh --run -xd来动态指定和覆盖
+excludeDirectories = ""
+
 // 其他自定义配置
 customConf1 = "test_custom_conf_value"
 ```
@@ -517,6 +524,15 @@ thread, lazyCheck, events, connect, selectUnionAll
 # 测试demo目录下的sql_action
 ./run-regression-test.sh --run -d demo -s sql_action
 
+# 测试demo目录下用例，排除sql_action用例
+./run-regression-test.sh --run -d demo -xs sql_action
+
+# 排除demo目录的用例
+./run-regression-test.sh --run -xd demo
+
+# 排除demo group的用例
+./run-regression-test.sh --run -xg demo
+
 # 自定义配置
 ./run-regression-test.sh --run -conf a=b
 

regression-test/conf/regression-conf.groovy

@@ -41,6 +41,13 @@ testSuites = ""
 // empty directories will test all directories
 testDirectories = ""
 
+// this groups will not be executed
+excludeGroups = ""
+// this suites will not be executed
+excludeSuites = ""
+// this directories will not be executed
+excludeDirectories = ""
+
 customConf1 = "test_custom_conf_value"
 
 // for test csv with header

regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy

@@ -47,8 +47,11 @@ class Config {
     public String dataPath
 
     public String testGroups
+    public String excludeGroups
     public String testSuites
+    public String excludeSuites
     public String testDirectories
+    public String excludeDirectories
     public boolean generateOutputFile
     public boolean forceGenerateOutputFile
     public boolean randomOrder
@@ -58,6 +61,11 @@ class Config {
     public Set<String> suiteWildcard = new HashSet<>()
     public Set<String> groups = new HashSet<>()
     public Set<String> directories = new HashSet<>()
+
+    public Set<String> excludeSuiteWildcard = new HashSet<>()
+    public Set<String> excludeGroupSet = new HashSet<>()
+    public Set<String> excludeDirectorySet = new HashSet<>()
+
     public InetSocketAddress feHttpInetSocketAddress
     public Integer parallel
     public Integer suiteParallel
@@ -69,7 +77,8 @@ class Config {
 
     Config(String defaultDb, String jdbcUrl, String jdbcUser, String jdbcPassword,
            String feHttpAddress, String feHttpUser, String feHttpPassword,
-           String suitePath, String dataPath, String testGroups, String testSuites, String testDirectories) {
+           String suitePath, String dataPath, String testGroups, String excludeGroups,
+           String testSuites, String excludeSuites, String testDirectories, String excludeDirectories) {
         this.defaultDb = defaultDb
         this.jdbcUrl = jdbcUrl
         this.jdbcUser = jdbcUser
@@ -80,8 +89,11 @@ class Config {
         this.suitePath = suitePath
         this.dataPath = dataPath
         this.testGroups = testGroups
+        this.excludeGroups = excludeGroups
         this.testSuites = testSuites
+        this.excludeSuites = excludeSuites
         this.testDirectories = testDirectories
+        this.excludeDirectories = excludeDirectories
     }
 
     static Config fromCommandLine(CommandLine cmd) {
@@ -103,8 +115,8 @@ class Config {
         config.dataPath = FileUtils.getCanonicalPath(cmd.getOptionValue(dataOpt, config.dataPath))
         config.suiteWildcard = cmd.getOptionValue(suiteOpt, config.testSuites)
                 .split(",")
-                .collect({g -> g.trim()})
-                .findAll({g -> g != null && g.length() > 0})
+                .collect({s -> s.trim()})
+                .findAll({s -> s != null && s.length() > 0})
                 .toSet()
         config.groups = cmd.getOptionValue(groupsOpt, config.testGroups)
                 .split(",")
@@ -116,6 +128,21 @@ class Config {
                 .collect({d -> d.trim()})
                 .findAll({d -> d != null && d.length() > 0})
                 .toSet()
+        config.excludeSuiteWildcard = cmd.getOptionValue(excludeSuiteOpt, config.excludeSuites)
+                .split(",")
+                .collect({s -> s.trim()})
+                .findAll({s -> s != null && s.length() > 0})
+                .toSet()
+        config.excludeGroupSet = cmd.getOptionValue(excludeGroupsOpt, config.excludeGroups)
+                .split(",")
+                .collect({g -> g.trim()})
+                .findAll({g -> g != null && g.length() > 0})
+                .toSet()
+        config.excludeDirectorySet = cmd.getOptionValue(excludeDirectoriesOpt, config.excludeDirectories)
+                .split(",")
+                .collect({d -> d.trim()})
+                .findAll({d -> d != null && d.length() > 0})
+                .toSet()
 
         config.feHttpAddress = cmd.getOptionValue(feHttpAddressOpt, config.feHttpAddress)
         try {
@@ -162,8 +189,11 @@ class Config {
             configToString(obj.suitePath),
             configToString(obj.dataPath),
             configToString(obj.testGroups),
+            configToString(obj.excludeGroups),
             configToString(obj.testSuites),
-            configToString(obj.testDirectories)
+            configToString(obj.excludeSuites),
+            configToString(obj.testDirectories),
+            configToString(obj.excludeDirectories)
         )
 
         def declareFileNames = config.getClass()
@@ -230,16 +260,31 @@ class Config {
             log.info("Set testGroups to '${config.testGroups}' because not specify.".toString())
         }
 
+        if (config.excludeGroups == null) {
+            config.excludeGroups = ""
+            log.info("Set excludeGroups to empty because not specify.".toString())
+        }
+
         if (config.testDirectories == null) {
             config.testDirectories = ""
             log.info("Set testDirectories to empty because not specify.".toString())
         }
 
+        if (config.excludeDirectories == null) {
+            config.excludeDirectories = ""
+            log.info("Set excludeDirectories to empty because not specify.".toString())
+        }
+
         if (config.testSuites == null) {
             config.testSuites = ""
             log.info("Set testSuites to empty because not specify.".toString())
         }
 
+        if (config.excludeSuites == null) {
+            config.excludeSuites = ""
+            log.info("Set excludeSuites to empty because not specify.".toString())
+        }
+
         if (config.parallel == null) {
             config.parallel = 1
             log.info("Set parallel to 1 because not specify.".toString())
@@ -289,7 +334,7 @@ class Config {
 
     Predicate<String> getDirectoryFilter() {
         return (Predicate<String>) { String directoryName ->
-            if (directories.isEmpty()) {
+            if (directories.isEmpty() && excludeDirectorySet.isEmpty()) {
                 return true
             }
 
@@ -302,7 +347,13 @@ class Config {
                 parentPath = currentPath + File.separator
             }
 
-            return allLevelPaths.any {directories.contains(it) }
+            if (!directories.isEmpty() && !allLevelPaths.any({directories.contains(it) })) {
+                return false
+            }
+            if (!excludeDirectorySet.isEmpty() && allLevelPaths.any({ excludeDirectorySet.contains(it) })) {
+                return false
+            }
+            return true
         }
     }
 

regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy

@@ -38,8 +38,11 @@ class ConfigOptions {
     static Option pathOpt
     static Option dataOpt
     static Option suiteOpt
+    static Option excludeSuiteOpt
     static Option groupsOpt
+    static Option excludeGroupsOpt
     static Option directoriesOpt
+    static Option excludeDirectoriesOpt
     static Option confOpt
     static Option genOutOpt
     static Option forceGenOutOpt
@@ -123,6 +126,15 @@ class ConfigOptions {
                 .longOpt("suite")
                 .desc("the suite name wildcard to be test")
                 .build()
+        excludeSuiteOpt = Option.builder("xs")
+                .argName("excludeSuiteName")
+                .required(false)
+                .hasArg(true)
+                .optionalArg(true)
+                .type(String.class)
+                .longOpt("excludeSuite")
+                .desc("the suite name wildcard will not be tested")
+                .build()
         groupsOpt = Option.builder("g")
                 .argName("groups")
                 .required(false)
@@ -132,6 +144,15 @@ class ConfigOptions {
                 .longOpt("groups")
                 .desc("the suite group to be test")
                 .build()
+        excludeGroupsOpt = Option.builder("xg")
+                .argName("excludeGroupNames")
+                .required(false)
+                .hasArg(true)
+                .optionalArg(true)
+                .type(String.class)
+                .longOpt("excludeGroups")
+                .desc("the suite group will not be tested")
+                .build()
         directoriesOpt = Option.builder("d")
                 .argName("directories")
                 .required(false)
@@ -141,6 +162,15 @@ class ConfigOptions {
                 .longOpt("directories")
                 .desc("only the use cases in these directories can be executed")
                 .build()
+        excludeDirectoriesOpt = Option.builder("xd")
+                .argName("excludeDirectoryNames")
+                .required(false)
+                .hasArg(true)
+                .optionalArg(true)
+                .type(String.class)
+                .longOpt("excludeDirectories")
+                .desc("the use cases in these directories will not be tested")
+                .build()
         feHttpAddressOpt = Option.builder("ha")
                 .argName("address")
                 .required(false)
@@ -239,8 +269,11 @@ class ConfigOptions {
                 .addOption(dataOpt)
                 .addOption(confOpt)
                 .addOption(suiteOpt)
+                .addOption(excludeSuiteOpt)
                 .addOption(groupsOpt)
+                .addOption(excludeGroupsOpt)
                 .addOption(directoriesOpt)
+                .addOption(excludeDirectoriesOpt)
                 .addOption(feHttpAddressOpt)
                 .addOption(feHttpUserOpt)
                 .addOption(feHttpPasswordOpt)

regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy

@@ -173,18 +173,39 @@ class RegressionTest {
         return recorder
     }
 
-    static boolean canRun(Config config, String suiteName, String group) {
+    static boolean filterSuites(Config config, String suiteName) {
+        if (config.suiteWildcard.isEmpty() && config.excludeSuiteWildcard.isEmpty()) {
+            return true
+        }
+        if (!config.suiteWildcard.isEmpty() && !config.suiteWildcard.any {
+                    suiteWildcard -> Wildcard.match(suiteName, suiteWildcard)
+                }) {
+            return false
+        }
+        if (!config.excludeSuiteWildcard.isEmpty() && config.excludeSuiteWildcard.any {
+                    excludeSuiteWildcard -> Wildcard.match(suiteName, excludeSuiteWildcard)
+                }) {
+            return false
+        }
+        return true
+    }
+
+    static boolean filterGroups(Config config, String group) {
+        if (config.groups.isEmpty() && config.excludeGroupSet.isEmpty()) {
+            return true
+        }
         Set<String> suiteGroups = group.split(',').collect { g -> g.trim() }.toSet()
-        if (config.suiteWildcard.size() == 0 ||
-                (suiteName != null && (config.suiteWildcard.any {
-                suiteWildcard -> Wildcard.match(suiteName, suiteWildcard)
-                }))) {
-            if (config.groups == null || config.groups.isEmpty()
-                    || !config.groups.intersect(suiteGroups).isEmpty()) {
-                return true
-            }
+        if (!config.groups.isEmpty() && config.groups.intersect(suiteGroups).isEmpty()) {
+            return false
         }
-        return false
+        if (!config.excludeGroupSet.isEmpty() && !config.excludeGroupSet.intersect(suiteGroups).isEmpty()) {
+            return false
+        }
+        return true
+    }
+
+    static boolean canRun(Config config, String suiteName, String group) {
+        return filterGroups(config, group) && filterSuites(config, suiteName)
     }
 
     static List<EventListener> getEventListeners(Config config, Recorder recorder) {
@@ -226,7 +247,7 @@ class RegressionTest {
             String successList = recorder.successList.collect { info ->
                 "${info.file.absolutePath}: group=${info.group}, name=${info.suiteName}"
             }.join('\n')
-            log.info("SuccessList suites:\n${successList}".toString())
+            log.info("Success suites:\n${successList}".toString())
         }
 
         // print failure list

regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptSource.groovy

@@ -59,26 +59,27 @@ class SqlFileSource implements ScriptSource {
         return SuiteScript.getDefaultGroups(suiteRoot, file)
     }
 
-    @Override
-    SuiteScript toScript(ScriptContext scriptContext, GroovyShell shell) {
-        String suiteName = file.name.substring(0, file.name.lastIndexOf("."))
-        String groupName = getGroup()
-        boolean order = suiteName.endsWith("_order")
-        String tag = suiteName
-        String sql = file.text
-
-        List<String> sqls
+    List<String> getSqls(String sql) {
         try {
-            sqls = SqlUtils.splitAndGetNonEmptySql(sql)
+            return SqlUtils.splitAndGetNonEmptySql(sql)
         } catch (Throwable t) {
-            sqls = [sql]
             log.warn("Try to execute whole file text as one sql, because can not split sql:\n${sql}", t)
+            return [sql]
         }
+    }
+
+    @Override
+    SuiteScript toScript(ScriptContext scriptContext, GroovyShell shell) {
+        String suiteName = file.name.substring(0, file.name.lastIndexOf("."))
+        String groupName = getGroup()
 
         SuiteScript script = new SuiteScript() {
             @Override
             Object run() {
                 suite(suiteName, groupName) {
+                    String tag = suiteName
+                    boolean order = suiteName.endsWith("_order")
+                    List<String> sqls = getSqls(file.text)
                     for (int i = 0; i < sqls.size(); ++i) {
                         String singleSql = sqls.get(i)
                         String tagName = (i == 0) ? tag : "${tag}_${i + 1}"