diff --git a/.gitignore b/.gitignore index cc1b2859a4adfb..6571a6b9481a72 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,9 @@ samples/**/.classpath fe/fe-core/src/main/resources/static/ nohup.out +regression-test/framework/target +regression-test/framework/.flattened-pom.xml +regression-test/framework/dependency-reduced-pom.xml # ignore eclipse project file & idea project file .cproject diff --git a/.licenserc.yaml b/.licenserc.yaml index 35293656b48753..f5bc904fc12ddb 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -50,5 +50,7 @@ header: - 'be/src/util/sse2neon.h' - 'be/src/util/utf8_check.cpp' - 'build-support/run_clang_format.py' + - 'regression-test/suites' + - 'regression-test/data' comment: on-failure diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js index 9ca42dba913e8e..f622fb122ea68c 100644 --- a/docs/.vuepress/sidebar/zh-CN.js +++ b/docs/.vuepress/sidebar/zh-CN.js @@ -713,6 +713,7 @@ module.exports = [ "How-to-Share-blogs", "minidump", "bitmap-hll-file-format", + "regression-testing", ], }, { diff --git a/docs/zh-CN/developer-guide/regression-testing.md b/docs/zh-CN/developer-guide/regression-testing.md new file mode 100644 index 00000000000000..eff0cda429c0fc --- /dev/null +++ b/docs/zh-CN/developer-guide/regression-testing.md @@ -0,0 +1,487 @@ +--- +{ + "title": "回归测试", + "language": "zh-CN" +} + +--- + + + +# 回归测试 + +## 概念 +1. `Suite`: 一个测试用例，目前仅用来指代测试用例文件名 +2. `Group`: 一个测试集，目前仅用于指代测试用例所属的目录 +3. `Action`: 一个封装好的具体测试行为，比如用于执行sql的`sql` Action，用于校验结果的`test` Action，用于导入数据的`streamLoad` Action等 + +## 测试步骤 +1. 需要预先安装好集群 +2. 修改配置文件`${DORIS_HOME}/conf/regression-conf.groovy`，设置jdbc url、用户等配置项 +3. 创建测试用例文件并编写用例 +4. 如果用例文件包含`qt` Action，则需要创建关联的的data文件，比如`suites/demo/qt_action.groovy`这个例子，需要用到`data/demo/qt_action.out`这个TSV文件来校验输出是否一致 +5. 运行`${DORIS_HOME}/run-regression-test.sh`测试全部用例,或运行`${DORIS_HOME}/run-regression-test.sh --run ` 测试若干用例，更多例子见"启动脚本例子"章节 + +## 目录结构 +开发时需要关注的重要文件/目录 +1. `run-regression-test.sh`: 启动脚本 +2. `regression-conf.groovy`: 回归测试的默认配置 +3. `data`: 存放输入数据和输出校验数据 +4. `suites`: 存放用例 + +``` +./${DORIS_HOME} + |-- **run-regression-test.sh** 回归测试启动脚本 + |-- regression-test + | |-- conf + | | |-- logback.xml 日志配置文件 + | | |-- **regression-conf.groovy** 默认配置文件 + | | + | |-- framework 回归测试框架源码 + | |-- **data** 用例的输入输出文件 + | | |-- demo 存放demo的输入输出文件 + | | |-- correctness 存放正确性测试用例的输入输出文件 + | | |-- performance 存放性能测试用例的输入输出文件 + | | |-- utils 存放其他工具的输入输出文件 + | | + | |-- **suites** 回归测试用例 + | |-- demo 存放测试用例的demo + | |-- correctness 存放正确性测试用例 + | |-- performance 存放性能测试用例 + | |-- utils 其他工具 + | + |-- output + |-- regression-test + |-- log 回归测试日志 +``` + + +## 框架默认配置 +测试时需要实际情况修改jdbc和fe的配置 +```groovy + +/* ============ 一般只需要关注下面这部分 ============ */ +// 默认DB,如果未创建,则会尝试创建这个db +defaultDb = "regression_test" + +// Jdbc配置 +jdbcUrl = "jdbc:mysql://127.0.0.1:9030/?" +jdbcUser = "root" +jdbcPassword = "" + +// fe地址配置, 用于stream load +feHttpAddress = "127.0.0.1:8030" +feHttpUser = "root" +feHttpPassword = "" + +/* ============ 一般不需要修改下面的部分 ============ */ + +// DORIS_HOME变量是通过run-regression-test.sh传入的 +// 即 java -DDORIS_HOME=./ + +// 设置回归测试用例的目录 +suitePath = "${DORIS_HOME}/regression-test/suites" +// 设置输入输出数据的目录 +dataPath = "${DORIS_HOME}/regression-test/data" + +// 默认会读所有的组,读多个组可以用半角逗号隔开，如: "demo,performance" +// 一般不需要在配置文件中修改，而是通过run-regression-test.sh来动态指定和覆盖 +testGroups = "" +// 默认会读所有的用例, 同样可以使用run-regression-test.sh来动态指定和覆盖 +testSuites = "" + +// 其他自定义配置 +customConf1 = "test_custom_conf_value" +``` + +## 编写用例的步骤 +1. 进入`${DORIS_HOME}/regression-test`目录 +2. 根据测试的目的来选择用例的目录，正确性测试存在`suites/correctness`，而性能测试存在`suites/performance` +3. 新建一个groovy用例文件，增加若干`Action`用于测试，Action讲在后续章节具体说明 + +## Action +Action是一个测试框架默认提供的测试行为，使用DSL来定义。 + +### sql action +sql action用于提交sql并获取结果，如果查询失败则会抛出异常 + +参数如下 +- String sql: 输入的sql字符串 +- return List>: 查询结果，如果是DDL/DML，则返回一行一列，唯一的值是updateRowCount + +下面的样例代码存放于`${DORIS_HOME}/regression-test/suites/demo/sql_action.groovy`: +```groovy +// execute sql and ignore result +sql "show databases" + +// execute sql and get result, outer List denote rows, inner List denote columns in a single row +List> tables = sql "show tables" + +// assertXxx() will invoke junit5's Assertions.assertXxx() dynamically +assertTrue(tables.size() >= 0) // test rowCount >= 0 + +// syntax error +try { + sql "a b c d e" + throw new IllegalStateException("Should be syntax error") +} catch (java.sql.SQLException t) { + assertTrue(true) +} + +def testTable = "test_sql_action1" + +try { + sql "DROP TABLE IF EXISTS ${testTable}" + + // multi-line sql + def result1 = sql """ + CREATE TABLE IF NOT EXISTS ${testTable} ( + id int + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + // DDL/DML return 1 row and 1 column, the only value is update row count + assertTrue(result1.size() == 1) + assertTrue(result1[0].size() == 1) + assertTrue(result1[0][0] == 0, "Create table should update 0 rows") + + def result2 = sql "INSERT INTO test_sql_action1 values(1), (2), (3)" + assertTrue(result2.size() == 1) + assertTrue(result2[0].size() == 1) + assertTrue(result2[0][0] == 3, "Insert should update 3 rows") +} finally { + /** + * try_xxx(args) means: + * + * try { + * return xxx(args) + * } catch (Throwable t) { + * // do nothing + * } + */ + try_sql("DROP TABLE IF EXISTS ${testTable}") + + // you can see the error sql will not throw exception and return + try { + def errorSqlResult = try_sql("a b c d e f g") + assertTrue(errorSqlResult == null) + } catch (Throwable t) { + assertTrue(false, "Never catch exception") + } +} + +// order_sql(sqlStr) equals to sql(sqlStr, isOrder=true) +// sort result by string dict +def list = order_sql """ + select 2 + union all + select 1 + union all + select null + union all + select 15 + union all + select 3 + """ +assertEquals(null, list[0][0]) +assertEquals(1, list[1][0]) +assertEquals(15, list[2][0]) +assertEquals(2, list[3][0]) +assertEquals(3, list[4][0]) +``` + +### qt action +qt action用于提交sql，并使用对应的.out TSV文件来校验结果 +- String sql: 输入sql字符串 +- return void + +下面的样例代码存放于`${DORIS_HOME}/regression-test/suites/demo/qt_action.groovy`: +```groovy +/** + * qt_xxx sql equals to quickTest(xxx, sql) witch xxx is tag. + * the result will be compare to the relate file: ${DORIS_HOME}/regression_test/data/qt_action.out. + * + * if you want to generate .out tsv file for real execute result. you can run with -genOut or -forceGenOut option. + * e.g + * ${DORIS_HOME}/run-regression-test.sh --run qt_action -genOut + * ${DORIS_HOME}/run-regression-test.sh --run qt_action -forceGenOut + */ +qt_select "select 1, 'beijing' union all select 2, 'shanghai'" + +qt_select2 "select 2" + +// order result by string dict then compare to .out file. +// order_qt_xxx sql equals to quickTest(xxx, sql, true). +order_qt_union_all """ + select 2 + union all + select 1 + union all + select null + union all + select 15 + union all + select 3 + """ +``` + +### test action +test action可以使用更复杂的校验规则来测试，比如验证行数、执行时间、是否抛出异常 + +可用参数 +- String sql: 输入的sql字符串 +- List> result: 提供一个List对象，用于校验真实查询结果对比是否相等 +- String exception: 校验抛出的异常是否包含某些字符串 +- long rowNum: 验证结果行数 +- long time: 验证执行时间是否小于这个值，单位是毫秒 +- Closure>, Throwable, Long, Long> check: 自定义回调校验，可传入结果、异常、时间。存在回调函数时，其他校验方式会失效。 + +下面的样例代码存放于`${DORIS_HOME}/regression-test/suites/demo/qt_action.groovy`: +```groovy +test { + sql "abcdefg" + // check exception message contains + exception "errCode = 2, detailMessage = Syntax error" +} + +test { + sql """ + select * + from ( + select 1 id + union all + select 2 + ) a + order by id""" + + // multi check condition + + // check return 2 rows + rowNum 2 + // execute time must <= 5000 millisecond + time 5000 + // check result, must be 2 rows and 1 column, the first row is 1, second is 2 + result( + [[1], [2]] + ) +} + +test { + sql "a b c d e f g" + + // other check will not work because already declared a check callback + exception "aaaaaaaaa" + + // callback + check { result, exception, startTime, endTime -> + // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically + assertTrue(exception != null) + } +} + +test { + sql """ + select 2 + union all + select 1 + union all + select null + union all + select 15 + union all + select 3 + """ + + check { result, ex, startTime, endTime -> + // same as order_sql(sqlStr) + result = sortRows(result) + + assertEquals(null, result[0][0]) + assertEquals(1, result[1][0]) + assertEquals(15, result[2][0]) + assertEquals(2, result[3][0]) + assertEquals(3, result[4][0]) + } +} +``` + +### explain action +explain action用来校验explain返回的字符串是否包含某些字符串 + +可用参数: +- String sql: 查询的sql，需要去掉sql中的explain +- String contains: 校验explain是否包含某些字符串，可多次调用校验同时多个结果 +- String notContains: 校验explain是否不含某些字符串，可多次调用校验同时多个结果 +- Closure check: 自定义校验回调函数，可以获取返回的字符串，存在校验函数时，其他校验方式会失效 +- Closure check: 自定义校验回调函数，可以额外获取异常和时间 + +下面的样例代码存放于`${DORIS_HOME}/regression-test/suites/demo/explain_action.groovy`: +```groovy +explain { + sql("select 100") + + // contains("OUTPUT EXPRS: 100\n") && contains("PARTITION: UNPARTITIONED\n") + contains "OUTPUT EXPRS: 100\n" + contains "PARTITION: UNPARTITIONED\n" +} + +explain { + sql("select 100") + + // contains(" 100\n") && !contains("abcdefg") && !("1234567") + contains " 100\n" + notContains "abcdefg" + notContains "1234567" +} + +explain { + sql("select 100") + // simple callback + check { explainStr -> explainStr.contains("abcdefg") || explainStr.contains(" 100\n") } +} + +explain { + sql("a b c d e") + // callback with exception and time + check { explainStr, exception, startTime, endTime -> + // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically + assertTrue(exception != null) + } +} +``` + +### streamLoad action +streamLoad action用于导入数据 +可用参数为 +- String db: db，默认值为regression-conf.groovy中的defaultDb +- String table: 表名 +- String file: 要导入的文件路径，可以写data目录下的相对路径，或者写http url来导入网络文件 +- Iterator> inputIterator: 要导入的迭代器 +- String inputText: 要导入的文本, 较为少用 +- InputStream inputStream: 要导入的字节流，较为少用 +- long time: 验证执行时间是否小于这个值，单位是毫秒 +- void set(String key, String value): 设置stream load的http请求的header，如label、columnSeparator +- Closure check: 自定义校验回调函数，可以获取返回结果、异常和超时时间。当存在回调函数时，其他校验项会失效。 + +下面的样例代码存放于`${DORIS_HOME}/regression-test/suites/demo/streamLoad_action.groovy`: +```groovy +def tableName = "test_streamload_action1" + +sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + id int, + name varchar(255) + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) +""" + +streamLoad { + // you can skip declare db, because a default db already specify in ${DORIS_HOME}/conf/regression-conf.groovy + // db 'regression_test' + table tableName + + // default label is UUID: + // set 'label' UUID.randomUUID().toString() + + // default column_separator is specify in doris fe config, usually is '\t'. + // this line change to ',' + set 'column_separator', ',' + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file 'streamload_input.csv' + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows +} + + +// stream load 100 rows +def rowCount = 100 +def rowIt = java.util.stream.LongStream.range(0, rowCount) // [0, rowCount) + .mapToObj({i -> [i, "a_" + i]}) // change Long to List + .iterator() + +streamLoad { + table tableName + // also, you can upload a memory iterator + inputIterator rowIt + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } +} +``` + +## 启动脚本例子 +```shell +# 查看脚本参数说明 +./run-regression-test.sh h + +# 查看框架参数说明 +./run-regression-test.sh --run -h + +# 测试所有用例 +./run-regression-test.sh + +# 删除测试框架编译结果和测试日志 +./run-regression-test.sh --clean + +# 测试suiteName为sql_action的用例, 目前suiteName等于文件名前缀，例子对应的用例文件是sql_action.groovy +./run-regression-test.sh --run sql_action + +# 测试suiteName包含'sql'的用例，**注意需要用单引号括起来** +./run-regression-test.sh --run '*sql*' + +# 测试demo和perfomance group +./run-regression-test.sh --run -g 'demo,performance' + +# 测试demo group下的sql_action +./run-regression-test.sh --run -g demo -s sql_action + +# 自定义配置 +./run-regression-test.sh --run -conf a=b +``` + +## 使用查询结果自动生成.out文件 +```shell +# 使用查询结果自动生成sql_action用例的.out文件，如果.out文件存在则忽略 +./run-regression-test.sh --run sql_action -genOut + +# 使用查询结果自动生成sql_action用例的.out文件，如果.out文件存在则覆盖 +./run-regression-test.sh --run sql_action -forceGenOut +``` \ No newline at end of file diff --git a/regression-test/conf/logback.xml b/regression-test/conf/logback.xml new file mode 100644 index 00000000000000..2bae4b568353dc --- /dev/null +++ b/regression-test/conf/logback.xml @@ -0,0 +1,43 @@ + + + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} %level [%thread] \(%F:%L\) - %msg%n + + + + + + ${LOG_PATH:-./log}/doris-regression-test.${LOG_TIME}.log + + %d{yyyy-MM-dd HH:mm:ss.SSS} %level [%thread] \(%F:%L\) - %msg%n + + + + + + + + diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy new file mode 100644 index 00000000000000..7c34d464f152ee --- /dev/null +++ b/regression-test/conf/regression-conf.groovy @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* ******* Do not commit this file unless you know what you are doing ******* */ + +// **Note**: default db will be create if not exist +defaultDb = "regression_test" + +jdbcUrl = "jdbc:mysql://127.0.0.1:9030/?" +jdbcUser = "root" +jdbcPassword = "" + +feHttpAddress = "127.0.0.1:8030" +feHttpUser = "root" +feHttpPassword = "" + +// set DORIS_HOME by system properties +// e.g. java -DDORIS_HOME=./ +suitePath = "${DORIS_HOME}/regression-test/suites" +dataPath = "${DORIS_HOME}/regression-test/data" + +// will test /.groovy +// empty group will test all group +testGroups = "" +// empty suite will test all suite +testSuites = "" + +customConf1 = "test_custom_conf_value" \ No newline at end of file diff --git a/regression-test/data/correctness/test_select_constant.out b/regression-test/data/correctness/test_select_constant.out new file mode 100644 index 00000000000000..d3ed51bcc74093 --- /dev/null +++ b/regression-test/data/correctness/test_select_constant.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select1 -- +100 test 2021-01-02 + diff --git a/regression-test/data/demo/qt_action.out b/regression-test/data/demo/qt_action.out new file mode 100644 index 00000000000000..ea3e7b473443e5 --- /dev/null +++ b/regression-test/data/demo/qt_action.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1 beijing +2 shanghai + +-- !select2 -- +2 + +-- !union -- +\N +1 +15 +2 +3 + diff --git a/regression-test/data/demo/streamload_input.csv b/regression-test/data/demo/streamload_input.csv new file mode 100644 index 00000000000000..6b9bcb226ff467 --- /dev/null +++ b/regression-test/data/demo/streamload_input.csv @@ -0,0 +1,3 @@ +1,BeiJing +2,ShangHai +3,GuangZhou \ No newline at end of file diff --git a/regression-test/framework/README b/regression-test/framework/README new file mode 100644 index 00000000000000..2b8a083742008b --- /dev/null +++ b/regression-test/framework/README @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# framework diff --git a/regression-test/framework/checkstyle-apache-header.txt b/regression-test/framework/checkstyle-apache-header.txt new file mode 100644 index 00000000000000..6e778edd7530ea --- /dev/null +++ b/regression-test/framework/checkstyle-apache-header.txt @@ -0,0 +1,16 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. \ No newline at end of file diff --git a/regression-test/framework/checkstyle.xml b/regression-test/framework/checkstyle.xml new file mode 100644 index 00000000000000..7eba1555fa2a13 --- /dev/null +++ b/regression-test/framework/checkstyle.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + diff --git a/regression-test/framework/pom.xml b/regression-test/framework/pom.xml new file mode 100644 index 00000000000000..bcf2e4e015acd2 --- /dev/null +++ b/regression-test/framework/pom.xml @@ -0,0 +1,219 @@ + + + + 4.0.0 + + org.apache + apache + 23 + + org.apache.doris + regression-test + ${revision} + jar + + Doris Regression Test Project + https://doris.apache.org/ + + + Apache 2.0 License + https://www.apache.org/licenses/LICENSE-2.0.html + repo + + + + scm:git:https://git@github.com/apache/incubator-doris.git + scm:git:https://git@github.com/apache/incubator-doris.git + scm:git:https://git@github.com/apache/incubator-doris.git + HEAD + + + GitHub + https://github.com/apache/incubator-doris/issues + + + + Dev Mailing List + dev@doris.apache.org + dev-subscribe@doris.apache.org + dev-unsubscribe@doris.apache.org + + + Commits Mailing List + commits@doris.apache.org + commits-subscribe@doris.apache.org + commits-unsubscribe@doris.apache.org + + + + ${basedir}/../../ + UTF-8 + 1.8 + 1.8 + 1.0-SNAPSHOT + github + 3.0.7 + 3.0.7-01 + 3.7.0 + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.2.5 + + + org.apache.maven.plugins + maven-compiler-plugin + 3.9.0 + + groovy-eclipse-compiler + ${maven.compiler.source} + ${maven.compiler.target} + true + + + + org.codehaus.groovy + groovy-eclipse-compiler + ${groovy-eclipse-compiler.version} + + + org.codehaus.groovy + groovy-eclipse-batch + ${groovy-eclipse-batch.version} + + + + + org.codehaus.groovy + groovy-eclipse-compiler + ${groovy-eclipse-compiler.version} + true + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + package + + shade + + + + + org.apache.doris.regression.RegressionTest + + + + + + + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.2.5 + + true + bom + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + + + + + + + + org.codehaus.groovy + groovy-all + ${groovy.version} + pom + + + org.junit.jupiter + junit-jupiter-api + 5.8.2 + + + mysql + mysql-connector-java + 8.0.28 + + + commons-cli + commons-cli + 1.5.0 + + + org.jodd + jodd-core + 5.3.0 + + + ch.qos.logback + logback-classic + 1.2.10 + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + commons-io + commons-io + 2.11.0 + + + org.apache.commons + commons-csv + 1.9.0 + + + com.google.guava + guava + 31.0.1-jre + + + \ No newline at end of file diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy new file mode 100644 index 00000000000000..3b578cca2ab892 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy @@ -0,0 +1,259 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j + +import com.google.common.collect.Maps +import org.apache.commons.cli.CommandLine +import org.apache.doris.regression.util.FileUtils +import org.apache.doris.regression.util.JdbcUtils + +import java.sql.Connection +import java.sql.DriverManager + +import static org.apache.doris.regression.ConfigOptions.* + +@Slf4j +@CompileStatic +class Config { + public String jdbcUrl + public String jdbcUser + public String jdbcPassword + public String defaultDb + + public String feHttpAddress + public String feHttpUser + public String feHttpPassword + + public String suitePath + public String dataPath + + public String testGroups + public String testSuites + public boolean generateOutputFile + public boolean forceGenerateOutputFile + + public Properties otherConfigs = new Properties() + + public Set suiteWildcard = new HashSet<>() + public Set groups = new HashSet<>() + public InetSocketAddress feHttpInetSocketAddress + + Config() {} + + Config(String defaultDb, String jdbcUrl, String jdbcUser, String jdbcPassword, + String feHttpAddress, String feHttpUser, String feHttpPassword, + String suitePath, String dataPath, String testGroups, String testSuites) { + this.defaultDb = defaultDb + this.jdbcUrl = jdbcUrl + this.jdbcUser = jdbcUser + this.jdbcPassword = jdbcPassword + this.feHttpAddress = feHttpAddress + this.feHttpUser = feHttpUser + this.feHttpPassword = feHttpPassword + this.suitePath = suitePath + this.dataPath = dataPath + this.testGroups = testGroups + this.testSuites = testSuites + } + + static Config fromCommandLine(CommandLine cmd) { + String confFilePath = cmd.getOptionValue(confFileOpt, "") + File confFile = new File(confFilePath) + Config config = new Config() + if (confFile.exists() && confFile.isFile()) { + log.info("Load config file ${confFilePath}".toString()) + def configSlurper = new ConfigSlurper() + def systemProperties = Maps.newLinkedHashMap(System.getProperties()) + configSlurper.setBinding(systemProperties) + ConfigObject configObj = configSlurper.parse(new File(confFilePath).toURI().toURL()) + config = Config.fromConfigObject(configObj) + } + + fillDefaultConfig(config) + + config.suitePath = FileUtils.getCanonicalPath(cmd.getOptionValue(pathOpt, config.suitePath)) + config.dataPath = FileUtils.getCanonicalPath(cmd.getOptionValue(dataOpt, config.dataPath)) + config.suiteWildcard = cmd.getOptionValue(suiteOpt, config.testSuites) + .split(",") + .collect({g -> g.trim()}) + .findAll({g -> g != null && g.length() > 0}) + .toSet() + config.groups = cmd.getOptionValue(groupsOpt, config.testGroups) + .split(",") + .collect({g -> g.trim()}) + .findAll({g -> g != null && g.length() > 0}) + .toSet() + + config.feHttpAddress = cmd.getOptionValue(feHttpAddressOpt, config.feHttpAddress) + try { + Inet4Address host = Inet4Address.getByName(config.feHttpAddress.split(":")[0]) as Inet4Address + int port = Integer.valueOf(config.feHttpAddress.split(":")[1]) + config.feHttpInetSocketAddress = new InetSocketAddress(host, port) + } catch (Throwable t) { + throw new IllegalStateException("Can not parse stream load address: ${config.feHttpAddress}", t) + } + + config.defaultDb = cmd.getOptionValue(jdbcOpt, config.defaultDb) + config.jdbcUrl = cmd.getOptionValue(jdbcOpt, config.jdbcUrl) + config.jdbcUser = cmd.getOptionValue(userOpt, config.jdbcUser) + config.jdbcPassword = cmd.getOptionValue(passwordOpt, config.jdbcPassword) + config.feHttpUser = cmd.getOptionValue(feHttpUserOpt, config.feHttpUser) + config.feHttpPassword = cmd.getOptionValue(feHttpPasswordOpt, config.feHttpPassword) + config.generateOutputFile = cmd.hasOption(genOutOpt) + config.forceGenerateOutputFile = cmd.hasOption(forceGenOutOpt) + Properties props = cmd.getOptionProperties("conf") + config.otherConfigs.putAll(props) + + config.tryCreateDbIfNotExist() + config.buildUrlWithDefaultDb() + + return config + } + + static Config fromConfigObject(ConfigObject obj) { + def config = new Config( + configToString(obj.defaultDb), + configToString(obj.jdbcUrl), + configToString(obj.jdbcUser), + configToString(obj.jdbcPassword), + configToString(obj.feHttpAddress), + configToString(obj.feHttpUser), + configToString(obj.feHttpPassword), + configToString(obj.suitePath), + configToString(obj.dataPath), + configToString(obj.testGroups), + configToString(obj.testSuites) + ) + + def declareFileNames = config.getClass() + .getDeclaredFields() + .collect({f -> f.name}) + .toSet() + for (def kv : obj.toProperties().entrySet()) { + String key = kv.getKey() as String + if (!declareFileNames.contains(key)) { + config.otherConfigs.put(key, kv.getValue()) + } + } + return config + } + + static void fillDefaultConfig(Config config) { + if (config.defaultDb == null) { + config.defaultDb = "regression_test" + log.info("Set defaultDb to '${config.defaultDb}' because not specify.".toString()) + } + + if (config.jdbcUrl == null) { + config.jdbcUrl = "jdbc:mysql://127.0.0.1:9030" + log.info("Set jdbcUrl to '${config.jdbcUrl}' because not specify.".toString()) + } + + if (config.jdbcUser == null) { + config.jdbcUser = "root" + log.info("Set jdbcUser to '${config.jdbcUser}' because not specify.".toString()) + } + + if (config.jdbcPassword == null) { + config.jdbcPassword = "" + log.info("Set jdbcPassword to empty because not specify.".toString()) + } + + if (config.feHttpAddress == null) { + config.feHttpAddress = "127.0.0.1:8030" + log.info("Set feHttpAddress to '${config.feHttpAddress}' because not specify.".toString()) + } + + if (config.feHttpUser == null) { + config.feHttpUser = "root" + log.info("Set feHttpUser to '${config.feHttpUser}' because not specify.".toString()) + } + + if (config.feHttpPassword == null) { + config.feHttpPassword = "" + log.info("Set feHttpPassword to empty because not specify.".toString()) + } + + if (config.suitePath == null) { + config.suitePath = "regression-test/suites" + log.info("Set suitePath to '${config.suitePath}' because not specify.".toString()) + } + + if (config.dataPath == null) { + config.dataPath = "regression-test/suites" + log.info("Set dataPath to '${config.dataPath}' because not specify.".toString()) + } + + if (config.testGroups == null) { + config.testGroups = "default" + log.info("Set testGroups to '${config.testGroups}' because not specify.".toString()) + } + + if (config.testSuites == null) { + config.testSuites = "" + log.info("Set testSuites to empty because not specify.".toString()) + } + } + + static String configToString(Object obj) { + return (obj instanceof String || obj instanceof GString) ? obj.toString() : null + } + + void tryCreateDbIfNotExist() { + // connect without specify default db + try { + String sql = "CREATE DATABASE IF NOT EXISTS ${defaultDb}" + log.info("Try to create db, sql: ${sql}".toString()) + getConnection().withCloseable { conn -> + JdbcUtils.executeToList(conn, sql) + } + } catch (Throwable t) { + throw new IllegalStateException("Create database failed, jdbcUrl: ${jdbcUrl}", t) + } + } + + Connection getConnection() { + return DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword) + } + + private void buildUrlWithDefaultDb() { + String urlWithDb = jdbcUrl + String urlWithoutSchema = jdbcUrl.substring(jdbcUrl.indexOf("://") + 3) + if (urlWithoutSchema.indexOf("/") >= 0) { + if (jdbcUrl.contains("?")) { + // e.g: jdbc:mysql://locahost:8080/?a=b + urlWithDb = jdbcUrl.substring(0, jdbcUrl.lastIndexOf("/")) + urlWithDb += ("/" + defaultDb) + jdbcUrl.substring(jdbcUrl.lastIndexOf("?")) + } else { + // e.g: jdbc:mysql://locahost:8080/ + urlWithDb += defaultDb + } + } else { + // e.g: jdbc:mysql://locahost:8080 + urlWithDb += ("/" + defaultDb) + } + this.jdbcUrl = urlWithDb + log.info("Reset jdbcUrl to ${jdbcUrl}".toString()) + + // check connection with default db + getConnection().close() + } +} \ No newline at end of file diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy new file mode 100644 index 00000000000000..9f02986470cd31 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression + +import groovy.transform.CompileStatic +import org.apache.commons.cli.CommandLine +import org.apache.commons.cli.DefaultParser +import org.apache.commons.cli.HelpFormatter +import org.apache.commons.cli.Option +import org.apache.commons.cli.Options + +@CompileStatic +class ConfigOptions { + static Option helpOption + static Option confFileOpt + static Option defaultDbOpt + static Option jdbcOpt + static Option userOpt + static Option passwordOpt + static Option feHttpAddressOpt + static Option feHttpUserOpt + static Option feHttpPasswordOpt + static Option pathOpt + static Option dataOpt + static Option suiteOpt + static Option groupsOpt + static Option confOpt + static Option genOutOpt + static Option forceGenOutOpt + + static CommandLine initCommands(String[] args) { + helpOption = Option.builder("h") + .required(false) + .hasArg(false) + .longOpt("help") + .desc("print this usage help") + .build() + confFileOpt = Option.builder("cf") + .argName("confFilePath") + .required(false) + .hasArg(true) + .type(String.class) + .longOpt("confFile") + .desc("the configure file path") + .build() + defaultDbOpt = Option.builder("db") + .argName("db") + .required(false) + .hasArg(true) + .type(String.class) + .longOpt("defaultDb") + .desc("default db") + .build() + jdbcOpt = Option.builder("c") + .argName("url") + .required(false) + .hasArg(true) + .type(String.class) + .longOpt("jdbc") + .desc("jdbc url") + .build() + userOpt = Option.builder("u") + .argName("user") + .required(false) + .hasArg(true) + .type(String.class) + .longOpt("user") + .desc("the username of jdbc connection") + .build() + passwordOpt = Option.builder("p") + .argName("password") + .required(false) + .hasArg(true) + .optionalArg(true) + .type(String.class) + .longOpt("password") + .desc("the password of jdbc connection") + .build() + pathOpt = Option.builder("P") + .argName("path") + .required(false) + .hasArg(true) + .type(String.class) + .longOpt("path") + .desc("the suite path") + .build() + dataOpt = Option.builder("D") + .argName("dataPath") + .required(false) + .hasArg(true) + .type(String.class) + .longOpt("dataPath") + .desc("the data path") + .build() + suiteOpt = Option.builder("s") + .argName("suiteName") + .required(false) + .hasArg(true) + .optionalArg(true) + .type(String.class) + .longOpt("suite") + .desc("the suite name wildcard to be test") + .build() + groupsOpt = Option.builder("g") + .argName("groups") + .required(false) + .hasArg(true) + .optionalArg(true) + .type(String.class) + .longOpt("groups") + .desc("the suite group to be test") + .build() + feHttpAddressOpt = Option.builder("ha") + .argName("address") + .required(false) + .hasArg(true) + .type(String.class) + .longOpt("feHttpAddress") + .desc("the fe http address, format is ip:port") + .build() + feHttpUserOpt = Option.builder("hu") + .argName("userName") + .required(false) + .hasArg(true) + .type(String.class) + .longOpt("feHttpUser") + .desc("the user of fe http server") + .build() + feHttpPasswordOpt = Option.builder("hp") + .argName("password") + .required(false) + .hasArg(true) + .type(String.class) + .longOpt("feHttpPassword") + .desc("the password of fe http server") + .build() + genOutOpt = Option.builder("genOut") + .required(false) + .hasArg(false) + .desc("generate qt .out file if not exist") + .build() + forceGenOutOpt = Option.builder("forceGenOut") + .required(false) + .hasArg(false) + .desc("delete and generate qt .out file") + .build() + confOpt = Option.builder("conf") + .argName("conf") + .required(false) + .hasArgs() + .valueSeparator(('=' as char)) + .longOpt("configurations, format: key=value") + .desc("set addition context configurations") + .build() + + Options options = new Options() + .addOption(helpOption) + .addOption(jdbcOpt) + .addOption(userOpt) + .addOption(passwordOpt) + .addOption(pathOpt) + .addOption(dataOpt) + .addOption(confOpt) + .addOption(suiteOpt) + .addOption(groupsOpt) + .addOption(feHttpAddressOpt) + .addOption(feHttpUserOpt) + .addOption(feHttpPasswordOpt) + .addOption(genOutOpt) + .addOption(confFileOpt) + .addOption(forceGenOutOpt) + + CommandLine cmd = new DefaultParser().parse(options, args, true) + if (cmd.hasOption(helpOption)) { + printHelp(options) + return null + } + return cmd + } + + static void printHelp(Options options) { + HelpFormatter hf = new HelpFormatter() + hf.printHelp("regression-test", options, true) + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy new file mode 100644 index 00000000000000..5f0bd6efe7e45c --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy @@ -0,0 +1,189 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression + +import groovy.transform.CompileStatic +import jodd.util.Wildcard +import org.apache.doris.regression.suite.Suite +import org.apache.doris.regression.suite.SuiteContext +import org.apache.doris.regression.util.Recorder +import groovy.util.logging.Slf4j +import org.apache.commons.cli.* +import org.apache.doris.regression.util.SuiteInfo +import org.codehaus.groovy.control.CompilerConfiguration + +import java.util.stream.Collectors + +@Slf4j +@CompileStatic +class RegressionTest { + static ClassLoader classloader + static CompilerConfiguration compileConfig + static GroovyShell shell + + static void main(String[] args) { + CommandLine cmd = ConfigOptions.initCommands(args) + if (cmd == null) { + return + } + + Config config = Config.fromCommandLine(cmd) + initGroovyEnv(config) + Recorder recorder = runSuites(config) + printResult(config, recorder) + } + + static void initGroovyEnv(Config config) { + classloader = new GroovyClassLoader() + compileConfig = new CompilerConfiguration() + compileConfig.setScriptBaseClass((Suite as Class).name) + shell = new GroovyShell(classloader, new Binding(), compileConfig) + } + + static List findSuiteFiles(String root) { + if (root == null) { + log.warn("Not specify suite path") + return new ArrayList() + } + List files = new ArrayList<>() + new File(root).eachFileRecurse { f -> + if (f.isFile() && f.name.endsWith(".groovy")) { + files.add(f) + } + } + return files + } + + static String parseGroup(Config config, File suiteFile) { + String group = new File(config.suitePath).relativePath(suiteFile) + int separatorIndex = group.lastIndexOf(File.separator) + if (separatorIndex == -1) { + return "" + } else { + return group.substring(0, separatorIndex) + } + } + + static Recorder runSuites(Config config) { + def files = findSuiteFiles(config.suitePath) + def recorder = new Recorder() + List runScripts = files.stream().map({ file -> + String suiteName = file.name.substring(0, file.name.lastIndexOf(".")) + String group = parseGroup(config, file) + return new SuiteFile(file, suiteName, group) + }).filter({ sf -> + canRun(config, sf.suiteName, sf.group) + }).collect(Collectors.toList()) + + log.info("Start to run suites") + int totalFile = runScripts.size() + runScripts.eachWithIndex { sf, i -> + File file = sf.file + String suiteName = sf.suiteName + String group = sf.group + def suiteConn = config.getConnection() + new SuiteContext(file, suiteConn, config, recorder).withCloseable { context -> + try { + log.info("[${i + 1}/${totalFile}] Run ${suiteName} in $file".toString()) + Suite suite = shell.parse(file) as Suite + suite.init(suiteName, group, context) + suite.run() + recorder.onSuccess(new SuiteInfo(file, group, suiteName)) + log.info("Run ${suiteName} in ${file.absolutePath} succeed".toString()) + } catch (Throwable t) { + recorder.onFailure(new SuiteInfo(file, group, suiteName)) + log.error("Run ${suiteName} in ${file.absolutePath} failed".toString(), t) + } + } + } + return recorder + } + + static boolean canRun(Config config, String suiteName, String group) { + Set suiteGroups = group.split(",").collect {g -> g.trim()}.toSet() + if (config.suiteWildcard.size() == 0 || + (suiteName != null && (config.suiteWildcard.any { + suiteWildcard -> Wildcard.match(suiteName, suiteWildcard) + }))) { + + if (config.groups == null || config.groups.isEmpty() + || !config.groups.intersect(suiteGroups).isEmpty()) { + return true + } + } + return false + } + + static void printResult(Config config, Recorder recorder) { + int allSuiteNum = recorder.successList.size() + recorder.failureList.size() + int failedSuiteNum = recorder.failureList.size() + log.info("Test ${allSuiteNum} suites, failed ${failedSuiteNum} suites".toString()) + + // print success list + { + String successList = recorder.successList.collect { info -> + "${info.file.absolutePath}: group=${info.group}, name=${info.suiteName}" + }.join("\n") + log.info("successList suites:\n${successList}".toString()) + } + + // print failure list + if (!recorder.failureList.isEmpty()) { + def failureList = recorder.failureList.collect() { info -> + "${info.file.absolutePath}: group=${info.group}, name=${info.suiteName}" + }.join("\n") + log.info("Failure suites:\n${failureList}".toString()) + printFailed() + throw new IllegalStateException("Test failed") + } else { + printPassed() + } + } + + static void printPassed() { + log.info("""All suites success. + | ____ _ ____ ____ _____ ____ + || _ \\ / \\ / ___/ ___|| ____| _ \\ + || |_) / _ \\ \\___ \\___ \\| _| | | | | + || __/ ___ \\ ___) |__) | |___| |_| | + ||_| /_/ \\_\\____/____/|_____|____/ + |""".stripMargin()) + } + + static void printFailed() { + log.info("""Some suites failed. + | _____ _ ___ _ _____ ____ + || ___/ \\ |_ _| | | ____| _ \\ + || |_ / _ \\ | || | | _| | | | | + || _/ ___ \\ | || |___| |___| |_| | + ||_|/_/ \\_\\___|_____|_____|____/ + |""".stripMargin()) + } + + static class SuiteFile { + File file + String suiteName + String group + + SuiteFile(File file, String suiteName, String group) { + this.file = file + this.suiteName = suiteName + this.group = group + } + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ExplainAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ExplainAction.groovy new file mode 100644 index 00000000000000..9617b79b39a1ff --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ExplainAction.groovy @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.action + +import org.apache.doris.regression.suite.SuiteContext +import org.apache.doris.regression.util.JdbcUtils +import groovy.util.logging.Slf4j + +import java.util.stream.Collectors + +@Slf4j +class ExplainAction implements SuiteAction { + private String sql + private SuiteContext context + private Set containsStrings = new LinkedHashSet<>() + private Set notContainsStrings = new LinkedHashSet<>() + private Closure checkFunction + + ExplainAction(SuiteContext context) { + this.context = context + } + + void sql(String sql) { + this.sql = sql + } + + void sql(Closure sqlSupplier) { + this.sql = sqlSupplier.call() + } + + void contains(String subString) { + containsStrings.add(subString) + } + + void notContains(String subString) { + notContainsStrings.add(subString) + } + + void check(Closure checkFunction) { + this.checkFunction = checkFunction + } + + @Override + void run() { + String explainSql = "explain\n" + sql + def result = doTest(explainSql) + String explainString = result.result + if (checkFunction != null) { + try { + Boolean checkResult = null + if (checkFunction.parameterTypes.size() == 1) { + if (result.exception == null) { + checkResult = checkFunction(explainString) + } else { + throw result.exception + } + } else { + checkResult = checkFunction(explainString, result.exception, result.startTime, result.endTime) + } + if (checkResult != null && checkResult.booleanValue() == false) { + String msg = "Explain and custom check failed, actual explain string is:\n${explainString}".toString() + throw new IllegalStateException(msg) + } + } catch (Throwable t) { + log.error("Explain and custom check failed", t) + List resList = [context.file.getName(), 'explain', sql, t] + context.recorder.reportDiffResult(resList) + throw t + } + } else if (result.exception != null) { + String msg = "Explain failed" + log.error(msg, result.exception) + List resList = [context.file.getName(), 'explain', sql, result.exception] + context.recorder.reportDiffResult(resList) + throw new IllegalStateException(msg, result.exception) + } else { + for (String string : containsStrings) { + if (!explainString.contains(string)) { + String msg = ("Explain and check failed, expect contains '${string}'," + + "but actual explain string is:\n${explainString}").toString() + log.info(msg) + def t = new IllegalStateException(msg) + List resList = [context.file.getName(), 'explain', sql, t] + context.recorder.reportDiffResult(resList) + throw t + } + } + for (String string : notContainsStrings) { + if (explainString.contains(string)) { + String msg = ("Explain and check failed, expect not contains '${string}'," + + "but actual explain string is:\n${explainString}").toString() + log.info(msg) + def t = new IllegalStateException(msg) + List resList = [context.file.getName(), 'explain', sql, t] + context.recorder.reportDiffResult(resList) + throw t + } + } + } + } + + private ActionResult doTest(String explainSql) { + log.info("Execute sql:\n${explainSql}".toString()) + long startTime = System.currentTimeMillis() + String explainString = null + try { + explainString = JdbcUtils.executeToList(context.conn, explainSql).stream() + .map({row -> row.get(0).toString()}) + .collect(Collectors.joining("\n")) + return new ActionResult(explainString, null, startTime, System.currentTimeMillis()) + } catch (Throwable t) { + return new ActionResult(explainString, t, startTime, System.currentTimeMillis()) + } + } + + class ActionResult { + String result + Throwable exception + long startTime + long endTime + + ActionResult(String result, Throwable exception, long startTime, long endTime) { + this.result = result + this.exception = exception + this.startTime = startTime + this.endTime = endTime + } + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy new file mode 100644 index 00000000000000..54953c980a87d7 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy @@ -0,0 +1,297 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.action + +import com.google.common.collect.Iterators +import org.apache.doris.regression.suite.SuiteContext +import org.apache.doris.regression.util.BytesInputStream +import org.apache.doris.regression.util.OutputUtils +import groovy.json.JsonSlurper +import groovy.util.logging.Slf4j +import org.apache.http.HttpEntity +import org.apache.http.client.methods.RequestBuilder +import org.apache.http.entity.FileEntity +import org.apache.http.entity.InputStreamEntity +import org.apache.http.entity.StringEntity +import org.apache.http.impl.client.CloseableHttpClient +import org.apache.http.impl.client.HttpClients +import org.apache.http.util.EntityUtils +import org.junit.Assert + +@Slf4j +class StreamLoadAction implements SuiteAction { + public final InetSocketAddress address + public final String user + public final String password + String db + String table + String file + InputStream inputStream + String inputText + Iterator> inputIterator + long time + Closure check + Map headers + SuiteContext context + + StreamLoadAction(SuiteContext context) { + this.address = context.config.feHttpInetSocketAddress + this.user = context.config.feHttpUser + this.password = context.config.feHttpPassword + this.db = context.config.defaultDb + this.context = context + this.headers = new LinkedHashMap<>() + this.headers.put('label', UUID.randomUUID().toString()) + } + + void db(String db) { + this.db = db + } + + void db(Closure db) { + this.db = db.call() + } + + void table(String table) { + this.table = table + } + + void table(Closure table) { + this.table = table.call() + } + + void inputStream(InputStream inputStream) { + this.inputStream = inputStream + } + + void inputStream(Closure inputStream) { + this.inputStream = inputStream.call() + } + + void inputIterator(Iterator> inputIterator) { + this.inputIterator = inputIterator + } + + void inputIterator(Closure>> inputIterator) { + this.inputIterator = inputIterator.call() + } + + void inputText(String inputText) { + this.inputText = inputText + } + + void inputText(Closure inputText) { + this.inputText = inputText.call() + } + + void file(String file) { + this.file = file + } + + void file(Closure file) { + this.file = file.call() + } + + void time(long time) { + this.time = time + } + + void time(Closure time) { + this.time = time.call() + } + + void check(Closure check) { + this.check = check + } + + void set(String key, String value) { + headers.put(key, value) + } + + @Override + void run() { + String responseText = null + Throwable ex = null + long startTime = System.currentTimeMillis() + try { + def uri = "http://${address.hostString}:${address.port}/api/${db}/${table}/_stream_load" + HttpClients.createDefault().withCloseable { client -> + RequestBuilder requestBuilder = prepareRequestHeader(RequestBuilder.put(uri)) + HttpEntity httpEntity = prepareHttpEntity(client) + String beLocation = streamLoadToFe(client, requestBuilder) + responseText = streamLoadToBe(client, requestBuilder, beLocation, httpEntity) + } + } catch (Throwable t) { + ex = t + } + long endTime = System.currentTimeMillis() + log.info("Stream load elapsed ${endTime - startTime} ms".toString()) + checkResult(responseText, ex, startTime, endTime) + } + + private String httpGetString(CloseableHttpClient client, String url) { + return client.execute(RequestBuilder.get(url).build()).withCloseable { resp -> + EntityUtils.toString(resp.getEntity()) + } + } + + private InputStream httpGetStream(CloseableHttpClient client, String url) { + return client.execute(RequestBuilder.get(url).build()).getEntity().getContent() + } + + private RequestBuilder prepareRequestHeader(RequestBuilder requestBuilder) { + String encoding = Base64.getEncoder() + .encodeToString((user + ":" + (password == null ? "" : password)).getBytes("UTF-8")) + requestBuilder.setHeader("Authorization", "Basic ${encoding}") + + for (Map.Entry entry : headers.entrySet()) { + requestBuilder.setHeader(entry.key, entry.value) + } + requestBuilder.setHeader("Expect", "100-Continue") + return requestBuilder + } + + private HttpEntity prepareHttpEntity(CloseableHttpClient client) { + HttpEntity entity = null + if (inputStream != null) { + entity = new InputStreamEntity(inputStream) + } else if (inputText != null) { + entity = new StringEntity(inputText) + } else if (inputIterator != null) { + def bytesIt = Iterators.transform(inputIterator, + {row -> (OutputUtils.toCsvString(row) + "\n").getBytes()}) + entity = new InputStreamEntity(new BytesInputStream(bytesIt)) + } else { + String fileName = this.file + if (fileName.startsWith("http://") || fileName.startsWith("https://")) { + log.info("Set stream load input: ${fileName}".toString()) + entity = new InputStreamEntity(httpGetStream(client, fileName)) + } else { // local file + if (!new File(fileName).isAbsolute()) { + fileName = new File(context.dataPath, fileName).getAbsolutePath() + } + def file = new File(fileName) + if (!file.exists()) { + log.warn("Stream load input file not exists: ${file}".toString()) + } + log.info("Set stream load input: ${file.canonicalPath}".toString()) + entity = new FileEntity(file) + } + } + return entity + } + + private String streamLoadToFe(CloseableHttpClient client, RequestBuilder requestBuilder) { + log.info("Stream load to ${requestBuilder.uri}".toString()) + String backendStreamLoadUri = null + client.execute(requestBuilder.build()).withCloseable { resp -> + resp.withCloseable { + String body = EntityUtils.toString(resp.getEntity()) + def respCode = resp.getStatusLine().getStatusCode() + // should redirect to backend + if (respCode != 307) { + List resList = [context.file.getName(), 'streamLoad', '', "Expect frontend stream load response code is 307, " + + "but meet ${respCode}\nbody: ${body}"] + context.recorder.reportDiffResult(resList) + throw new IllegalStateException("Expect frontend stream load response code is 307, " + + "but meet ${respCode}\nbody: ${body}") + } + backendStreamLoadUri = resp.getFirstHeader("location").getValue() + } + } + return backendStreamLoadUri + } + + private String streamLoadToBe(CloseableHttpClient client, RequestBuilder requestBuilder, String beLocation, HttpEntity httpEntity) { + log.info("Redirect stream load to ${beLocation}".toString()) + requestBuilder.setUri(beLocation) + requestBuilder.setEntity(httpEntity) + String responseText + try{ + client.execute(requestBuilder.build()).withCloseable { resp -> + resp.withCloseable { + String body = EntityUtils.toString(resp.getEntity()) + def respCode = resp.getStatusLine().getStatusCode() + if (respCode != 200) { + List resList = [context.file.getName(), 'streamLoad', '', "Expect backend stream load response code is 200, " + + "but meet ${respCode}\nbody: ${body}"] + context.recorder.reportDiffResult(resList) + + throw new IllegalStateException("Expect backend stream load response code is 200, " + + "but meet ${respCode}\nbody: ${body}") + } + responseText = body + } + } + } catch (Throwable t) { + log.info("StreamLoadAction Exception: ", t) + List resList = [context.file.getName(), 'streamLoad', '', t] + context.recorder.reportDiffResult(resList) + } + return responseText + } + + private void checkResult(String responseText, Throwable ex, long startTime, long endTime) { + if (check != null) { + check.call(responseText, ex, startTime, endTime) + } else { + if (ex != null) { + List resList = [context.file.getName(), 'streamLoad', '', ex] + context.recorder.reportDiffResult(resList) + throw ex + } + + def jsonSlurper = new JsonSlurper() + def result = jsonSlurper.parseText(responseText) + String status = result.Status + if (!"Success".equalsIgnoreCase(status)) { + String errorUrl = result.ErrorURL + if (errorUrl != null) { + String errorDetails = HttpClients.createDefault().withCloseable { client -> + httpGetString(client, errorUrl) + } + List resList = [context.file.getName(), 'streamLoad', '', "Stream load failed:\n${responseText}\n${errorDetails}"] + context.recorder.reportDiffResult(resList) + throw new IllegalStateException("Stream load failed:\n${responseText}\n${errorDetails}") + } + List resList = [context.file.getName(), 'streamLoad', '', "Stream load failed:\n${responseText}"] + context.recorder.reportDiffResult(resList) + throw new IllegalStateException("Stream load failed:\n${responseText}") + } + long numberTotalRows = result.NumberTotalRows.toLong() + long numberLoadedRows = result.NumberLoadedRows.toLong() + if (numberTotalRows != numberLoadedRows) { + List resList = [context.file.getName(), 'streamLoad', '', "Stream load rows mismatch:\n${responseText}"] + context.recorder.reportDiffResult(resList) + throw new IllegalStateException("Stream load rows mismatch:\n${responseText}") + + } + + if (time > 0) { + long elapsed = endTime - startTime + try{ + Assert.assertTrue("Expect elapsed <= ${time}, but meet ${elapsed}", elapsed <= time) + } catch (Throwable t) { + List resList = [context.file.getName(), 'streamLoad', '', "Expect elapsed <= ${time}, but meet ${elapsed}"] + context.recorder.reportDiffResult(resList) + throw new IllegalStateException("Expect elapsed <= ${time}, but meet ${elapsed}") + } + } + } + } +} \ No newline at end of file diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/SuiteAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/SuiteAction.groovy new file mode 100644 index 00000000000000..29dc0bbfbfdcfc --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/SuiteAction.groovy @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.action + +interface SuiteAction { + void run() +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy new file mode 100644 index 00000000000000..7b0e3cd92ad8b4 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.action + +import groovy.util.logging.Slf4j +import java.sql.Connection + +import org.apache.doris.regression.suite.SuiteContext +import org.apache.doris.regression.util.JdbcUtils +import org.junit.Assert + +@Slf4j +class TestAction implements SuiteAction { + private String sql + private Object result + private long time + private long rowNum = -1 + private String exception + private Closure check + SuiteContext context + + TestAction(SuiteContext context) { + this.context = context + } + + @Override + void run() { + try{ + def result = doRun(context.conn) + if (check != null) { + check.call(result.result, result.exception, result.startTime, result.endTime) + } else { + if (exception != null || result.exception != null) { + def msg = result.exception?.toString() + Assert.assertTrue("Expect exception msg contains '${exception}', but meet '${msg}'", + msg != null && exception != null && msg.contains(exception)) + } + if (time > 0) { + long elapsed = result.endTime - result.startTime + Assert.assertTrue("Expect elapsed <= ${time}, but meet ${elapsed}", elapsed <= time) + } + if (rowNum >= 0) { + if (result.result instanceof Integer || result.result instanceof Long) { + def realRowNum = ((Number) result.result).longValue() + Assert.assertEquals("RowNum", rowNum, realRowNum) + } else if (result.result instanceof List) { + def realRowNum = ((List) result.result).size().longValue() + Assert.assertEquals("RowNum", rowNum, realRowNum) + } else { + log.warn("Unknown result: ${result.result}, can not check row num".toString()) + } + } + if (this.result != null) { + Assert.assertEquals(this.result, result.result) + } + } + } catch (Throwable t) { + log.info("TestAction Exception: ", t) + List resList = [context.file.getName(), 'test', sql, t] + context.recorder.reportDiffResult(resList) + throw t + } + } + + ActionResult doRun(Connection conn) { + Object result = null + Throwable ex = null + long startTime = System.currentTimeMillis() + try { + log.info("Execute sql:\n${sql}".toString()) + result = JdbcUtils.executeToList(conn, sql) + } catch (Throwable t) { + ex = t + } + long endTime = System.currentTimeMillis() + return new ActionResult(result, ex, startTime, endTime) + } + + void sql(String sql) { + this.sql = sql + } + + void sql(Closure sqlSupplier) { + this.sql = sqlSupplier.call() + } + + void time(long time) { + this.time = time + } + + void time(Closure timeSupplier) { + this.time = timeSupplier.call() + } + + void rowNum(long rowNum) { + this.rowNum = rowNum + } + + void rowNum(Closure rowNum) { + this.rowNum = rowNum.call() + } + + void result(Object result) { + this.result = result + } + + void result(Closure resultSupplier) { + this.result = resultSupplier.call() + } + + void exception(String exceptionMsg) { + this.exception = exceptionMsg + } + + void exception(Closure exceptionMsgSupplier) { + this.exception = exceptionMsgSupplier.call() + } + + void check(Closure check) { + this.check = check + } + + class ActionResult { + Object result + Throwable exception + long startTime + long endTime + + ActionResult(Object result, Throwable exception, long startTime, long endTime) { + this.result = result + this.exception = exception + this.startTime = startTime + this.endTime = endTime + } + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy new file mode 100644 index 00000000000000..371c89503435eb --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.suite + +import groovy.json.JsonSlurper +import groovy.util.logging.Slf4j + +import com.google.common.collect.ImmutableList +import org.apache.doris.regression.util.DataUtils +import org.apache.doris.regression.util.OutputUtils +import org.apache.doris.regression.action.ExplainAction +import org.apache.doris.regression.action.StreamLoadAction +import org.apache.doris.regression.action.SuiteAction +import org.apache.doris.regression.action.TestAction +import org.apache.doris.regression.util.JdbcUtils +import org.junit.jupiter.api.Assertions + +import static org.apache.doris.regression.util.DataUtils.sortByToString + +@Slf4j +abstract class Suite extends Script implements GroovyInterceptable { + SuiteContext context + String name + String group + + void init(String name, String group, SuiteContext context) { + this.name = name + this.group = group + this.context = context + } + + String getConf(String key, String defaultValue = null) { + String value = context.config.otherConfigs.get(key) + return value == null ? defaultValue : value + } + + Properties getConfs(String prefix) { + Properties p = new Properties() + for (String name : context.config.otherConfigs.stringPropertyNames()) { + if (name.startsWith(prefix + ".")) { + p.put(name.substring(prefix.length() + 1), context.config.getProperty(name)) + } + } + return p + } + + String toCsv(List rows) { + StringBuilder sb = new StringBuilder() + for (int i = 0; i < rows.size(); ++i) { + Object row = rows.get(i) + if (!(row instanceof List)) { + row = ImmutableList.of(row) + } + sb.append(OutputUtils.toCsvString(row as List)).append("\n") + } + sb.toString() + } + + Object parseJson(String str) { + def jsonSlurper = new JsonSlurper() + return jsonSlurper.parseText(str) + } + + Object sql(String sqlStr, boolean isOrder = false) { + log.info("Execute sql: ${sqlStr}".toString()) + def result = JdbcUtils.executeToList(context.conn, sqlStr) + if (isOrder) { + result = DataUtils.sortByToString(result) + } + return result + } + + Object order_sql(String sqlStr) { + return sql(sqlStr, true) + } + + List> sortRows(List> result) { + if (result == null) { + return null + } + return DataUtils.sortByToString(result) + } + + void explain(Closure actionSupplier) { + runAction(new ExplainAction(context), actionSupplier) + } + + void test(Closure actionSupplier) { + runAction(new TestAction(context), actionSupplier) + } + + void streamLoad(Closure actionSupplier) { + runAction(new StreamLoadAction(context), actionSupplier) + } + + void runAction(SuiteAction action, Closure actionSupplier) { + actionSupplier.setDelegate(action) + actionSupplier.setResolveStrategy(Closure.DELEGATE_FIRST) + actionSupplier.call() + action.run() + } + + void quickTest(String tag, String sql, boolean order = false) { + log.info("Execute tag: ${tag}, sql: ${sql}".toString()) + + if (context.config.generateOutputFile || context.config.forceGenerateOutputFile) { + def result = JdbcUtils.executorToStringList(context.conn, sql) + if (order) { + result = sortByToString(result) + } + Iterator> realResults = result.iterator() + // generate and save to .out file + def writer = context.getOutputWriter(context.config.forceGenerateOutputFile) + writer.write(realResults, tag) + } else { + if (context.outputIterator == null) { + String res = "Missing outputFile: ${context.outputFile.getAbsolutePath()}" + List excelContentList = [context.file.getName(), context.file, context.file, res] + context.recorder.reportDiffResult(excelContentList) + throw new IllegalStateException("Missing outputFile: ${context.outputFile.getAbsolutePath()}") + } + + if (!context.outputIterator.hasNext()) { + String res = "Missing output block for tag '${tag}': ${context.outputFile.getAbsolutePath()}" + List excelContentList = [context.file.getName(), tag, context.file, res] + context.recorder.reportDiffResult(excelContentList) + throw new IllegalStateException("Missing output block for tag '${tag}': ${context.outputFile.getAbsolutePath()}") + } + + try { + Iterator> expectCsvResults = context.outputIterator.next() as Iterator + List> realResults = JdbcUtils.executorToStringList(context.conn, sql) + if (order) { + realResults = sortByToString(realResults) + } + def res = OutputUtils.assertEquals(expectCsvResults, realResults.iterator(), "Tag '${tag}' wrong") + if (res) { + List excelContentList = [context.file.getName(), tag, sql.trim(), res] + context.recorder.reportDiffResult(excelContentList) + throw new IllegalStateException("'${tag}' line not match . Detailed results is : '${res}'") + } + } catch (Throwable t) { + if (t.toString().contains('line not match . Detailed results is')) { + throw t + } else { + List excelContentList = [context.file.getName(), tag, sql.trim(), t] + context.recorder.reportDiffResult(excelContentList) + throw new IllegalStateException("'${tag}' run failed . Detailed failure information is : '${t}'", t) + } + } + } + } + + @Override + Object invokeMethod(String name, Object args) { + // qt: quick test + if (name.startsWith("qt_")) { + return quickTest(name.substring("qt_".length()), (args as Object[])[0] as String) + } else if (name.startsWith("order_qt_")) { + return quickTest(name.substring("order_qt_".length()), (args as Object[])[0] as String, true) + } else if (name.startsWith("assert") && name.length() > "assert".length()) { + // delegate to junit Assertions dynamically + return Assertions."$name"(*args) // *args: spread-dot + } else if (name.startsWith("try_")) { + String realMethod = name.substring("try_".length()) + try { + return this."$realMethod"(*args) + } catch (Throwable t) { + // do nothing + } + } else { + // invoke origin method + return metaClass.invokeMethod(this, name, args) + } + } + + private Object invokeAssertions(String name, Object args) { + + } +} + diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy new file mode 100644 index 00000000000000..41e7c698111895 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.suite + +import groovy.transform.CompileStatic +import org.apache.doris.regression.Config +import org.apache.doris.regression.util.OutputUtils +import org.apache.doris.regression.util.Recorder +import groovy.util.logging.Slf4j +import org.apache.doris.regression.util.CloseableIterator + +import java.sql.Connection + +@Slf4j +@CompileStatic +class SuiteContext implements Closeable { + public final File file + public final Connection conn + public final Config config + public final File dataPath + public final File outputFile + public final Recorder recorder +// public final File tmpOutputPath + public final CloseableIterator>> outputIterator + private volatile OutputUtils.OutputBlocksWriter outputBlocksWriter + + SuiteContext(File file, Connection conn, Config config, Recorder recorder) { + this.file = file + this.conn = conn + this.config = config + this.recorder = recorder + + def path = new File(config.suitePath).relativePath(file) + def outputRelativePath = path.substring(0, path.lastIndexOf(".")) + ".out" + this.outputFile = new File(new File(config.dataPath), outputRelativePath) + this.dataPath = this.outputFile.getParentFile().getCanonicalFile() + if (!config.otherConfigs.getProperty("qt.generate.out", "false").toBoolean() + && outputFile.exists()) { + this.outputIterator = OutputUtils.iterator(outputFile) + } +// def dataParentPath = new File(config.dataPath).parentFile.absolutePath +// def tmpOutputPath = "${dataParentPath}/tmp_output/${outputRelativePath}".toString() +// this.tmpOutputPath = new File(tmpOutputPath) + } + + OutputUtils.OutputBlocksWriter getOutputWriter(boolean deleteIfExist) { + if (outputBlocksWriter != null) { + return outputBlocksWriter + } + synchronized (this) { + if (outputBlocksWriter != null) { + return outputBlocksWriter + } else if (outputFile.exists() && deleteIfExist) { + log.info("Delete ${outputFile}".toString()) + outputFile.delete() + log.info("Generate ${outputFile}".toString()) + outputFile.createNewFile() + outputBlocksWriter = OutputUtils.writer(outputFile) + } else if (!outputFile.exists()) { + outputFile.parentFile.mkdirs() + outputFile.createNewFile() + log.info("Generate ${outputFile}".toString()) + outputBlocksWriter = OutputUtils.writer(outputFile) + } else { + log.info("Skip generate output file because exists: ${outputFile}".toString()) + outputBlocksWriter = new OutputUtils.OutputBlocksWriter(null) + } + return outputBlocksWriter + } + } + + @Override + void close() { + if (outputIterator != null) { + try { + outputIterator.close() + } catch (Throwable t) { + log.warn("Close outputFile failed", t) + } + } + + if (outputBlocksWriter != null) { + outputBlocksWriter.close() + } + + try { + conn.close() + } catch (Throwable t) { + log.warn("Close connection failed", t) + } + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/BytesInputStream.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/BytesInputStream.groovy new file mode 100644 index 00000000000000..ce64ff09e489df --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/BytesInputStream.groovy @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.util + +import groovy.transform.CompileStatic + +@CompileStatic +class BytesInputStream extends InputStream { + private Iterator bytesIt + private ByteArrayInputStream currentStream = new ByteArrayInputStream(new byte[0]) + + BytesInputStream(Iterator bytesIt) { + this.bytesIt = bytesIt + } + + @Override + int read() throws IOException { + int byteValue = currentStream.read() + if (byteValue == -1) { + if (bytesIt.hasNext()) { + currentStream = new ByteArrayInputStream(bytesIt.next()) + return read() + } else { + return -1 + } + } + return byteValue + } + + @Override + int read(byte[] b, int off, int len) throws IOException { + int readSize = 0 + + while (readSize < len) { + int read = currentStream.read(b, off + readSize, len - readSize) + if (read == -1) { + if (bytesIt.hasNext()) { + currentStream = new ByteArrayInputStream(bytesIt.next()) + continue + } else { + return readSize > 0 ? readSize : -1 + } + } else if (read > 0) { + readSize += read + } else if (read == 0) { + break + } + } + return readSize + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/CloseableIterator.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/CloseableIterator.groovy new file mode 100644 index 00000000000000..5cd77a036ebe84 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/CloseableIterator.groovy @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.util + +import groovy.transform.CompileStatic + +@CompileStatic +interface CloseableIterator extends Iterator, Closeable { +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/DataUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/DataUtils.groovy new file mode 100644 index 00000000000000..ec85ca645a6eb9 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/DataUtils.groovy @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.util + +import groovy.transform.CompileStatic + +@CompileStatic +class DataUtils { + // null first, order by column.toString asc + static List> sortByToString(List> originData) { + def comparator = Comparator.naturalOrder() + originData.sort(false, { row1, row2 -> + for (int i = 0; i < row1.size(); ++i) { + Object column1 = row1[i] + Object column2 = row2[i] + if (column1 == column2) { + continue + } + if (column1 == null) { + return -1 + } else if (column2 == null) { + return 1 + } + int result = comparator.compare(column1.toString(), column2.toString()) + if (result != 0) { + return result + } + } + return 0 + }) + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/FileUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/FileUtils.groovy new file mode 100644 index 00000000000000..dac8b9e80cbae8 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/FileUtils.groovy @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.util + +import groovy.transform.CompileStatic + +@CompileStatic +class FileUtils { + static String getCanonicalPath(String path) { + if (path == null) { + return null + } + if (!new File(path as String).isAbsolute()) { + path = new File(path).getCanonicalPath() + } + return path + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/JdbcUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/JdbcUtils.groovy new file mode 100644 index 00000000000000..b0a0bcdecbde6a --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/JdbcUtils.groovy @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.util + +import com.google.common.collect.ImmutableList + +import java.sql.Connection +import java.sql.ResultSet + +class JdbcUtils { + static List> executeToList(Connection conn, String sql) { + conn.prepareStatement(sql).withCloseable { stmt -> + boolean hasResultSet = stmt.execute() + if (!hasResultSet) { + return ImmutableList.of(ImmutableList.of(stmt.getUpdateCount())) + } else { + toList(stmt.resultSet) + } + } + } + + static List> executorToStringList(Connection conn, String sql) { + conn.prepareStatement(sql).withCloseable { stmt -> + boolean hasResultSet = stmt.execute() + if (!hasResultSet) { + return ImmutableList.of(ImmutableList.of(stmt.getUpdateCount())) + } else { + toStringList(stmt.resultSet) + } + } + } + + static List> toList(ResultSet resultSet) { + resultSet.withCloseable { + List> rows = new ArrayList<>() + def columnCount = resultSet.metaData.columnCount + while (resultSet.next()) { + def row = new ArrayList<>() + for (int i = 1; i <= columnCount; ++i) { + row.add(resultSet.getObject(i)) + } + rows.add(row) + } + return rows + } + } + + static List> toStringList(ResultSet resultSet) { + resultSet.withCloseable { + List> rows = new ArrayList<>() + def columnCount = resultSet.metaData.columnCount + while (resultSet.next()) { + def row = new ArrayList<>() + for (int i = 1; i <= columnCount; ++i) { + // row.add(resultSet.getObject(i)) + // row.add(resultSet.getString(i)) + try { + row.add(resultSet.getObject(i)) + } catch (Throwable t) { + if(resultSet.getBytes(i) != null){ + row.add(new String(resultSet.getBytes(i))) + } else { + row.add(resultSet.getObject(i)) + } + } + } + rows.add(row) + } + return rows + } + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy new file mode 100644 index 00000000000000..d5ecb51ba6dc4e --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy @@ -0,0 +1,275 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.util + +import com.google.common.collect.ImmutableList +import groovy.transform.CompileStatic +import org.apache.commons.csv.CSVFormat +import org.apache.commons.csv.CSVPrinter +import org.apache.commons.csv.CSVRecord +import org.apache.commons.io.LineIterator + +@CompileStatic +class OutputUtils { + static toCsvString(List row) { + StringWriter writer = new StringWriter() + def printer = new CSVPrinter(new PrintWriter(writer), CSVFormat.MYSQL) + for (int i = 0; i < row.size(); ++i) { + printer.print(row.get(i)) + } + return writer.toString() + } + + static assertEquals(Iterator> expect, Iterator> real, String info) { + while (true) { + if (expect.hasNext() && !real.hasNext()) { + def res = "${info}, line not match, real line is empty, but expect is ${expect.next()}" + return res + // throw new IllegalStateException("${info}, line not match, real line is empty, but expect is ${expect.next()}") + } + if (!expect.hasNext() && real.hasNext()) { + def res = "${info}, line not match, expect line is empty, but real is ${toCsvString(real.next())}" + return res + // throw new IllegalStateException("${info}, line not match, expect line is empty, but real is ${toCsvString(real.next())}") + } + if (!expect.hasNext() && !real.hasNext()) { + break + } + + def expectCsvString = toCsvString(expect.next() as List) + def realCsvString = toCsvString(real.next()) + if (!expectCsvString.equals(realCsvString)) { + def res = "${info}, line not match.\nExpect line is: ${expectCsvString}\nBut real is : ${realCsvString}" + return res + // throw new IllegalStateException("${info}, line not match.\nExpect line is: ${expectCsvString}\nBut real is : ${realCsvString}") + } + } + } + + static CloseableIterator>> iterator(File file) { + def it = new ReusableIterator(new LineIteratorAdaptor(new LineIterator(new FileReader(file)))) + return new OutputBlocksIterator(it) + } + + static OutputBlocksWriter writer(File file) { + return new OutputBlocksWriter(file) + } + + static class LineIteratorAdaptor implements CloseableIterator { + LineIterator lineIt + + LineIteratorAdaptor(LineIterator lineIt) { + this.lineIt = lineIt + } + + @Override + void close() throws IOException { + lineIt.close() + } + + @Override + boolean hasNext() { + return lineIt.hasNext() + } + + @Override + String next() { + return lineIt.next() + } + } + + static class OutputBlocksWriter { + private PrintWriter writer + + OutputBlocksWriter(File file) { + if (file != null) { + writer = file.newPrintWriter() + writer.println("""-- This file is automatically generated. You should know what you did if you want to edit this""") + } + } + + void write(Iterator> real, String comment) { + if (writer != null) { + writer.println("-- !${comment} --") + while (real.hasNext()) { + writer.println(toCsvString(real.next() as List)) + } + writer.println() + } + } + + void close() { + if (writer != null) { + writer.close() + } + } + } + + static class OutputBlocksIterator implements CloseableIterator>> { + private ReusableIterator lineIt + private CsvParserIterator cache + private boolean cached + + OutputBlocksIterator(ReusableIterator lineIt) { + this.lineIt = lineIt + } + + @Override + void close() throws IOException { + lineIt.close() + } + + @Override + boolean hasNext() { + if (!cached) { + if (cache != null) { + while (cache.hasNext()) { + cache.next() + } + } + if (!lineIt.hasNext()) { + return false + } + + // find next comment block + while (true) { + String blockComment = lineIt.next() // skip block comment, e.g. -- !qt_sql_1 -- + if (blockComment.startsWith("-- !") && blockComment.endsWith(" --")) { + break + } + if (!lineIt.hasNext()) { + return false + } + } + cache = new CsvParserIterator(new SkipLastEmptyLineIterator(new OutputBlockIterator(lineIt))) + cached = true + return true + } else { + return true + } + } + + @Override + Iterator> next() { + if (hasNext()) { + cached = false + return cache + } + throw new NoSuchElementException() + } + } + + static class CsvParserIterator implements Iterator> { + private Iterator it + + CsvParserIterator(Iterator it) { + this.it = it + } + + @Override + boolean hasNext() { + return it.hasNext() + } + + @Override + List next() { + String line = it.next() + if (line.size() == 0) { + return ImmutableList.of(line) + } + CSVRecord record = CSVFormat.MYSQL.parse(new StringReader(line)).first() + List row = new ArrayList(record.size()) + for (int i = 0; i < record.size(); ++i) { + row.add(record.get(i)) + } + return row + } + } + + static class SkipLastEmptyLineIterator implements Iterator { + private Iterator it + private String cache + private boolean cached + + SkipLastEmptyLineIterator(Iterator it) { + this.it = it + } + + @Override + boolean hasNext() { + if (!cached) { + if (!it.hasNext()) { + return false + } + String next = it.next() + if (next.length() == 0 && !it.hasNext()) { + return false + } + cache = next + cached = true + return true + } else { + return true + } + } + + @Override + String next() { + if (hasNext()) { + cached = false + return cache + } + throw new NoSuchElementException() + } + } + + static class OutputBlockIterator implements Iterator { + private ReusableIterator it + + OutputBlockIterator(ReusableIterator it) { + this.it = it + } + + @Override + boolean hasNext() { + while (true) { + if (!it.hasNext()) { + return false + } + // predict next line + String line = it.preRead() + if (line.startsWith("-- !") && line.endsWith(" --")) { + return false + } else if (line.startsWith("-- ")) { + it.next() + continue + } + return true + } + return false + } + + @Override + String next() { + if (hasNext()) { + return it.next() + } + throw new NoSuchElementException() + } + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Recorder.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Recorder.groovy new file mode 100644 index 00000000000000..b1cb460580d67c --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Recorder.groovy @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.util + +import groovy.transform.CompileStatic + +@CompileStatic +class Recorder { + public final List successList = new ArrayList<>() + public final List failureList = new ArrayList<>() + + void onSuccess(SuiteInfo suiteInfo) { + successList.add(suiteInfo) + } + + void onFailure(SuiteInfo suiteInfo) { + failureList.add(suiteInfo) + } + + void reportDiffResult(List res) { + // TODO + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ReusableIterator.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ReusableIterator.groovy new file mode 100644 index 00000000000000..68cf9e8cb91706 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ReusableIterator.groovy @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.util + +import groovy.transform.CompileStatic + +@CompileStatic +class ReusableIterator implements CloseableIterator { + private CloseableIterator it + private T next + private boolean cached + + ReusableIterator(CloseableIterator it) { + this.it = it + } + + @Override + void close() throws IOException { + it.close() + } + + @Override + boolean hasNext() { + if (!cached) { + if (it.hasNext()) { + next = it.next() + cached = true + return true + } else { + return false + } + } else { + return true + } + } + + T preRead() { + return next + } + + @Override + T next() { + if (hasNext()) { + cached = false + return next + } + throw new NoSuchElementException() + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/SuiteInfo.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/SuiteInfo.groovy new file mode 100644 index 00000000000000..589d5b882c91e5 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/SuiteInfo.groovy @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.util + +import groovy.transform.CompileStatic + +@CompileStatic +class SuiteInfo { + File file + String group + String suiteName + + SuiteInfo(File file, String group, String suiteName) { + this.file = file + this.group = group + this.suiteName = suiteName + } +} diff --git a/regression-test/suites/correctness/test_select_constant.groovy b/regression-test/suites/correctness/test_select_constant.groovy new file mode 100644 index 00000000000000..2bf7589167e586 --- /dev/null +++ b/regression-test/suites/correctness/test_select_constant.groovy @@ -0,0 +1 @@ +qt_select1 'select 100, "test", date("2021-01-02")' \ No newline at end of file diff --git a/regression-test/suites/demo/explain_action.groovy b/regression-test/suites/demo/explain_action.groovy new file mode 100644 index 00000000000000..a5161a60eec8a7 --- /dev/null +++ b/regression-test/suites/demo/explain_action.groovy @@ -0,0 +1,31 @@ +explain { + sql("select 100") + + // contains("OUTPUT EXPRS: 100\n") && contains("PARTITION: UNPARTITIONED\n") + contains "OUTPUT EXPRS: 100\n" + contains "PARTITION: UNPARTITIONED\n" +} + +explain { + sql("select 100") + + // contains(" 100\n") && !contains("abcdefg") && !("1234567") + contains " 100\n" + notContains "abcdefg" + notContains "1234567" +} + +explain { + sql("select 100") + // simple callback + check { explainStr -> explainStr.contains("abcdefg") || explainStr.contains(" 100\n") } +} + +explain { + sql("a b c d e") + // callback with exception and time + check { explainStr, exception, startTime, endTime -> + // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically + assertTrue(exception != null) + } +} \ No newline at end of file diff --git a/regression-test/suites/demo/qt_action.groovy b/regression-test/suites/demo/qt_action.groovy new file mode 100644 index 00000000000000..e7a79d8c714235 --- /dev/null +++ b/regression-test/suites/demo/qt_action.groovy @@ -0,0 +1,26 @@ +/** + * qt_xxx sql equals to quickTest(xxx, sql) witch xxx is tag. + * the result will be compare to the relate file: ${DORIS_HOME}/regression_test/data/qt_action.out. + * + * if you want to generate .out tsv file for real execute result. you can run with -genOut or -forceGenOut option. + * e.g + * ${DORIS_HOME}/run-regression-test.sh --run qt_action -genOut + * ${DORIS_HOME}/run-regression-test.sh --run qt_action -forceGenOut + */ +qt_select "select 1, 'beijing' union all select 2, 'shanghai'" + +qt_select2 "select 2" + +// order result by string dict then compare to .out file. +// order_qt_xxx sql equals to quickTest(xxx, sql, true). +order_qt_union_all """ + select 2 + union all + select 1 + union all + select null + union all + select 15 + union all + select 3 + """ \ No newline at end of file diff --git a/regression-test/suites/demo/sql_action.groovy b/regression-test/suites/demo/sql_action.groovy new file mode 100644 index 00000000000000..9c737bde003407 --- /dev/null +++ b/regression-test/suites/demo/sql_action.groovy @@ -0,0 +1,81 @@ +// execute sql and ignore result +sql "show databases" + +// execute sql and get result, outer List denote rows, inner List denote columns in a single row +List> tables = sql "show tables" + +// assertXxx() will invoke junit5's Assertions.assertXxx() dynamically +assertTrue(tables.size() >= 0) // test rowCount >= 0 + +// syntax error +try { + sql "a b c d e" + throw new IllegalStateException("Should be syntax error") +} catch (java.sql.SQLException t) { + assertTrue(true) +} + +def testTable = "test_sql_action1" + +try { + sql "DROP TABLE IF EXISTS ${testTable}" + + // multi-line sql + def result1 = sql """ + CREATE TABLE IF NOT EXISTS ${testTable} ( + id int + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + // DDL/DML return 1 row and 1 column, the only value is update row count + assertTrue(result1.size() == 1) + assertTrue(result1[0].size() == 1) + assertTrue(result1[0][0] == 0, "Create table should update 0 rows") + + def result2 = sql "INSERT INTO test_sql_action1 values(1), (2), (3)" + assertTrue(result2.size() == 1) + assertTrue(result2[0].size() == 1) + assertTrue(result2[0][0] == 3, "Insert should update 3 rows") +} finally { + /** + * try_xxx(args) means: + * + * try { + * return xxx(args) + * } catch (Throwable t) { + * // do nothing + * } + */ + try_sql("DROP TABLE IF EXISTS ${testTable}") + + // you can see the error sql will not throw exception and return + try { + def errorSqlResult = try_sql("a b c d e f g") + assertTrue(errorSqlResult == null) + } catch (Throwable t) { + assertTrue(false, "Never catch exception") + } +} + +// order_sql(sqlStr) equals to sql(sqlStr, isOrder=true) +// sort result by string dict +def list = order_sql """ + select 2 + union all + select 1 + union all + select null + union all + select 15 + union all + select 3 + """ +assertEquals(null, list[0][0]) +assertEquals(1, list[1][0]) +assertEquals(15, list[2][0]) +assertEquals(2, list[3][0]) +assertEquals(3, list[4][0]) \ No newline at end of file diff --git a/regression-test/suites/demo/streamLoad_action.groovy b/regression-test/suites/demo/streamLoad_action.groovy new file mode 100644 index 00000000000000..b2881ae6d880d3 --- /dev/null +++ b/regression-test/suites/demo/streamLoad_action.groovy @@ -0,0 +1,59 @@ +def tableName = "test_streamload_action1" + +sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + id int, + name varchar(255) + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) +""" + +streamLoad { + // you can skip declare db, because a default db already specify in ${DORIS_HOME}/conf/regression-conf.groovy + // db 'regression_test' + table tableName + + // default label is UUID: + // set 'label' UUID.randomUUID().toString() + + // default column_separator is specify in doris fe config, usually is '\t'. + // this line change to ',' + set 'column_separator', ',' + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file 'streamload_input.csv' + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows +} + + +// stream load 100 rows +def rowCount = 100 +def rowIt = java.util.stream.LongStream.range(0, rowCount) // [0, rowCount) + .mapToObj({i -> [i, "a_" + i]}) // change Long to List + .iterator() + +streamLoad { + table tableName + // also, you can upload a memory iterator + inputIterator rowIt + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } +} \ No newline at end of file diff --git a/regression-test/suites/demo/test_action.groovy b/regression-test/suites/demo/test_action.groovy new file mode 100644 index 00000000000000..c84ea23b4054aa --- /dev/null +++ b/regression-test/suites/demo/test_action.groovy @@ -0,0 +1,65 @@ +test { + sql "abcdefg" + // check exception message contains + exception "errCode = 2, detailMessage = Syntax error" +} + +test { + sql """ + select * + from ( + select 1 id + union all + select 2 + ) a + order by id""" + + // multi check condition + + // check return 2 rows + rowNum 2 + // execute time must <= 5000 millisecond + time 5000 + // check result, must be 2 rows and 1 column, the first row is 1, second is 2 + result( + [[1], [2]] + ) +} + +test { + sql "a b c d e f g" + + // other check will not work because already declared a check callback + exception "aaaaaaaaa" + + // callback + check { result, exception, startTime, endTime -> + // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically + assertTrue(exception != null) + } +} + +test { + sql """ + select 2 + union all + select 1 + union all + select null + union all + select 15 + union all + select 3 + """ + + check { result, ex, startTime, endTime -> + // same as order_sql(sqlStr) + result = sortRows(result) + + assertEquals(null, result[0][0]) + assertEquals(1, result[1][0]) + assertEquals(15, result[2][0]) + assertEquals(2, result[3][0]) + assertEquals(3, result[4][0]) + } +} \ No newline at end of file diff --git a/regression-test/suites/performance/test_streamload_perfomance.groovy b/regression-test/suites/performance/test_streamload_perfomance.groovy new file mode 100644 index 00000000000000..df30baa19fba0e --- /dev/null +++ b/regression-test/suites/performance/test_streamload_perfomance.groovy @@ -0,0 +1,27 @@ +def tableName = "test_streamload_performance1" + +try { + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + id int, + name varchar(255) + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + def rowCount = 10000 + def rowIt = java.util.stream.LongStream.range(0, rowCount) + .mapToObj({i -> [i, "a_" + i]}) + .iterator() + + streamLoad { + table tableName + time 5000 + inputIterator rowIt + } +} finally { + try_sql "DROP TABLE IF EXISTS ${tableName}" +} \ No newline at end of file diff --git a/run-regression-test.sh b/run-regression-test.sh new file mode 100755 index 00000000000000..6af3b6dace040c --- /dev/null +++ b/run-regression-test.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +##################################################################### +# This script is used to run regression test of Doris Backend +# Usage: $0 +# Optional shell_options: +# --clean clean output of regression test +# --run run regression test. build framework if necessary +# +# Optional framework_options +# -h print all other_options +# -s xxx suite name +# -g xxx group name +# -c xxx jdbc url +# -u xxx jdbc user +# -genOut generate .out file +# -forceGenOut delete and generate .out file +# +# log to ${DORIS_HOME}/output/regression/log +##################################################################### + +set -eo pipefail +#set -x + +ROOT=`dirname "$0"` +ROOT=`cd "$ROOT"; pwd` + +DORIS_HOME=${ROOT} + +# Check args +usage() { + echo " +Usage: $0 + Optional shell_options: + --clean clean output of regression test framework + --run run regression test. build framework if necessary + + Optional framework_options: + -s run a specified suite + -g run a specified group + -h **print all framework options usage** + -genOut generate .out file if not exist + -forceGenOut delete and generate .out file if not exist + + Eg. + $0 build regression test framework and run all suite which in default group + $0 --run test_select run a suite which named as test_select + $0 --run 'test*' run all suite which named start with 'test', note that you must quota with '' + $0 --run -s test_select run a suite which named as test_select + $0 --run test_select -genOut generate output file for test_select if not exist + $0 --run -g default run all suite in the group which named as default + $0 --clean clean output of regression test framework + $0 --clean --run test_select clean output and build regression test framework and run a suite which named as test_select + $0 --run -h print framework options + +Log path: \${DORIS_HOME}/output/regression-test/log +Default config file: \${DORIS_HOME}/regression-test/conf/regression-conf.groovy + " + exit 1 +} + +CLEAN= +WRONG_CMD= +RUN= +if [ $# == 0 ] ; then + #default + CLEAN=0 + WRONG_CMD=0 + RUN=1 +else + CLEAN=0 + RUN=0 + WRONG_CMD=0 + while true; do + case "$1" in + --clean) CLEAN=1 ; shift ;; + --run) RUN=1 ; shift ;; + *) + if [ ${RUN} -eq 0 ] && [ ${CLEAN} -eq 0 ]; then + WRONG_CMD=1 + fi + break ;; + esac + done +fi + +if [ ${WRONG_CMD} -eq 1 ]; then + usage + exit 1 +fi + +# set maven +MVN_CMD=mvn +if [[ ! -z ${CUSTOM_MVN} ]]; then + MVN_CMD=${CUSTOM_MVN} +fi +if ! ${MVN_CMD} --version; then + echo "Error: mvn is not found" + exit 1 +fi +export MVN_CMD + +CONF_DIR=${DORIS_HOME}/regression-test/conf +CONFIG_FILE=${CONF_DIR}/regression-conf.groovy +LOG_CONFIG_FILE=${CONF_DIR}/logback.xml + +FRAMEWORK_SOURCE_DIR=${DORIS_HOME}/regression-test/framework +REGRESSION_TEST_BUILD_DIR=${FRAMEWORK_SOURCE_DIR}/target + +OUTPUT_DIR=${DORIS_HOME}/output/regression-test +LOG_OUTPUT_FILE=${OUTPUT_DIR}/log +RUN_JAR=${OUTPUT_DIR}/lib/regression-test-*.jar + +if [ ${CLEAN} -eq 1 ]; then + rm -rf ${REGRESSION_TEST_BUILD_DIR} + rm -rf ${OUTPUT_DIR} +fi + +if [ ${RUN} -ne 1 ]; then + echo "Finished" + exit 0 +fi + +if [ ! -f ${RUN_JAR} ]; then + echo "===== Build Regression Test Framework =====" + cd ${DORIS_HOME}/regression-test/framework + ${MVN_CMD} package + cd ${DORIS_HOME} + + mkdir -p ${OUTPUT_DIR}/{lib,log} + cp -r ${REGRESSION_TEST_BUILD_DIR}/regression-test-*.jar ${OUTPUT_DIR}/lib +fi + +# check java home +if [[ -z ${JAVA_HOME} ]]; then + echo "Error: JAVA_HOME is not set" + exit 1 +fi + +# check java version +export JAVA=${JAVA_HOME}/bin/java + + +REGRESSION_OPTIONS_PREFIX= + +# contains framework options and not start with - +# it should be suite name +if [ $# -ne 0 ] && [[ "$1" =~ ^[^-].* ]]; then + # specify suiteName + REGRESSION_OPTIONS_PREFIX='-s' +fi + +echo "===== Run Regression Test =====" + +$JAVA -DDORIS_HOME=$DORIS_HOME \ + -DLOG_PATH=$LOG_OUTPUT_FILE \ + -Dlogback.configurationFile=${LOG_CONFIG_FILE} \ + -jar ${RUN_JAR} \ + -cf ${CONFIG_FILE} \ + ${REGRESSION_OPTIONS_PREFIX} "$@"