diff --git a/.gitignore b/.gitignore index 775ba55c13..588206a441 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ target .idea/ .vscode .dir-locals.el +integration_tests/dist_query/dist-query-testing +integration_tests/dist_query/tsbs +integration_tests/dist_query/output diff --git a/integration_tests/Makefile b/integration_tests/Makefile index ebb0d69793..8eef988ac3 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -27,6 +27,7 @@ export RUST_BACKTRACE=1 # Whether update related repos export UPDATE_REPOS_TO_LATEST ?= true +export NO_INIT ?= false clean: rm -rf $(CERESDB_DATA_DIR) $(CERESDB_DATA_DIR_0) $(CERESDB_DATA_DIR_1) $(CERESMETA_DATA_DIR) diff --git a/integration_tests/dist_query/diff.py b/integration_tests/dist_query/diff.py new file mode 100644 index 0000000000..5af74ec2ac --- /dev/null +++ b/integration_tests/dist_query/diff.py @@ -0,0 +1,23 @@ +import difflib + +def main(): + # args = get_args() + # init_before_check = args['init_before_check'] + # ts = args['timestamp'] + # test_tables = get_test_tables(args['timestamp']) + f = open("./dist-query-testing/tsbs-cpu-only/queries.result") + lines1 = f.read().splitlines() + + f = open("./dist-query-testing/tsbs-cpu-only/queries.result.new") + lines2 = f.read().splitlines() + + text1_lines = lines1 + text2_lines = lines2 + + d = difflib.Differ() + l = list(d.compare(text1_lines, text2_lines)) + + print('\n'.join(l)) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/integration_tests/dist_query/run.sh b/integration_tests/dist_query/run.sh index 766bdab17a..aef2347acb 100755 --- a/integration_tests/dist_query/run.sh +++ b/integration_tests/dist_query/run.sh @@ -48,44 +48,46 @@ set -x # Init trap cleanup EXIT cleanup() { - ls -lha ${LOG_DIR} - ls -lha ${CURR_DIR}/tsbs - ls -lha ${BULK_DATA_DIR} - curl -XPOST "${CERESDB_HTTP_ADDR}/sql" -d 'DROP TABLE `cpu`' + ls -lha ${LOG_DIR} + ls -lha ${CURR_DIR}/tsbs + ls -lha ${BULK_DATA_DIR} } mkdir -p ${LOG_DIR} # Prepare components -## tsbs -if [[ -d ${TSBS_REPO_PATH} && $UPDATE_REPOS_TO_LATEST == 'true' ]]; then +## Tsbs +if [[ -d ${TSBS_REPO_PATH} && $UPDATE_REPOS_TO_LATEST == 'true' && $NO_INIT == 'false' ]]; then echo "Remove old tsbs..." rm -rf ${TSBS_REPO_PATH} fi -if [[ ! -d ${TSBS_REPO_PATH} ]]; then +if [[ ! -d ${TSBS_REPO_PATH} && $NO_INIT == 'false' ]]; then echo "Pull tsbs repo..." git clone -b support-partitioned-table --depth 1 --single-branch https://github.com/Rachelint/tsbs.git fi -## data -if [[ -d ${DATA_REPO_PATH} && $UPDATE_REPOS_TO_LATEST == 'true' ]]; then +## Data +if [[ -d ${DATA_REPO_PATH} && $UPDATE_REPOS_TO_LATEST == 'true' && $NO_INIT == 'false' ]]; then echo "Remove old dist query testing..." rm -rf ${DATA_REPO_PATH} fi -if [[ ! -d ${DATA_REPO_PATH} ]]; then +if [[ ! -d ${DATA_REPO_PATH} && $NO_INIT == 'false' ]]; then echo "Pull old dist query testing repo..." git clone -b main --depth 1 --single-branch https://github.com/CeresDB/dist-query-testing.git fi -## build tsbs bins +## Build tsbs bins cd tsbs go build ./cmd/tsbs_generate_data go build ./cmd/tsbs_load_ceresdb go build ./cmd/tsbs_generate_queries go build ./cmd/tsbs_run_queries_ceresdb +# Clean old table if exist +curl -XPOST "${CERESDB_HTTP_ADDR}/sql" -d 'DROP TABLE IF EXISTS `cpu`' + # Write data to ceresdb -${CURR_DIR}/tsbs/tsbs_load_ceresdb --ceresdb-addr=${CERESDB_ADDR} --file ${DATA_FILE} --batch-size ${WRITE_BATCH_SIZE} --workers ${WRITE_WORKER_NUM} --access-mode proxy --partition-keys hostname | tee ${LOG_DIR}/write.log +${CURR_DIR}/tsbs/tsbs_load_ceresdb --ceresdb-addr=${CERESDB_ADDR} --file ${DATA_FILE} --batch-size ${WRITE_BATCH_SIZE} --workers ${WRITE_WORKER_NUM} --access-mode proxy --partition-keys hostname --update-mode APPEND | tee ${LOG_DIR}/write.log # Run queries against ceresdb # TODO: support more kinds of queries besides 5-8-1.