tmp

.gitignore

@@ -3,3 +3,6 @@ target
 .idea/
 .vscode
 .dir-locals.el
+integration_tests/dist_query/dist-query-testing
+integration_tests/dist_query/tsbs
+integration_tests/dist_query/output

integration_tests/Makefile

@@ -27,6 +27,7 @@ export RUST_BACKTRACE=1
 
 # Whether update related repos
 export UPDATE_REPOS_TO_LATEST ?= true
+export NO_INIT ?= false
 
 clean:
 	rm -rf $(CERESDB_DATA_DIR) $(CERESDB_DATA_DIR_0) $(CERESDB_DATA_DIR_1) $(CERESMETA_DATA_DIR)

integration_tests/dist_query/diff.py

@@ -0,0 +1,23 @@
+import difflib
+
+def main():
+    # args = get_args()
+    # init_before_check = args['init_before_check']
+    # ts = args['timestamp']
+    # test_tables = get_test_tables(args['timestamp'])
+    f = open("./dist-query-testing/tsbs-cpu-only/queries.result")
+    lines1 = f.read().splitlines()
+
+    f = open("./dist-query-testing/tsbs-cpu-only/queries.result.new")
+    lines2 = f.read().splitlines()
+
+    text1_lines = lines1
+    text2_lines = lines2
+
+    d = difflib.Differ()
+    l = list(d.compare(text1_lines, text2_lines))
+
+    print('\n'.join(l))
+
+if __name__ == '__main__':
+    main()

integration_tests/dist_query/run.sh

@@ -48,44 +48,46 @@ set -x
 # Init
 trap cleanup EXIT
 cleanup() {
-  ls -lha ${LOG_DIR}
-  ls -lha ${CURR_DIR}/tsbs
-  ls -lha ${BULK_DATA_DIR}
-  curl -XPOST "${CERESDB_HTTP_ADDR}/sql" -d 'DROP TABLE `cpu`'
+    ls -lha ${LOG_DIR}
+    ls -lha ${CURR_DIR}/tsbs
+    ls -lha ${BULK_DATA_DIR}
 }
 
 mkdir -p ${LOG_DIR}
 
 # Prepare components
-## tsbs
-if [[ -d ${TSBS_REPO_PATH} && $UPDATE_REPOS_TO_LATEST == 'true' ]]; then
+## Tsbs
+if [[ -d ${TSBS_REPO_PATH} && $UPDATE_REPOS_TO_LATEST == 'true' && $NO_INIT == 'false' ]]; then
   echo "Remove old tsbs..."
   rm -rf ${TSBS_REPO_PATH}
 fi
 
-if [[ ! -d ${TSBS_REPO_PATH} ]]; then
+if [[ ! -d ${TSBS_REPO_PATH} && $NO_INIT == 'false' ]]; then
     echo "Pull tsbs repo..."
     git clone -b support-partitioned-table --depth 1 --single-branch https://github.com/Rachelint/tsbs.git
 fi
-## data
-if [[ -d ${DATA_REPO_PATH} && $UPDATE_REPOS_TO_LATEST == 'true' ]]; then
+## Data
+if [[ -d ${DATA_REPO_PATH} && $UPDATE_REPOS_TO_LATEST == 'true' && $NO_INIT == 'false' ]]; then
   echo "Remove old dist query testing..."
   rm -rf ${DATA_REPO_PATH}
 fi
 
-if [[ ! -d ${DATA_REPO_PATH} ]]; then
+if [[ ! -d ${DATA_REPO_PATH} && $NO_INIT == 'false' ]]; then
     echo "Pull old dist query testing repo..."
     git clone -b main --depth 1 --single-branch https://github.com/CeresDB/dist-query-testing.git
 fi
-## build tsbs bins
+## Build tsbs bins
 cd tsbs
 go build ./cmd/tsbs_generate_data
 go build ./cmd/tsbs_load_ceresdb
 go build ./cmd/tsbs_generate_queries
 go build ./cmd/tsbs_run_queries_ceresdb
 
+# Clean old table if exist
+curl -XPOST "${CERESDB_HTTP_ADDR}/sql" -d 'DROP TABLE IF EXISTS `cpu`'
+
 # Write data to ceresdb
-${CURR_DIR}/tsbs/tsbs_load_ceresdb --ceresdb-addr=${CERESDB_ADDR} --file ${DATA_FILE} --batch-size ${WRITE_BATCH_SIZE} --workers ${WRITE_WORKER_NUM}  --access-mode proxy --partition-keys hostname | tee ${LOG_DIR}/write.log
+${CURR_DIR}/tsbs/tsbs_load_ceresdb --ceresdb-addr=${CERESDB_ADDR} --file ${DATA_FILE} --batch-size ${WRITE_BATCH_SIZE} --workers ${WRITE_WORKER_NUM}  --access-mode proxy --partition-keys hostname --update-mode APPEND | tee ${LOG_DIR}/write.log
 
 # Run queries against ceresdb
 # TODO: support more kinds of queries besides 5-8-1.