tests: add e2e test (pingcap#34)

* tests: add e2e test

* fix ci

* Update tests/_utils/check_sync_diff

Co-Authored-By: kennytm <kennytm@gmail.com>

* fix ci

* fix ci

* remove pd-server

* fix ci

* fix ci

* fix ci

* fix ci

* debug ci

* debug ci

* fix ci

* leave TODO

* Update tests/_utils/check_sync_diff

Co-Authored-By: Ian <ArGregoryIan@gmail.com>

Co-authored-by: kennytm <kennytm@gmail.com>
Co-authored-by: Ian <ArGregoryIan@gmail.com>

.gitignore

@@ -3,3 +3,4 @@ bin/
 coverage.txt
 .idea
 var
+fix.sql

.travis.yml

@@ -4,6 +4,9 @@ dist: bionic
 go:
   - 1.13.x
 
+install:
+  - sh install.sh
+
 script:
   - make test WITH_RACE=1
   - make integration_test

install.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+set -e
+
+# FIXME: change to latest version after lightning fix issue
+# https://github.com/pingcap/tidb-lightning/issues/277
+TAG="v3.1.0-beta.1"
+pwd=$(pwd)
+
+mkdir bin/
+
+# download lightning and sync_diff_inspector
+wget http://download.pingcap.org/tidb-toolkit-$TAG-linux-amd64.tar.gz -O tools.tar.gz
+tar -xzvf tools.tar.gz
+mv tidb-toolkit-$TAG-linux-amd64/bin/* bin/
+
+# download tidb-server
+git clone -b $TAG https://github.com/pingcap/tidb
+cd $pwd/tidb && make
+cd $pwd
+mv tidb/bin/tidb-server bin/

tests/_utils/check_sync_diff

@@ -0,0 +1,30 @@
+#!/bin/bash
+# parameter 1: config file for sync_diff_inspector
+# parameter 2: max check times
+
+conf=$1
+check_time=${2-10}
+
+LOG=$DUMPLING_OUTPUT_DIR/sync_diff_inspector.log
+
+i=0
+while [ $i -lt $check_time ]
+do
+    bin/sync_diff_inspector --config=$conf >> $LOG 2>&1
+    ret=$?
+    if [ "$ret" == 0 ]; then
+        echo "check diff successfully"
+        break
+    fi
+    ((i++))
+    echo "check diff failed $i-th time, retry later"
+    sleep 2
+done
+
+if [ $i -ge $check_time ]; then
+    echo "check data failed, some data are different!!"
+    # show \n and other blanks
+    printf "$(cat $LOG)\n"
+    exit 1
+fi
+cd $PWD

tests/_utils/run_lightning

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -eu
+
+echo "[$(date)] Executing bin/tidb-lightning..."
+
+conf=$1
+
+bin/tidb-lightning -c $1
+
+echo "[$(date)] Executed bin/tidb-lightning"

tests/_utils/run_services

@@ -0,0 +1,56 @@
+#!/bin/sh
+
+set -eu
+
+PD_ADDR="127.0.0.1:2379"
+TIDB_IP="127.0.0.1"
+TIDB_PORT="4000"
+TIDB_ADDR="127.0.0.1:4000"
+TIDB_STATUS_ADDR="127.0.0.1:10080"
+
+stop_services() {
+    killall -9 tidb-server || true
+
+    find "$DUMPLING_TEST_DIR" -maxdepth 1 -not -path "$DUMPLING_TEST_DIR" -not -name "*.log" | xargs rm -r || true
+}
+
+start_services() {
+    stop_services
+    echo "Ensure mysql can connected..."
+
+cat > "$DUMPLING_TEST_DIR/mysql.cnf" <<EOF
+[client]
+user = ${DUMPLING_TEST_USER}
+host = ${DUMPLING_TEST_HOST}
+port = ${DUMPLING_TEST_PORT}
+password = ${DUMPLING_TEST_PASSWORD}
+default-character-set = utf8mb4
+EOF
+    i=0
+    while ! run_sql 'select 0 limit 0' > /dev/null; do
+       i=$((i+1))
+       if [ "$i" -gt 10 ]; then
+          echo 'Failed to ping MySQL Server'
+          exit 1
+       fi
+       sleep 3
+    done
+
+    bin/tidb-server \
+        -P 4000 \
+        --status 10080 \
+        --store mocktikv \
+        --log-file "$DUMPLING_TEST_DIR/tidb.log" &
+
+    echo "Verifying TiDB is started..."
+    i=0
+    while ! curl -o /dev/null -sf "http://$TIDB_IP:10080/status"; do
+        i=$((i+1))
+        if [ "$i" -gt 10 ]; then
+            echo 'Failed to start TiDB'
+            exit 1
+        fi
+        sleep 3
+    done
+}
+

tests/e2e/conf/diff_config.toml

@@ -0,0 +1,42 @@
+# diff Configuration.
+
+log-level = "info"
+
+chunk-size = 1000
+
+check-thread-count = 4
+
+sample-percent = 100
+
+use-rowid = false
+
+use-checksum = true
+
+fix-sql-file = "fix.sql"
+
+# tables need to check.
+[[check-tables]]
+schema = "e2e"
+tables = ["~t.*"]
+
+[[table-config]]
+schema = "e2e"
+table = "t"
+
+[[table-config.source-tables]]
+instance-id = "source-1"
+schema = "e2e"
+table  = "t"
+
+[[source-db]]
+host = "127.0.0.1"
+port = 3306
+user = "root"
+password = ""
+instance-id = "source-1"
+
+[target-db]
+host = "127.0.0.1"
+port = 4000
+user = "root"
+password = ""

tests/e2e/conf/lightning.toml

@@ -0,0 +1,20 @@
+### tidb-lightning config
+
+[lightning]
+server-mode = false
+level = "error"
+check-requirements = false
+
+[tikv-importer]
+backend="tidb"
+on-duplicate = "error"
+
+[mydumper]
+data-source-dir = "/tmp/dumpling_test_result/sql_res.e2e"
+
+[tidb]
+host = "127.0.0.1"
+port = 4000
+user = "root"
+password = ""
+status-port = 10080

tests/e2e/run.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+set -eu
+cur=$(cd `dirname $0`; pwd)
+
+DB_NAME="e2e"
+TABLE_NAME="t"
+
+# drop database on tidb
+export DUMPLING_TEST_PORT=4000
+run_sql "drop database if exists $DB_NAME;"
+
+# drop database on mysql
+export DUMPLING_TEST_PORT=3306
+run_sql "drop database if exists $DB_NAME;"
+
+# build data on mysql
+run_sql "create database $DB_NAME;"
+run_sql "create table $DB_NAME.$TABLE_NAME (a int(255));"
+
+# insert 100 records
+run_sql "insert into $DB_NAME.$TABLE_NAME values $(seq -s, 100 | sed 's/,*$//g' | sed "s/[0-9]*/('1')/g");"
+
+# dumping
+export DUMPLING_TEST_DATABASE=$DB_NAME
+run_dumpling
+
+cat "$cur/conf/lightning.toml"
+# use lightning import data to tidb
+run_lightning $cur/conf/lightning.toml
+
+# check mysql and tidb data
+check_sync_diff $cur/conf/diff_config.toml
+
+

tests/file_size/run.sh

@@ -10,7 +10,7 @@ run_sql "create table t (a varchar(255))"
 chars_20="1111_0000_1111_0000_"
 
 # insert 100 records, each occupies 20 bytes
-run_sql "insert into t values $(seq -s, 100 | sed "s/[0-9]\+/('$chars_20')/g")"
+run_sql "insert into t values $(seq -s, 100 | sed 's/,*$//g' | sed "s/[0-9]*/('$chars_20')/g");"
 
 # dumping with file size = 200 bytes
 run_dumpling -F 200

tests/run.sh

@@ -20,29 +20,16 @@ set -eu
 
 mkdir -p "$DUMPLING_TEST_DIR"
 PATH="tests/_utils:$PATH"
+. "tests/_utils/run_services"
 
-cat > "$DUMPLING_TEST_DIR/mysql.cnf" <<EOF
-[client]
-user = ${DUMPLING_TEST_USER}
-host = ${DUMPLING_TEST_HOST}
-port = ${DUMPLING_TEST_PORT}
-password = ${DUMPLING_TEST_PASSWORD}
-default-character-set = utf8mb4
-EOF
-
-test_connection() {
-    i=0
-    while ! run_sql 'select 0 limit 0' > /dev/null; do
-        i=$((i+1))
-        if [ "$i" -gt 5 ]; then
-            echo 'Failed to ping MySQL Server'
-            exit 1
-        fi
-        sleep 3
-    done
-}
-
-test_connection
+
+file_should_exist bin/tidb-server
+file_should_exist bin/tidb-lightning
+file_should_exist bin/dumpling
+file_should_exist bin/sync_diff_inspector
+
+trap stop_services EXIT
+start_services
 
 for script in tests/*/run.sh; do
     echo "****************** Running test $script..."
@@ -52,11 +39,12 @@ for script in tests/*/run.sh; do
     DUMPLING_OUTPUT_DIR="$DUMPLING_TEST_DIR"/sql_res."$TEST_NAME"
     export DUMPLING_OUTPUT_DIR
 
-    echo "Cleaning up test output dir: $DUMPLING_OUTPUT_DIR"
-    rm "$DUMPLING_OUTPUT_DIR" -rf
-
     PATH="tests/_utils:$PATH" \
     sh "$script"
+
+    echo "Cleaning up test output dir: $DUMPLING_OUTPUT_DIR"
+    rm -rf "$DUMPLING_OUTPUT_DIR"
+
 done
 
 echo "Passed integration tests."

tests/views/run.sh

@@ -9,7 +9,7 @@ export DUMPLING_TEST_DATABASE="views"
 run_sql "create table t (a bigint, b varchar(255))"
 run_sql "create definer = 'root'@'localhost' view v as select * from t;"
 # insert 20 records to `t`.
-run_sql "insert into t values $(seq -s, 0 19 | sed 's/[0-9]\+/(\0,"\0")/g')"
+run_sql "insert into t values $(seq -s, 20 | sed 's/,*$//g' | sed 's/[0-9]*/(\0,"\0")/g')"
 
 run_dumpling --no-views
 file_not_exist "$DUMPLING_OUTPUT_DIR/views.v-schema.sql"