Add integration test

.github/workflows/pr-build.yaml

@@ -4,7 +4,7 @@ on: pull_request
 
 jobs:
   backend:
-    name: backend
+    name: Backend
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -34,7 +34,7 @@ jobs:
           git diff --exit-code
 
   frontend:
-    name: frontend
+    name: Frontend
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code

.github/workflows/e2e-test.yaml → .github/workflows/test.yaml

@@ -7,8 +7,51 @@ on:
       - release
 
 jobs:
+  backend_ut:
+    name: Backend Unit Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - uses: actions/setup-go@v1
+        with:
+          go-version: "1.13.5"
+      - name: Load go module cache
+        uses: actions/cache@v1
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - name: Run unit test
+        run: |
+          make unit_test
+  backend_integration:
+    name: Backend Integration Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    strategy:
+      matrix:
+        tidb_version: [latest, v4.0.1]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - uses: actions/setup-go@v1
+        with:
+          go-version: "1.13.5"
+      - name: Load go module cache
+        uses: actions/cache@v1
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - name: Run integration test
+        run: |
+          make integration_test TIDB_VERSION=${{ matrix.tidb_version }}
   e2e_test:
-    name: UI
+    name: E2E
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:

.gitignore

@@ -13,6 +13,7 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+coverage/
 
 # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839

Makefile

@@ -1,4 +1,4 @@
-.PHONY: install_tools lint dev yarn_dependencies go_generate ui server run
+.PHONY: install_tools lint dev yarn_dependencies go_generate ui server run test unit_test integration_test
 
 DASHBOARD_PKG := github.com/pingcap/tidb-dashboard
 
@@ -16,14 +16,29 @@ LDFLAGS += -X "$(DASHBOARD_PKG)/pkg/utils/version.PDVersion=N/A"
 LDFLAGS += -X "$(DASHBOARD_PKG)/pkg/utils/version.BuildTime=$(shell date -u '+%Y-%m-%d %I:%M:%S')"
 LDFLAGS += -X "$(DASHBOARD_PKG)/pkg/utils/version.BuildGitHash=$(shell git rev-parse HEAD)"
 
+TIDB_VERSION ?= latest
+
 default: server
 
+clean:
+	rm -rf ./coverage
+
 install_tools:
 	scripts/install_go_tools.sh
 
 lint:
 	scripts/lint.sh
 
+test: clean unit_test integration_test
+
+unit_test:
+	@mkdir -p ./coverage
+	GO111MODULE=on go test -v -cover -coverprofile=coverage/ut.out ./pkg/... ./util/...
+
+integration_test:
+	@mkdir -p ./coverage
+	@TIDB_VERSION=${TIDB_VERSION} tests/run.sh
+
 dev: lint default
 
 yarn_dependencies: install_tools

go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69
 	github.com/henrylee2cn/ameda v1.4.10
 	github.com/jarcoal/httpmock v1.0.8
-	github.com/joho/godotenv v1.3.0
+	github.com/joho/godotenv v1.4.0
 	github.com/joomcode/errorx v1.0.1
 	github.com/minio/sio v0.3.0
 	github.com/oleiade/reflections v1.0.1
@@ -33,8 +33,9 @@ require (
 	github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c
 	github.com/pingcap/log v0.0.0-20210906054005-afc726e70354
 	github.com/rs/cors v1.7.0
+	github.com/shhdgit/testfixtures/v3 v3.6.2-0.20211219171712-c4f264d673d3
 	github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0
-	github.com/spf13/pflag v1.0.1
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.7.0
 	github.com/swaggo/http-swagger v0.0.0-20200308142732-58ac5e232fba
 	github.com/swaggo/swag v1.6.6-0.20200529100950-7c765ddd0476

pkg/apiserver/apiserver.go

@@ -112,7 +112,7 @@ func (s *Service) Start(ctx context.Context) error {
 			tidb.NewTiDBClient,
 			tikv.NewTiKVClient,
 			tiflash.NewTiFlashClient,
-			utils.NewSysSchema,
+			utils.ProvideSysSchema,
 			info.NewService,
 			clusterinfo.NewService,
 			logsearch.NewService,

pkg/apiserver/diagnose/report_test.go

@@ -3,13 +3,9 @@
 package diagnose
 
 import (
-	"fmt"
-	"strings"
 	"testing"
 
 	. "github.com/pingcap/check"
-	"gorm.io/driver/mysql"
-	"gorm.io/gorm"
 )
 
 func TestT(t *testing.T) {
@@ -21,98 +17,6 @@ var _ = Suite(&testReportSuite{})
 
 type testReportSuite struct{}
 
-//func (t *testReportSuite) TestReport(c *C) {
-//	cli, err := gorm.Open("mysql", "root:@tcp(172.16.5.40:4009)/test?charset=utf8&parseTime=True&loc=Local")
-//	c.Assert(err, IsNil)
-//	defer cli.Close()
-//
-//	startTime := "2020-03-03 17:18:00"
-//	endTime := "2020-03-03 17:21:00"
-//
-//	tables := GetReportTablesForDisplay(startTime, endTime, cli)
-//	for _, tbl := range tables {
-//		printRows(tbl)
-//	}
-//}
-
-func (t *testReportSuite) TestGetTable(c *C) {
-	cli, err := gorm.Open(mysql.Open("root:@tcp(172.16.5.40:4009)/test?charset=utf8&parseTime=True&loc=Local"))
-	c.Assert(err, IsNil)
-
-	startTime := "2020-03-25 23:00:00"
-	endTime := "2020-03-25 23:05:00"
-
-	var table TableDef
-	table, err = GetLoadTable(startTime, endTime, cli)
-	c.Assert(err, IsNil)
-	printRows(&table)
-}
-
-func (t *testReportSuite) TestGetCompareTable(c *C) {
-	cli, err := gorm.Open(mysql.Open("root:@tcp(172.16.5.40:4009)/test?charset=utf8&parseTime=True&loc=Local"))
-	c.Assert(err, IsNil)
-
-	//startTime1 := "2020-03-12 20:17:00"
-	//endTime1 := "2020-03-12 20:39:00"
-	//
-	//startTime2 := "2020-03-12 20:17:00"
-	//endTime2 := "2020-03-12 20:39:00"
-
-	startTime1 := "2020-04-02 12:13:00"
-	endTime1 := "2020-04-02 12:15:00"
-
-	startTime2 := "2020-04-02 12:15:00"
-	endTime2 := "2020-04-02 12:17:00"
-
-	tables := GetCompareReportTablesForDisplay(startTime1, endTime1, startTime2, endTime2, cli, nil, "ID")
-	for _, tbl := range tables {
-		printRows(tbl)
-	}
-}
-
-func (t *testReportSuite) TestInspection(c *C) {
-	cli, err := gorm.Open(mysql.Open("root:@tcp(172.16.5.40:4009)/test?charset=utf8&parseTime=True&loc=Local"))
-	c.Assert(err, IsNil)
-
-	// affect by big query join
-	startTime1 := "2020-03-08 01:36:00"
-	endTime1 := "2020-03-08 01:41:00"
-
-	startTime2 := "2020-03-08 01:46:30"
-	endTime2 := "2020-03-08 01:51:30"
-
-	// affect by big write with conflict
-	//startTime1 := "2020-03-10 12:35:00"
-	//endTime1 := "2020-03-10 12:39:00"
-	//
-	//startTime2 := "2020-03-10 12:41:00"
-	//endTime2 := "2020-03-10 12:45:00"
-
-	// affect by big write without conflict
-	//startTime1 := "	2020-03-10 13:20:00"
-	//endTime1 := "	2020-03-10 13:23:00"
-	//
-	//startTime2 := "2020-03-10 13:24:00"
-	//endTime2 := "2020-03-10 13:27:00"
-
-	// diagnose for server down
-	// startTime1 := "2020-03-09 20:35:00"
-	// endTime1 := "2020-03-09 21:20:00"
-	// startTime2 := "2020-03-08 20:35:00"
-	// endTime2 := "2020-03-09 21:20:00"
-
-	// diagnose for disk slow , need more disk metric.
-	//startTime1 := "2020-03-10 12:48:00"
-	//endTime1 := "2020-03-10 12:50:00"
-	//
-	//startTime2 := "2020-03-10 12:54:30"
-	//endTime2 := "2020-03-10 12:56:30"
-
-	table, errRow := CompareDiagnose(startTime1, endTime1, startTime2, endTime2, cli)
-	c.Assert(errRow, IsNil)
-	printRows(&table)
-}
-
 func (t *testReportSuite) TestCompareTable(c *C) {
 	table1 := TableDef{
 		Category:       []string{"header"},
@@ -222,47 +126,3 @@ func (t *testReportSuite) TestRoundFloatString(c *C) {
 		c.Assert(result, Equals, cas.out)
 	}
 }
-
-func printRows(t *TableDef) {
-	if t == nil {
-		fmt.Println("table is nil")
-		return
-	}
-
-	fmt.Println(strings.Join(t.Category, " - "))
-	fmt.Println(t.Title)
-	fmt.Println(t.Comment)
-	if len(t.Rows) == 0 {
-		fmt.Println("table rows is 0")
-		return
-	}
-
-	fieldLen := t.ColumnWidth()
-	// fmt.Println(fieldLen)
-	printLine := func(values []string, comment string) {
-		line := ""
-		for i, s := range values {
-			for k := len(s); k < fieldLen[i]; k++ {
-				s += " "
-			}
-			if i > 0 {
-				line += "    |    "
-			}
-			line += s
-		}
-		if len(comment) != 0 {
-			line = line + "    |    " + comment
-		}
-		fmt.Println(line)
-	}
-
-	printLine(t.Column, "")
-
-	for _, row := range t.Rows {
-		printLine(row.Values, row.Comment)
-		for i := range row.SubValues {
-			printLine(row.SubValues[i], "")
-		}
-	}
-	fmt.Println("")
-}

pkg/apiserver/slowquery/queries.go

@@ -5,11 +5,14 @@ package slowquery
 import (
 	"strings"
 
+	"github.com/thoas/go-funk"
 	"gorm.io/gorm"
+
+	"github.com/pingcap/tidb-dashboard/pkg/utils"
 )
 
 const (
-	slowQueryTable = "INFORMATION_SCHEMA.CLUSTER_SLOW_QUERY"
+	SlowQueryTable = "INFORMATION_SCHEMA.CLUSTER_SLOW_QUERY"
 )
 
 type GetListRequest struct {
@@ -35,26 +38,29 @@ type GetDetailRequest struct {
 	ConnectID string `json:"connect_id" form:"connect_id"`
 }
 
-func (s *Service) querySlowLogList(db *gorm.DB, req *GetListRequest) ([]Model, error) {
-	tableColumns, err := s.params.SysSchema.GetTableColumnNames(db, slowQueryTable)
+func QuerySlowLogList(req *GetListRequest, sysSchema *utils.SysSchema, db *gorm.DB) ([]Model, error) {
+	slowQueryColumns, err := sysSchema.GetTableColumnNames(db, SlowQueryTable)
 	if err != nil {
 		return nil, err
 	}
 
 	reqFields := strings.Split(req.Fields, ",")
-	selectStmt, err := s.genSelectStmt(tableColumns, reqFields)
+	selectStmt, err := genSelectStmt(slowQueryColumns, reqFields)
 	if err != nil {
 		return nil, err
 	}
 
 	tx := db.
-		Table(slowQueryTable).
-		Select(selectStmt).
-		Where("Time BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?)", req.BeginTime, req.EndTime)
+		Select(selectStmt)
+
+	if req.BeginTime != 0 && req.EndTime != 0 {
+		tx = tx.Where("Time BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?)", req.BeginTime, req.EndTime)
+	}
 
-	if req.Limit > 0 {
-		tx = tx.Limit(req.Limit)
+	if req.Limit <= 0 {
+		req.Limit = 100
 	}
+	tx = tx.Limit(req.Limit)
 
 	if req.Text != "" {
 		lowerStr := strings.ToLower(req.Text)
@@ -78,8 +84,7 @@ func (s *Service) querySlowLogList(db *gorm.DB, req *GetListRequest) ([]Model, e
 	if req.OrderBy == "" {
 		req.OrderBy = "timestamp"
 	}
-
-	orderStmt, err := s.genOrderStmt(tableColumns, req.OrderBy, req.IsDesc)
+	orderStmt, err := genOrderStmt(slowQueryColumns, req.OrderBy, req.IsDesc)
 	if err != nil {
 		return nil, err
 	}
@@ -102,10 +107,9 @@ func (s *Service) querySlowLogList(db *gorm.DB, req *GetListRequest) ([]Model, e
 	return results, nil
 }
 
-func (s *Service) querySlowLogDetail(db *gorm.DB, req *GetDetailRequest) (*Model, error) {
+func QuerySlowLogDetail(req *GetDetailRequest, db *gorm.DB) (*Model, error) {
 	var result Model
 	err := db.
-		Table(slowQueryTable).
 		Select("*, (UNIX_TIMESTAMP(Time) + 0E0) AS timestamp").
 		Where("Digest = ?", req.Digest).
 		Where("Time = FROM_UNIXTIME(?)", req.Timestamp).
@@ -116,3 +120,11 @@ func (s *Service) querySlowLogDetail(db *gorm.DB, req *GetDetailRequest) (*Model
 	}
 	return &result, nil
 }
+
+func QueryTableColumns(sysSchema *utils.SysSchema, db *gorm.DB) ([]string, error) {
+	cs, err := sysSchema.GetTableColumnNames(db, SlowQueryTable)
+	if err != nil {
+		return nil, err
+	}
+	return funk.UniqString(append(cs, getVirtualFields()...)), nil
+}