Merge pull request #186 from XiaoMi/dev

fix #184 and #173

.travis.yml

@@ -29,6 +29,3 @@ script:
   - make docker
   - make cover
   - make test-cli
-
-after_success:
-  - bash <(curl -s https://codecov.io/bash)

CHANGES.md

@@ -1,8 +1,17 @@
 # CHANGELOG
 
-## 2018-12
+## 2019-01
 
 - DOING: english translation
+- add JSONFind function, which support JSON iterate
+- add new test database `world_x`
+- SplitStatement support optimizer hint `/*+xxx */`
+- include [bats](https://github.com/bats-core/bats-core) bash auto test framework
+- fix explain result with multi rows error
+- fix #178 JSON datatype only support utf8mb4
+
+## 2018-12
+
 - replace mysql database driver mymysql with go-sql-driver
 - add new -report-type [ast-json, tiast-json]
 - command line dsn args support '@', '/', ':' in password

Makefile

@@ -187,7 +187,7 @@ release: build
 
 .PHONY: docker
 docker:
-	@echo "$(CGREEN)Build mysql test enviorment ...$(CEND)"
+	@echo "$(CGREEN)Build mysql test environment ...$(CEND)"
 	@docker stop soar-mysql 2>/dev/null || true
 	@docker wait soar-mysql 2>/dev/null >/dev/null || true
 	@echo "docker run --name soar-mysql $(MYSQL_RELEASE):$(MYSQL_VERSION)"
@@ -204,7 +204,7 @@ docker:
 			timeout=`expr $$timeout - 1`; \
 			printf '.' ;  sleep 1 ; \
 		else \
-			echo "." ; echo "mysql test enviorment is ready!" ; break ; \
+			echo "." ; echo "mysql test environment is ready!" ; break ; \
 		fi ; \
 		if [ $$timeout = 0 ] ; then \
 			echo "." ; echo "$(CRED)docker soar-mysql start timeout(180 s)!$(CEND)" ; exit 1 ; \

advisor/heuristic.go

@@ -31,6 +31,7 @@ import (
 	"github.com/percona/go-mysql/query"
 	tidb "github.com/pingcap/parser/ast"
 	"github.com/pingcap/parser/mysql"
+	"github.com/tidwall/gjson"
 	"vitess.io/vitess/go/vt/sqlparser"
 )
 
@@ -1312,37 +1313,16 @@ func (q *Query4Audit) RuleLoadFile() Rule {
 func (q *Query4Audit) RuleMultiCompare() Rule {
 	var rule = q.RuleOK()
 	if q.TiStmt != nil {
-		for _, tiStmt := range q.TiStmt {
-			switch node := tiStmt.(type) {
-			case *tidb.SelectStmt:
-				switch where := node.Where.(type) {
-				case *tidb.BinaryOperationExpr:
-					switch where.L.(type) {
-					case *tidb.BinaryOperationExpr:
-						if where.Op.String() == "eq" {
-							rule = HeuristicRules["RES.009"]
-						}
-					}
-				}
-			case *tidb.UpdateStmt:
-				switch where := node.Where.(type) {
-				case *tidb.BinaryOperationExpr:
-					switch where.L.(type) {
-					case *tidb.BinaryOperationExpr:
-						if where.Op.String() == "eq" {
-							rule = HeuristicRules["RES.009"]
-						}
-					}
-				}
-			case *tidb.DeleteStmt:
-				switch where := node.Where.(type) {
-				case *tidb.BinaryOperationExpr:
-					switch where.L.(type) {
-					case *tidb.BinaryOperationExpr:
-						if where.Op.String() == "eq" {
-							rule = HeuristicRules["RES.009"]
-						}
-					}
+		json := ast.StmtNode2JSON(q.Query, "", "")
+		whereJSON := common.JSONFind(json, "Where")
+		for _, where := range whereJSON {
+			conds := []string{where}
+			conds = append(conds, common.JSONFind(where, "L")...)
+			conds = append(conds, common.JSONFind(where, "R")...)
+			for _, cond := range conds {
+				if gjson.Get(cond, "Op").Int() == 7 && gjson.Get(cond, "L.Op").Int() == 7 {
+					rule = HeuristicRules["RES.009"]
+					return rule
 				}
 			}
 		}

advisor/heuristic_test.go

@@ -946,6 +946,9 @@ func TestRuleMultiCompare(t *testing.T) {
 	sqls := [][]string{
 		{
 			"SELECT * FROM tbl WHERE col = col = 'abc'",
+			"SELECT * FROM tbl WHERE col = 'def' and col = col = 'abc'",
+			"SELECT * FROM tbl WHERE col = 'def' or col = col = 'abc'",
+			"SELECT * FROM tbl WHERE col = col = 'abc' and col = 'def'",
 			"UPDATE tbl set col = 1 WHERE col = col = 'abc'",
 			"DELETE FROM tbl WHERE col = col = 'abc'",
 		},

ast/tidb.go

@@ -17,14 +17,14 @@
 package ast
 
 import (
-	"encoding/json"
-
 	"github.com/XiaoMi/soar/common"
 
 	"github.com/kr/pretty"
 	"github.com/pingcap/parser"
 	"github.com/pingcap/parser/ast"
 
+	json "github.com/CorgiMan/json2"
+
 	// for pingcap parser
 	_ "github.com/pingcap/tidb/types/parser_driver"
 )

cmd/soar/tool.go

@@ -210,6 +210,7 @@ func initQuery(query string) string {
 		if err != nil {
 			common.Log.Critical("ioutil.ReadAll Error: %v", err)
 		}
+		common.Log.Debug("initQuery get query from os.Stdin")
 		return string(data)
 	}
 
@@ -219,6 +220,7 @@ func initQuery(query string) string {
 		if err != nil {
 			common.Log.Critical("ioutil.ReadFile Error: %v", err)
 		}
+		common.Log.Debug("initQuery get query from file: %s", query)
 		return string(data)
 	}
 

common/config.go

@@ -93,7 +93,7 @@ type Configuration struct {
 	MaxDistinctCount     int      `yaml:"max-distinct-count"`        // 单条 SQL 中 Distinct 的最大数量
 	MaxIdxColsCount      int      `yaml:"max-index-cols-count"`      // 复合索引中包含列的最大数量
 	MaxTextColsCount     int      `yaml:"max-text-cols-count"`       // 表中含有的 text/blob 列的最大数量
-	MaxTotalRows         int64    `yaml:"max-total-rows"`            // 计算散粒度时，当数据行数大于 MaxTotalRows 即开启数据库保护模式，散粒度返回结果可信度下降
+	MaxTotalRows         uint64   `yaml:"max-total-rows"`            // 计算散粒度时，当数据行数大于 MaxTotalRows 即开启数据库保护模式，散粒度返回结果可信度下降
 	MaxQueryCost         int64    `yaml:"max-query-cost"`            // last_query_cost 超过该值时将给予警告
 	SpaghettiQueryLength int      `yaml:"spaghetti-query-length"`    // SQL最大长度警告，超过该长度会给警告
 	AllowDropIndex       bool     `yaml:"allow-drop-index"`          // 允许输出删除重复索引的建议
@@ -426,7 +426,7 @@ func parseDSN(odbc string, d *Dsn) *Dsn {
 func ParseDSN(odbc string, d *Dsn) *Dsn {
 	cfg, err := mysql.ParseDSN(odbc)
 	if err != nil {
-		Log.Warn("go-sql-driver/mysql.ParseDSN Error: %s, DSN: %s, try to use old version parseDSN", err.Error(), odbc)
+		Log.Debug("go-sql-driver/mysql.ParseDSN Error: %s, DSN: %s, try to use old version parseDSN", err.Error(), odbc)
 		return parseDSN(odbc, d)
 	}
 	return newDSN(cfg)
@@ -596,7 +596,7 @@ func readCmdFlags() error {
 	maxDistinctCount := flag.Int("max-distinct-count", Config.MaxDistinctCount, "MaxDistinctCount, 单条 SQL 中 Distinct 的最大数量")
 	maxIdxColsCount := flag.Int("max-index-cols-count", Config.MaxIdxColsCount, "MaxIdxColsCount, 复合索引中包含列的最大数量")
 	maxTextColsCount := flag.Int("max-text-cols-count", Config.MaxTextColsCount, "MaxTextColsCount, 表中含有的 text/blob 列的最大数量")
-	maxTotalRows := flag.Int64("max-total-rows", Config.MaxTotalRows, "MaxTotalRows, 计算散粒度时，当数据行数大于MaxTotalRows即开启数据库保护模式，不计算散粒度")
+	maxTotalRows := flag.Uint64("max-total-rows", Config.MaxTotalRows, "MaxTotalRows, 计算散粒度时，当数据行数大于MaxTotalRows即开启数据库保护模式，不计算散粒度")
 	maxQueryCost := flag.Int64("max-query-cost", Config.MaxQueryCost, "MaxQueryCost, last_query_cost 超过该值时将给予警告")
 	spaghettiQueryLength := flag.Int("spaghetti-query-length", Config.SpaghettiQueryLength, "SpaghettiQueryLength, SQL最大长度警告，超过该长度会给警告")
 	allowDropIdx := flag.Bool("allow-drop-index", Config.AllowDropIndex, "AllowDropIndex, 允许输出删除重复索引的建议")

common/testdata/TestJSONFind.golden

@@ -0,0 +1,8 @@
+[McLaughlin Hunter Harold]
+[{
+      "title": "Sample Konfabulator Widget",
+      "name": "main_window",
+      "width": 500,
+      "height": 500
+    }]
+[    binary    binary  utf8mb4_bin  ]

common/tricks.go

@@ -25,6 +25,8 @@ import (
 	"path/filepath"
 	"reflect"
 	"sort"
+
+	"github.com/tidwall/gjson"
 )
 
 // GoldenDiff 从 gofmt 学来的测试方法
@@ -104,3 +106,35 @@ func SortedKey(m interface{}) []string {
 	sort.Strings(keys)
 	return keys
 }
+
+// jsonFind internal function
+func jsonFind(json string, name string, find *[]string) (next []string) {
+	res := gjson.Parse(json)
+	res.ForEach(func(key, value gjson.Result) bool {
+		if key.String() == name {
+			*find = append(*find, value.String())
+		} else {
+			switch value.Type {
+			case gjson.Number, gjson.True, gjson.False, gjson.Null:
+			default:
+				next = append(next, value.String())
+			}
+		}
+		return true // keep iterating
+	})
+	return next
+}
+
+// JSONFind iterate find name in json
+func JSONFind(json string, name string) []string {
+	var find []string
+	next := []string{json}
+	for len(next) > 0 {
+		var tmpNext []string
+		for _, subJSON := range next {
+			tmpNext = append(tmpNext, jsonFind(subJSON, name, &find)...)
+		}
+		next = tmpNext
+	}
+	return find
+}