support pushing bitwise operators and LogicXor down to kv (#1846)

distsql/xeval/eval.go

@@ -60,12 +60,16 @@ func (e *Evaluator) Eval(expr *tipb.Expr) (types.Datum, error) {
 		tipb.ExprType_NullEQ, tipb.ExprType_Like, tipb.ExprType_In:
 		return e.evalCompareOps(expr)
 	// logic operator
-	case tipb.ExprType_And, tipb.ExprType_Or, tipb.ExprType_Not:
+	case tipb.ExprType_And, tipb.ExprType_Or, tipb.ExprType_Xor, tipb.ExprType_Not:
 		return e.evalLogicOps(expr)
 	// arithmetic operator
 	case tipb.ExprType_Plus, tipb.ExprType_Div, tipb.ExprType_Minus,
 		tipb.ExprType_Mul, tipb.ExprType_IntDiv, tipb.ExprType_Mod:
 		return e.evalArithmeticOps(expr)
+	// bit operator
+	case tipb.ExprType_BitAnd, tipb.ExprType_BitOr, tipb.ExprType_BitNeg,
+		tipb.ExprType_BitXor, tipb.ExprType_LeftShift, tipb.ExprType_RighShift:
+		return e.evalBitOps(expr)
 	case tipb.ExprType_Case:
 		return e.evalCaseWhen(expr)
 	case tipb.ExprType_Coalesce:
@@ -76,7 +80,7 @@ func (e *Evaluator) Eval(expr *tipb.Expr) (types.Datum, error) {
 
 func (e *Evaluator) evalTwoChildren(expr *tipb.Expr) (left, right types.Datum, err error) {
 	if len(expr.Children) != 2 {
-		err = ErrInvalid.Gen("need 2 operands but got %d", len(expr.Children))
+		err = ErrInvalid.Gen("%s need 2 operands but got %d", tipb.ExprType_name[int32(expr.GetTp())], len(expr.Children))
 		return
 	}
 	left, err = e.Eval(expr.Children[0])

distsql/xeval/eval_bit_ops.go

@@ -0,0 +1,80 @@
+// Copyright 2016 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package xeval
+
+import (
+	"github.com/juju/errors"
+	"github.com/pingcap/tidb/util/types"
+	"github.com/pingcap/tipb/go-tipb"
+)
+
+func (e *Evaluator) evalBitOps(expr *tipb.Expr) (types.Datum, error) {
+	var result types.Datum
+	if expr.GetTp() == tipb.ExprType_BitNeg {
+		if len(expr.Children) != 1 {
+			err := ErrInvalid.Gen("BitNeg(~) need 1 operand but got %d", len(expr.Children))
+			return result, err
+		}
+		operand, err := e.Eval(expr.Children[0])
+		if err != nil {
+			return types.Datum{}, errors.Trace(err)
+		}
+		a, err := types.CoerceArithmetic(operand)
+		if err != nil {
+			return result, errors.Trace(err)
+		}
+		return types.ComputeBitNeg(a)
+	}
+	left, right, err := e.evalTwoChildren(expr)
+	if err != nil {
+		return result, errors.Trace(err)
+	}
+	return ComputeBit(expr.GetTp(), left, right)
+}
+
+// ComputeBit computes the bitwise operation on two datums.
+func ComputeBit(op tipb.ExprType, left, right types.Datum) (types.Datum, error) {
+	var result types.Datum
+	a, err := types.CoerceArithmetic(left)
+	if err != nil {
+		return result, errors.Trace(err)
+	}
+
+	b, err := types.CoerceArithmetic(right)
+	if err != nil {
+		return result, errors.Trace(err)
+	}
+	a, b, err = types.CoerceDatum(a, b)
+	if err != nil {
+		return result, errors.Trace(err)
+	}
+	if a.IsNull() || b.IsNull() {
+		return result, nil
+	}
+
+	switch op {
+	case tipb.ExprType_BitAnd:
+		return types.ComputeBitAnd(a, b)
+	case tipb.ExprType_BitOr:
+		return types.ComputeBitOr(a, b)
+	case tipb.ExprType_BitXor:
+		return types.ComputeBitXor(a, b)
+	case tipb.ExprType_LeftShift:
+		return types.ComputeLeftShift(a, b)
+	case tipb.ExprType_RighShift:
+		return types.ComputeRightShift(a, b)
+	default:
+		return result, errors.Errorf("Unknown binop type: %v", op)
+	}
+}

distsql/xeval/eval_compare_ops.go

@@ -232,7 +232,7 @@ func containsAlphabet(s string) bool {
 
 func (e *Evaluator) evalIn(expr *tipb.Expr) (types.Datum, error) {
 	if len(expr.Children) != 2 {
-		return types.Datum{}, ErrInvalid.Gen("IN need 2 operand, got %d", len(expr.Children))
+		return types.Datum{}, ErrInvalid.Gen("IN need 2 operands, got %d", len(expr.Children))
 	}
 	target, err := e.Eval(expr.Children[0])
 	if err != nil {
@@ -243,7 +243,7 @@ func (e *Evaluator) evalIn(expr *tipb.Expr) (types.Datum, error) {
 	}
 	valueListExpr := expr.Children[1]
 	if valueListExpr.GetTp() != tipb.ExprType_ValueList {
-		return types.Datum{}, ErrInvalid.Gen("the second children should be value list type")
+		return types.Datum{}, ErrInvalid.Gen("the second child should be value list type")
 	}
 	decoded, err := e.decodeValueList(valueListExpr)
 	if err != nil {

distsql/xeval/eval_logic_ops.go

@@ -33,6 +33,8 @@ func (e *Evaluator) evalLogicOps(expr *tipb.Expr) (types.Datum, error) {
 		return e.evalAnd(leftBool, rightBool)
 	case tipb.ExprType_Or:
 		return e.evalOr(leftBool, rightBool)
+	case tipb.ExprType_Xor:
+		return e.evalXor(leftBool, rightBool)
 	default:
 		return types.Datum{}, errors.Errorf("Unknown binop type: %v", op)
 	}
@@ -68,6 +70,21 @@ func (e *Evaluator) evalOr(leftBool, rightBool int64) (types.Datum, error) {
 	return d, nil
 }
 
+// evalXor computes result of (X XOR Y).
+func (e *Evaluator) evalXor(leftBool, rightBool int64) (types.Datum, error) {
+	var d types.Datum
+	if leftBool == compareResultNull || rightBool == compareResultNull {
+		d.SetNull()
+		return d, nil
+	}
+	if leftBool == rightBool {
+		d.SetInt64(0)
+		return d, nil
+	}
+	d.SetInt64(1)
+	return d, nil
+}
+
 // evalNot computes result of (!X).
 func (e *Evaluator) evalNot(expr *tipb.Expr) (types.Datum, error) {
 	if len(expr.Children) != 1 {

mysql/mydecimal.go

@@ -164,6 +164,11 @@ type MyDecimal struct {
 	wordBuf [maxWordBufLen]int32
 }
 
+// IsNegative returns whether a decimal is negative.
+func (d *MyDecimal) IsNegative() bool {
+	return d.negative
+}
+
 // String returns the decimal string representation rounded to resultFrac.
 func (d *MyDecimal) String() string {
 	tmp := *d

plan/expr_to_pb.go

@@ -125,8 +125,10 @@ func scalarFuncToPBExpr(client kv.Client, expr *expression.ScalarFunction) *tipb
 		return compareOpsToPBExpr(client, expr)
 	case ast.Plus, ast.Minus, ast.Mul, ast.Div, ast.Mod, ast.IntDiv:
 		return arithmeticalOpsToPBExpr(client, expr)
-	case ast.AndAnd, ast.OrOr, ast.UnaryNot:
+	case ast.AndAnd, ast.OrOr, ast.UnaryNot, ast.LogicXor:
 		return logicalOpsToPBExpr(client, expr)
+	case ast.And, ast.Or, ast.BitNeg, ast.Xor, ast.LeftShift, ast.RightShift:
+		return bitwiseFuncToPBExpr(client, expr)
 	case ast.Case, ast.Coalesce:
 		return builtinFuncToPBExpr(client, expr)
 	default:
@@ -230,9 +232,47 @@ func logicalOpsToPBExpr(client kv.Client, expr *expression.ScalarFunction) *tipb
 		tp = tipb.ExprType_And
 	case ast.OrOr:
 		tp = tipb.ExprType_Or
+	case ast.LogicXor:
+		tp = tipb.ExprType_Xor
 	case ast.UnaryNot:
 		return notToPBExpr(client, expr)
 	}
+	if !client.SupportRequestType(kv.ReqTypeSelect, int64(tp)) {
+		return nil
+	}
+
+	expr0 := exprToPB(client, expr.Args[0])
+	if expr0 == nil {
+		return nil
+	}
+	expr1 := exprToPB(client, expr.Args[1])
+	if expr1 == nil {
+		return nil
+	}
+	return &tipb.Expr{
+		Tp:       tp,
+		Children: []*tipb.Expr{expr0, expr1}}
+}
+
+func bitwiseFuncToPBExpr(client kv.Client, expr *expression.ScalarFunction) *tipb.Expr {
+	var tp tipb.ExprType
+	switch expr.FuncName.L {
+	case ast.And:
+		tp = tipb.ExprType_BitAnd
+	case ast.Or:
+		tp = tipb.ExprType_BitOr
+	case ast.Xor:
+		tp = tipb.ExprType_BitXor
+	case ast.LeftShift:
+		tp = tipb.ExprType_LeftShift
+	case ast.RightShift:
+		tp = tipb.ExprType_RighShift
+	case ast.BitNeg:
+		return bitNegToPBExpr(client, expr)
+	}
+	if !client.SupportRequestType(kv.ReqTypeSelect, int64(tp)) {
+		return nil
+	}
 	expr0 := exprToPB(client, expr.Args[0])
 	if expr0 == nil {
 		return nil
@@ -278,6 +318,21 @@ func notToPBExpr(client kv.Client, expr *expression.ScalarFunction) *tipb.Expr {
 		Children: []*tipb.Expr{child}}
 }
 
+func bitNegToPBExpr(client kv.Client, expr *expression.ScalarFunction) *tipb.Expr {
+	if !client.SupportRequestType(kv.ReqTypeSelect, int64(tipb.ExprType_BitNeg)) {
+		return nil
+	}
+
+	child := exprToPB(client, expr.Args[0])
+	if child == nil {
+		return nil
+	}
+
+	return &tipb.Expr{
+		Tp:       tipb.ExprType_BitNeg,
+		Children: []*tipb.Expr{child}}
+}
+
 func constListToPB(client kv.Client, list []expression.Expression) *tipb.Expr {
 	if !client.SupportRequestType(kv.ReqTypeSelect, int64(tipb.ExprType_ValueList)) {
 		return nil

plan/plan_test.go

@@ -118,20 +118,30 @@ func mockResolve(node ast.Node) error {
 
 func supportExpr(exprType tipb.ExprType) bool {
 	switch exprType {
-	case tipb.ExprType_Null, tipb.ExprType_Int64, tipb.ExprType_Uint64, tipb.ExprType_Float32,
-		tipb.ExprType_Float64, tipb.ExprType_String, tipb.ExprType_Bytes,
-		tipb.ExprType_MysqlDuration, tipb.ExprType_MysqlDecimal, tipb.ExprType_MysqlTime,
-		tipb.ExprType_ColumnRef,
-		tipb.ExprType_And, tipb.ExprType_Or,
-		tipb.ExprType_LT, tipb.ExprType_LE, tipb.ExprType_EQ, tipb.ExprType_NE,
+	// data type
+	case tipb.ExprType_Null, tipb.ExprType_Int64, tipb.ExprType_Uint64,
+		tipb.ExprType_Float32, tipb.ExprType_Float64, tipb.ExprType_String,
+		tipb.ExprType_Bytes, tipb.ExprType_MysqlDuration, tipb.ExprType_MysqlDecimal,
+		tipb.ExprType_MysqlTime, tipb.ExprType_ColumnRef:
+		return true
+	// logic operators
+	case tipb.ExprType_And, tipb.ExprType_Or, tipb.ExprType_Not, tipb.ExprType_Xor:
+		return true
+	// compare operators
+	case tipb.ExprType_LT, tipb.ExprType_LE, tipb.ExprType_EQ, tipb.ExprType_NE,
 		tipb.ExprType_GE, tipb.ExprType_GT, tipb.ExprType_NullEQ,
-		tipb.ExprType_In, tipb.ExprType_ValueList,
-		tipb.ExprType_Not,
-		tipb.ExprType_Like:
+		tipb.ExprType_In, tipb.ExprType_ValueList, tipb.ExprType_Like:
 		return true
-	case tipb.ExprType_Plus, tipb.ExprType_Div:
+	// arithmetic operators
+	case tipb.ExprType_Plus, tipb.ExprType_Div, tipb.ExprType_Minus,
+		tipb.ExprType_Mul, tipb.ExprType_IntDiv, tipb.ExprType_Mod:
 		return true
-	case tipb.ExprType_Count, tipb.ExprType_First, tipb.ExprType_Sum, tipb.ExprType_Avg, tipb.ExprType_Max, tipb.ExprType_Min:
+	// aggregate functions
+	case tipb.ExprType_Count, tipb.ExprType_First, tipb.ExprType_Sum,
+		tipb.ExprType_Avg, tipb.ExprType_Max, tipb.ExprType_Min:
+		return true
+	// bitwise operators
+	case tipb.ExprType_BitAnd, tipb.ExprType_BitOr, tipb.ExprType_BitXor, tipb.ExprType_BitNeg:
 		return true
 	case tipb.ExprType_Case, tipb.ExprType_Coalesce:
 		return true
@@ -276,7 +286,7 @@ func (s *testPlanSuite) TestTopnPushDown(c *C) {
 }
 
 // TestLogicOpsPushDown tests whether logic operators been pushed down successfully.
-func (s *testPlanSuite) TestLogicOpsPushDown(c *C) {
+func (s *testPlanSuite) TestOperatorsPushDown(c *C) {
 	defer testleak.AfterTest(c)()
 	cases := []struct {
 		sql    string
@@ -303,6 +313,31 @@ func (s *testPlanSuite) TestLogicOpsPushDown(c *C) {
 			cond:   "not(test.t.a)",
 			exprPB: "\b\xe9\a\x1a\r\b\xc9\x01\x12\b\x80\x00\x00\x00\x00\x00\x00\x01",
 		},
+		{
+			sql:    "a xor b",
+			cond:   "xor(test.t.a, test.t.b)",
+			exprPB: "\b\xff\x11\x1a\r\b\xc9\x01\x12\b\x80\x00\x00\x00\x00\x00\x00\x01\x1a\r\b\xc9\x01\x12\b\x80\x00\x00\x00\x00\x00\x00\x02",
+		},
+		{
+			sql:    "a & b",
+			cond:   "bitand(test.t.a, test.t.b)",
+			exprPB: "\b\xb5\x10\x1a\r\b\xc9\x01\x12\b\x80\x00\x00\x00\x00\x00\x00\x01\x1a\r\b\xc9\x01\x12\b\x80\x00\x00\x00\x00\x00\x00\x02",
+		},
+		{
+			sql:    "a | b",
+			cond:   "bitor(test.t.a, test.t.b)",
+			exprPB: "\b\xb6\x10\x1a\r\b\xc9\x01\x12\b\x80\x00\x00\x00\x00\x00\x00\x01\x1a\r\b\xc9\x01\x12\b\x80\x00\x00\x00\x00\x00\x00\x02",
+		},
+		{
+			sql:    "a ^ b",
+			cond:   "bitxor(test.t.a, test.t.b)",
+			exprPB: "\b\xb7\x10\x1a\r\b\xc9\x01\x12\b\x80\x00\x00\x00\x00\x00\x00\x01\x1a\r\b\xc9\x01\x12\b\x80\x00\x00\x00\x00\x00\x00\x02",
+		},
+		{
+			sql:    "~a",
+			cond:   "bitneg(test.t.a)",
+			exprPB: "\b\xeb\a\x1a\r\b\xc9\x01\x12\b\x80\x00\x00\x00\x00\x00\x00\x01",
+		},
 	}
 	for _, ca := range cases {
 		sql := "select * from t where " + ca.sql

store/localstore/local_client.go

@@ -51,21 +51,30 @@ func (c *dbClient) SupportRequestType(reqType, subType int64) bool {
 
 func supportExpr(exprType tipb.ExprType) bool {
 	switch exprType {
-	case tipb.ExprType_Null, tipb.ExprType_Int64, tipb.ExprType_Uint64, tipb.ExprType_Float32,
-		tipb.ExprType_Float64, tipb.ExprType_String, tipb.ExprType_Bytes,
-		tipb.ExprType_MysqlDuration, tipb.ExprType_MysqlDecimal, tipb.ExprType_MysqlTime,
-		tipb.ExprType_ColumnRef,
-		tipb.ExprType_And, tipb.ExprType_Or,
-		tipb.ExprType_LT, tipb.ExprType_LE, tipb.ExprType_EQ, tipb.ExprType_NE,
+	// data type.
+	case tipb.ExprType_Null, tipb.ExprType_Int64, tipb.ExprType_Uint64,
+		tipb.ExprType_Float32, tipb.ExprType_Float64, tipb.ExprType_String,
+		tipb.ExprType_Bytes, tipb.ExprType_MysqlDuration, tipb.ExprType_MysqlDecimal,
+		tipb.ExprType_MysqlTime, tipb.ExprType_ColumnRef:
+		return true
+	// logic operators.
+	case tipb.ExprType_And, tipb.ExprType_Or, tipb.ExprType_Not, tipb.ExprType_Xor:
+		return true
+	// compare operators.
+	case tipb.ExprType_LT, tipb.ExprType_LE, tipb.ExprType_EQ, tipb.ExprType_NE,
 		tipb.ExprType_GE, tipb.ExprType_GT, tipb.ExprType_NullEQ,
-		tipb.ExprType_In, tipb.ExprType_ValueList,
-		tipb.ExprType_Not,
-		tipb.ExprType_Like:
+		tipb.ExprType_In, tipb.ExprType_ValueList, tipb.ExprType_Like:
+		return true
+	// arithmetic operators.
+	case tipb.ExprType_Plus, tipb.ExprType_Div, tipb.ExprType_Minus,
+		tipb.ExprType_Mul, tipb.ExprType_IntDiv, tipb.ExprType_Mod:
 		return true
-	case tipb.ExprType_Plus, tipb.ExprType_Div, tipb.ExprType_Minus, tipb.ExprType_Mul,
-		tipb.ExprType_IntDiv, tipb.ExprType_Mod:
+	// aggregate functions.
+	case tipb.ExprType_Count, tipb.ExprType_First, tipb.ExprType_Sum,
+		tipb.ExprType_Avg, tipb.ExprType_Max, tipb.ExprType_Min:
 		return true
-	case tipb.ExprType_Count, tipb.ExprType_First, tipb.ExprType_Sum, tipb.ExprType_Avg, tipb.ExprType_Max, tipb.ExprType_Min:
+	// bitwise operators.
+	case tipb.ExprType_BitAnd, tipb.ExprType_BitOr, tipb.ExprType_BitXor, tipb.ExprType_BitNeg:
 		return true
 	case tipb.ExprType_Case, tipb.ExprType_Coalesce:
 		return true