Skip to content

Commit

Permalink
This is an automated cherry-pick of pingcap#38802
Browse files Browse the repository at this point in the history 
Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io>
  • Loading branch information
@winoros @ti-chi-bot
winoros authored and ti-chi-bot committed Nov 2, 2022
1 parent ecd6753 commit 85ba3e4
Show file tree
Hide file tree
Showing 4 changed files with 172 additions and 20 deletions.
139 changes: 139 additions & 0 deletions expression/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7651,3 +7651,142 @@ func TestRegexpPushdown(t *testing.T) {
" └─TableReader_7 10000.00 root data:TableFullScan_6",
" └─TableFullScan_6 10000.00 cop[tikv] table:regbin keep order:false, stats:pseudo"))
}
<<<<<<< HEAD
=======

func TestIssue35184(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")

tk.MustExec("drop table if exists ft")
tk.MustExec("create table ft (tint int, tdou double, tdec decimal(22,9),tchar char(44))")
tk.MustExec("insert into ft values(1234567890,123467890.1234,123467890.1234,'123467890.1234')")
tk.MustExec("insert into ft values(1234567890,123467890.123456789,123467890.123456789,'123467890.123456789')")

result := tk.MustQuery("SELECT FROM_UNIXTIME(tchar) from ft")
unixTime1 := "1973-11-30 08:38:10.123400"
unixTime2 := "1973-11-30 08:38:10.123457"
result.Check(testkit.Rows(unixTime1, unixTime2))

tk.MustExec("drop table if exists ft")
tk.MustExec("create table ft (tint int, tdou double, tdec decimal(22,9),tchar varchar(44))")
tk.MustExec("insert into ft values(1234567890,123467890.1234,123467890.1234,'123467890.1234')")
tk.MustExec("insert into ft values(1234567890,123467890.123456789,123467890.123456789,'123467890.123456789')")
result = tk.MustQuery("SELECT FROM_UNIXTIME(tchar) from ft")
result.Check(testkit.Rows(unixTime1, unixTime2))

tk.MustExec("drop table if exists ft")
tk.MustExec("create table ft (tint int, tdou double, tdec decimal(22,9),tchar blob)")
tk.MustExec("insert into ft values(1234567890,123467890.1234,123467890.1234,'123467890.1234')")
tk.MustExec("insert into ft values(1234567890,123467890.123456789,123467890.123456789,'123467890.123456789')")
result = tk.MustQuery("SELECT FROM_UNIXTIME(tchar) from ft")
result.Check(testkit.Rows(unixTime1, unixTime2))

tk.MustExec("drop table if exists ft")
tk.MustExec("create table ft (tint int, tdou double, tdec decimal(22,9),tchar tinyblob)")
tk.MustExec("insert into ft values(1234567890,123467890.1234,123467890.1234,'123467890.1234')")
tk.MustExec("insert into ft values(1234567890,123467890.123456789,123467890.123456789,'123467890.123456789')")
result = tk.MustQuery("SELECT FROM_UNIXTIME(tchar) from ft")
result.Check(testkit.Rows(unixTime1, unixTime2))

tk.MustExec("drop table if exists ft")
tk.MustExec("create table ft (tint int, tdou double, tdec decimal(22,9),tchar mediumblob)")
tk.MustExec("insert into ft values(1234567890,123467890.1234,123467890.1234,'123467890.1234')")
tk.MustExec("insert into ft values(1234567890,123467890.123456789,123467890.123456789,'123467890.123456789')")
result = tk.MustQuery("SELECT FROM_UNIXTIME(tchar) from ft")
result.Check(testkit.Rows(unixTime1, unixTime2))

tk.MustExec("drop table if exists ft")
tk.MustExec("create table ft (tint int, tdou double, tdec decimal(22,9),tchar longblob)")
tk.MustExec("insert into ft values(1234567890,123467890.1234,123467890.1234,'123467890.1234')")
tk.MustExec("insert into ft values(1234567890,123467890.123456789,123467890.123456789,'123467890.123456789')")
result = tk.MustQuery("SELECT FROM_UNIXTIME(tchar) from ft")
result.Check(testkit.Rows(unixTime1, unixTime2))

tk.MustExec("truncate table ft")
tk.MustExec("insert into ft values(1234567890,123467890.1234,123467890.1234,'123467890.1234000000000000000000100111111111')")
result = tk.MustQuery("SELECT FROM_UNIXTIME(tchar) from ft")
result.Check(testkit.Rows(unixTime1))
tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1292 Truncated incorrect DECIMAL value: '123467890.1234000000000000000000100111111111'"))

tk.MustExec("truncate table ft")
tk.MustExec("insert into ft values(1234567890,123467890.1234,123467890.1234,'11111123467890.1234')")
result = tk.MustQuery("SELECT FROM_UNIXTIME(tchar) from ft")
result.Check(testkit.Rows("<nil>"))

tk.MustExec("drop table if exists ft")
tk.MustExec("create table ft (tint int, tdou double, tdec decimal(22,9),tchar char(44))")
tk.MustExec("insert into ft values(1234567890,123467890.1234,123467890.1234,'123467890.1234')")
result = tk.MustQuery("SELECT FROM_UNIXTIME(tchar) from ft where FROM_UNIXTIME(tchar)= '1973-11-30 08:38:10.123400' ")
result.Check(testkit.Rows(unixTime1))

result = tk.MustQuery("SELECT FROM_UNIXTIME(cast(tchar as decimal(44,1))) from ft where FROM_UNIXTIME(tchar)= '1973-11-30 08:38:10.123400' ")
result.Check(testkit.Rows("1973-11-30 08:38:10.1"))

result = tk.MustQuery("SELECT FROM_UNIXTIME(tchar,'%Y%m%d') from ft where FROM_UNIXTIME(tchar)= '1973-11-30 08:38:10.123400' ")
result.Check(testkit.Rows("19731130"))
}

func TestFix38127(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("create table t(dou double, varc varchar(100))")
tk.MustExec("insert into t values (1.23e23, '111111111111111111111111111111111111111111111111111111111111111111111111111')")
tk.MustQuery("select from_unixtime(dou) from t").Check(testkit.Rows("<nil>"))
tk.MustQuery("select from_unixtime(varc) from t").Check(testkit.Rows("<nil>"))
tk.MustQuery("select from_unixtime(dou, '%Y-%m-%d') from t").Check(testkit.Rows("<nil>"))
tk.MustQuery("select from_unixtime(varc, '%Y-%m-%d') from t").Check(testkit.Rows("<nil>"))
}

func TestJSONStorageFree(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustQuery("select json_storage_free(NULL)").Check(testkit.Rows("<nil>"))
tk.MustQuery("select json_storage_free('{}')").Check(testkit.Rows("0"))
tk.MustQuery("select json_storage_free('1')").Check(testkit.Rows("0"))
tk.MustQuery(`select json_storage_free('{"a": "b"}')`).Check(testkit.Rows("0"))
err := tk.ExecToErr(`select json_storage_free('{"c":["a","b"]`)
require.Error(t, err, "[json:3140]Invalid JSON text: The document root must not be followed by other values.")
}

func TestIssue38736(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("CREATE TABLE t0(c0 BOOL, c1 INT);")
tk.MustExec("CREATE TABLE t1 LIKE t0;")
tk.MustExec("CREATE definer='root'@'localhost' VIEW v0(c0) AS SELECT IS_IPV4(t0.c1) FROM t0, t1;")
tk.MustExec("INSERT INTO t0(c0, c1) VALUES (true, 0);")
tk.MustExec("INSERT INTO t1(c0, c1) VALUES (true, 2);")

// The filter is evaled as false.
tk.MustQuery("SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);").Check(testkit.Rows())

// Also the filter is evaled as false.
tk.MustQuery("SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0) or v0.c0 > 0").Check(testkit.Rows())
}

func TestJSONExtractFromLast(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustQuery(`select json_extract('[{"a": [1,2,3,4]}]', '$[0] . a[last]')`).Check(testkit.Rows("4"))
tk.MustQuery(`select json_extract('[{"a": [1,2,3,4]}]', '$[0] . a [last - 1]')`).Check(testkit.Rows("3"))
tk.MustQuery(`select json_extract('[{"a": [1,2,3,4]}]', '$[0].a [last - 100]')`).Check(testkit.Rows("<nil>"))
}

func TestJSONExtractRange(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustQuery(`select json_extract('[{"a": [1,2,3,4]}]', '$[0].a[1 to last]')`).Check(testkit.Rows("[2, 3, 4]"))
tk.MustQuery(`select json_extract('[{"a": [1,2,3,4]}]', '$[0].a[1 to last - 1]')`).Check(testkit.Rows("[2, 3]"))
tk.MustQuery(`select json_extract('[{"a": [1,2,3,4]}]', '$[0].a[1 to last - 100]')`).Check(testkit.Rows("<nil>"))
tk.MustQuery(`select json_extract('[{"a": [1,2,3,4]}]', '$[0].a[1 to 100]')`).Check(testkit.Rows("[2, 3, 4]"))
tk.MustQuery(`select json_extract('[{"a": [1,2,3,4]}]', '$[0].a[0 to last]')`).Check(testkit.Rows("[1, 2, 3, 4]"))
tk.MustQuery(`select json_extract('[{"a": [1,2,3,4]}]', '$[0].a[0 to 2]')`).Check(testkit.Rows("[1, 2, 3]"))
}
>>>>>>> 0f62d1f42e (planner: projection should not push the expr that is not fully substituted (#38802))
42 changes: 22 additions & 20 deletions expression/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func (c *cowExprRef) Set(i int, changed bool, val Expression) {
return
}
c.new = make([]Expression, len(c.ref))
copy(c.new, c.ref[:i])
copy(c.new, c.ref)
c.new[i] = val
}

Expand Down Expand Up @@ -421,41 +421,43 @@ func ColumnSubstituteImpl(expr Expression, schema *Schema, newExprs []Expression
substituted := false
hasFail := false
if v.FuncName.L == ast.Cast {
newFunc := v.Clone().(*ScalarFunction)
substituted, hasFail, newFunc.GetArgs()[0] = ColumnSubstituteImpl(newFunc.GetArgs()[0], schema, newExprs, fail1Return)
var newArg Expression
substituted, hasFail, newArg = ColumnSubstituteImpl(v.GetArgs()[0], schema, newExprs, fail1Return)
if fail1Return && hasFail {
return substituted, hasFail, newFunc
return substituted, hasFail, v
}
if substituted {
// Workaround for issue https://github.com/pingcap/tidb/issues/28804
e := NewFunctionInternal(v.GetCtx(), v.FuncName.L, v.RetType, newFunc.GetArgs()...)
e := BuildCastFunction(v.GetCtx(), newArg, v.RetType)
e.SetCoercibility(v.Coercibility())
return true, false, e
}
return false, false, newFunc
return false, false, v
}
// cowExprRef is a copy-on-write util, args array allocation happens only
// when expr in args is changed
refExprArr := cowExprRef{v.GetArgs(), nil}
_, coll := DeriveCollationFromExprs(v.GetCtx(), v.GetArgs()...)
var tmpArgForCollCheck []Expression
if collate.NewCollationEnabled() {
tmpArgForCollCheck = make([]Expression, len(v.GetArgs()))
}
for idx, arg := range v.GetArgs() {
changed, hasFail, newFuncExpr := ColumnSubstituteImpl(arg, schema, newExprs, fail1Return)
if fail1Return && hasFail {
return changed, hasFail, v
changed, failed, newFuncExpr := ColumnSubstituteImpl(arg, schema, newExprs, fail1Return)
if fail1Return && failed {
return changed, failed, v
}
oldChanged := changed
if collate.NewCollationEnabled() {
if collate.NewCollationEnabled() && changed {
// Make sure the collation used by the ScalarFunction isn't changed and its result collation is not weaker than the collation used by the ScalarFunction.
if changed {
changed = false
tmpArgs := make([]Expression, 0, len(v.GetArgs()))
_ = append(append(append(tmpArgs, refExprArr.Result()[0:idx]...), refExprArr.Result()[idx+1:]...), newFuncExpr)
_, newColl := DeriveCollationFromExprs(v.GetCtx(), append(v.GetArgs(), newFuncExpr)...)
if coll == newColl {
changed = checkCollationStrictness(coll, newFuncExpr.GetType().GetCollate())
}
changed = false
copy(tmpArgForCollCheck, refExprArr.Result())
tmpArgForCollCheck[idx] = newFuncExpr
_, newColl := DeriveCollationFromExprs(v.GetCtx(), tmpArgForCollCheck...)
if coll == newColl {
changed = checkCollationStrictness(coll, newFuncExpr.GetType().GetCollate())
}
}
hasFail = hasFail || failed || oldChanged != changed
if fail1Return && oldChanged != changed {
// Only when the oldChanged is true and changed is false, we will get here.
// And this means there some dependency in this arg can be substituted with
Expand All @@ -470,7 +472,7 @@ func ColumnSubstituteImpl(expr Expression, schema *Schema, newExprs []Expression
}
}
if substituted {
return true, false, NewFunctionInternal(v.GetCtx(), v.FuncName.L, v.RetType, refExprArr.Result()...)
return true, hasFail, NewFunctionInternal(v.GetCtx(), v.FuncName.L, v.RetType, refExprArr.Result()...)
}
}
return false, false, expr
Expand Down
6 changes: 6 additions & 0 deletions planner/cascades/transformation_rules.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -550,8 +550,14 @@ func (*PushSelDownProjection) OnTransform(old *memo.ExprIter) (newExprs []*memo.
canBePushed := make([]expression.Expression, 0, len(sel.Conditions))
canNotBePushed := make([]expression.Expression, 0, len(sel.Conditions))
for _, cond := range sel.Conditions {
<<<<<<< HEAD
if !expression.HasGetSetVarFunc(cond) {
canBePushed = append(canBePushed, expression.ColumnSubstitute(cond, projSchema, proj.Exprs))
=======
substituted, hasFailed, newFilter := expression.ColumnSubstituteImpl(cond, projSchema, proj.Exprs, true)
if substituted && !hasFailed && !expression.HasGetSetVarFunc(newFilter) {
canBePushed = append(canBePushed, newFilter)
>>>>>>> 0f62d1f42e (planner: projection should not push the expr that is not fully substituted (#38802))
} else {
canNotBePushed = append(canNotBePushed, cond)
}
Expand Down
5 changes: 5 additions & 0 deletions planner/core/rule_predicate_push_down.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -470,8 +470,13 @@ func (p *LogicalProjection) PredicatePushDown(predicates []expression.Expression
}
}
for _, cond := range predicates {
<<<<<<< HEAD
newFilter := expression.ColumnSubstitute(cond, p.Schema(), p.Exprs)
if !expression.HasGetSetVarFunc(newFilter) {
=======
substituted, hasFailed, newFilter := expression.ColumnSubstituteImpl(cond, p.Schema(), p.Exprs, true)
if substituted && !hasFailed && !expression.HasGetSetVarFunc(newFilter) {
>>>>>>> 0f62d1f42e (planner: projection should not push the expr that is not fully substituted (#38802))
canBePushed = append(canBePushed, newFilter)
} else {
canNotBePushed = append(canNotBePushed, cond)
Expand Down

0 comments on commit 85ba3e4

Please sign in to comment.