Skip to content

Commit

Permalink
Merge branch 'master' into disable-mc-in-bench
Browse files Browse the repository at this point in the history
  • Loading branch information
@tiancaiamao
tiancaiamao authored Feb 24, 2022
2 parents 20b7c9c + 47f9e9c commit fcd4cd4
Show file tree
Hide file tree
Showing 7 changed files with 134 additions and 29 deletions.
6 changes: 3 additions & 3 deletions br/cmd/br/cmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/pingcap/tidb/br/pkg/task"
"github.com/pingcap/tidb/br/pkg/utils"
"github.com/pingcap/tidb/br/pkg/version/build"
"github.com/pingcap/tidb/config"
"github.com/pingcap/tidb/util/logutil"
"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -111,9 +112,8 @@ func Init(cmd *cobra.Command) (err error) {
// otherwise the info will be print in stdout...
tidbLogCfg.File.Filename = timestampLogFileName()
} else {
// Disable annoying TiDB Log.
// TODO: some error logs outputs randomly, we need to fix them in TiDB.
tidbLogCfg.Level = "fatal"
// Don't print slow log in br
config.GetGlobalConfig().Log.EnableSlowLog.Store(false)
}
e = logutil.InitLogger(&tidbLogCfg)
if e != nil {
Expand Down
4 changes: 2 additions & 2 deletions br/pkg/lightning/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1060,9 +1060,9 @@ func (cfg *Config) CheckAndAdjustFilePath() error {
if err != nil {
return errors.Annotatef(err, "covert data-source-dir '%s' to absolute path failed", cfg.Mydumper.SourceDir)
}
cfg.Mydumper.SourceDir = "file://" + filepath.ToSlash(absPath)
u.Path = absPath
u.Path = filepath.ToSlash(absPath)
u.Scheme = "file"
cfg.Mydumper.SourceDir = u.String()
}

found := false
Expand Down
42 changes: 27 additions & 15 deletions br/pkg/lightning/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,27 +154,39 @@ func TestCheckAndAdjustFilePath(t *testing.T) {
tmpDir := t.TempDir()
// use slashPath in url to be compatible with windows
slashPath := filepath.ToSlash(tmpDir)
pwd, err := os.Getwd()
require.NoError(t, err)
specialDir, err := os.MkdirTemp(tmpDir, "abc??bcd")
require.NoError(t, err)
specialDir1, err := os.MkdirTemp(tmpDir, "abc%3F%3F%3Fbcd")
require.NoError(t, err)

cfg := config.NewConfig()
cases := []string{
tmpDir,
".",
"file://" + slashPath,
"local://" + slashPath,
"s3://bucket_name",
"s3://bucket_name/path/to/dir",
"gcs://bucketname/path/to/dir",
"gs://bucketname/path/to/dir",
"noop:///",
}

cases := []struct {
test string
expect string
}{
{tmpDir, tmpDir},
{".", filepath.ToSlash(pwd)},
{specialDir, specialDir},
{specialDir1, specialDir1},
{"file://" + slashPath, slashPath},
{"local://" + slashPath, slashPath},
{"s3://bucket_name", ""},
{"s3://bucket_name/path/to/dir", "/path/to/dir"},
{"gcs://bucketname/path/to/dir", "/path/to/dir"},
{"gs://bucketname/path/to/dir", "/path/to/dir"},
{"noop:///", "/"},
}
for _, testCase := range cases {
cfg.Mydumper.SourceDir = testCase

err := cfg.CheckAndAdjustFilePath()
cfg.Mydumper.SourceDir = testCase.test
err = cfg.CheckAndAdjustFilePath()
require.NoError(t, err)
u, err := url.Parse(cfg.Mydumper.SourceDir)
require.NoError(t, err)
require.Equal(t, testCase.expect, u.Path)
}

}

func TestAdjustFileRoutePath(t *testing.T) {
Expand Down
19 changes: 18 additions & 1 deletion expression/builtin_cast.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1937,7 +1937,24 @@ func WrapWithCastAsDecimal(ctx sessionctx.Context, expr Expression) Expression {
}
types.SetBinChsClnFlag(tp)
tp.Flag |= expr.GetType().Flag & mysql.UnsignedFlag
return BuildCastFunction(ctx, expr, tp)
castExpr := BuildCastFunction(ctx, expr, tp)
// For const item, we can use find-grained precision and scale by the result.
if castExpr.ConstItem(ctx.GetSessionVars().StmtCtx) {
val, isnull, err := castExpr.EvalDecimal(ctx, chunk.Row{})
if !isnull && err == nil {
precision, frac := val.PrecisionAndFrac()
castTp := castExpr.GetType()
castTp.Decimal = frac
castTp.Flen = precision
if castTp.Flen > mysql.MaxDecimalWidth {
castTp.Flen = mysql.MaxDecimalWidth
}
if castTp.Decimal > mysql.MaxDecimalScale {
castTp.Decimal = mysql.MaxDecimalScale
}
}
}
return castExpr
}

// WrapWithCastAsString wraps `expr` with `cast` if the return type of expr is
Expand Down
76 changes: 76 additions & 0 deletions expression/builtin_cast_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1498,3 +1498,79 @@ func TestCastStringAsDecimalSigWithUnsignedFlagInUnion(t *testing.T) {
require.Equal(t, 0, res.Compare(c.res))
}
}

func TestCastConstAsDecimalFieldType(t *testing.T) {
type testCase struct {
input *Constant
resultFlen int
resultDecimal int
}
allTestCase := []testCase{
// test int
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong}, Value: types.NewIntDatum(0)}, 1, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong}, Value: types.NewIntDatum(1)}, 1, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong}, Value: types.NewIntDatum(-1)}, 1, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong}, Value: types.NewIntDatum(11111)}, 5, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong}, Value: types.NewIntDatum(-11111)}, 5, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong}, Value: types.NewIntDatum(1111111111)}, 10, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong}, Value: types.NewIntDatum(-1111111111)}, 10, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong}, Value: types.NewIntDatum(111111111111111)}, 15, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong}, Value: types.NewIntDatum(-111111111111111)}, 15, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong}, Value: types.NewIntDatum(9223372036854775807)}, 19, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong}, Value: types.NewIntDatum(-9223372036854775808)}, 19, 0},
// test uint
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong, Flag: mysql.UnsignedFlag}, Value: types.NewUintDatum(0)}, 1, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong, Flag: mysql.UnsignedFlag}, Value: types.NewUintDatum(1)}, 1, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong, Flag: mysql.UnsignedFlag}, Value: types.NewUintDatum(11111)}, 5, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong, Flag: mysql.UnsignedFlag}, Value: types.NewUintDatum(1111111111)}, 10, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong, Flag: mysql.UnsignedFlag}, Value: types.NewUintDatum(111111111111111)}, 15, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong, Flag: mysql.UnsignedFlag}, Value: types.NewUintDatum(9223372036854775807)}, 19, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeLonglong, Flag: mysql.UnsignedFlag}, Value: types.NewUintDatum(18446744073709551615)}, 20, 0},
// test decimal, use origin fieldType
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeNewDecimal, Flen: 10, Decimal: 5}, Value: types.NewDecimalDatum(types.NewDecFromStringForTest("12345"))}, 10, 5},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeNewDecimal, Flen: 2, Decimal: 1}, Value: types.NewDecimalDatum(types.NewDecFromStringForTest("1"))}, 2, 1},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeNewDecimal, Flen: 30, Decimal: 0}, Value: types.NewDecimalDatum(types.NewDecFromStringForTest("12345"))}, 30, 0},
// test real
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDouble, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewFloat64Datum(1.234)}, 4, 3},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDouble, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewFloat64Datum(1.23456789)}, 9, 8},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDouble, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewFloat64Datum(-1234567890.123456789)}, 17, 7}, // float precision lost
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDouble, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewFloat64Datum(-1234567890.1234567890123456789)}, 17, 7}, // float precision lost
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDouble, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewFloat64Datum(1e10)}, 11, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDouble, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewFloat64Datum(1e20)}, 21, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDouble, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewFloat64Datum(1e40)}, 41, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDouble, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewFloat64Datum(1e60)}, 61, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDouble, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewFloat64Datum(1e80)}, 65, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDouble, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewFloat64Datum(1e-10)}, 10, 10},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDouble, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewFloat64Datum(1e-20)}, 20, 20},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDouble, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewFloat64Datum(1e-40)}, 40, 30},
// test string
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeString, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewStringDatum("123.456")}, 6, 3},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeString, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewStringDatum("123.4560")}, 7, 4},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeString, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewStringDatum("123.456000000")}, 12, 9},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeString, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewStringDatum("123abcde")}, 3, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeString, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewStringDatum("1e80")}, 65, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeString, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewStringDatum("1e-40")}, 40, 30},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeString, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewStringDatum("-1234567890.123456789")}, 19, 9},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeString, Flen: types.UnspecifiedLength, Decimal: types.UnspecifiedLength}, Value: types.NewStringDatum("-1234567890.1234567890123456789")}, 29, 19},
// test time
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDuration, Flen: types.UnspecifiedLength, Decimal: 3}, Value: types.NewDurationDatum(types.NewDuration(10, 10, 10, 110, 3))}, 9, 3},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDuration, Flen: types.UnspecifiedLength, Decimal: 6}, Value: types.NewDurationDatum(types.NewDuration(10, 10, 10, 110, 6))}, 12, 6},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDuration, Flen: types.UnspecifiedLength, Decimal: 0}, Value: types.NewDurationDatum(types.NewDuration(10, 10, 10, 110, 0))}, 6, 0},
// test timestamp
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeTimestamp, Flen: types.UnspecifiedLength, Decimal: 0}, Value: types.NewTimeDatum(types.NewTime(types.FromDate(2020, 10, 10, 10, 10, 10, 110), mysql.TypeTimestamp, 0))}, 14, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeTimestamp, Flen: types.UnspecifiedLength, Decimal: 3}, Value: types.NewTimeDatum(types.NewTime(types.FromDate(2020, 10, 10, 10, 10, 10, 110), mysql.TypeTimestamp, 0))}, 17, 3},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeTimestamp, Flen: types.UnspecifiedLength, Decimal: 6}, Value: types.NewTimeDatum(types.NewTime(types.FromDate(2020, 10, 10, 10, 10, 10, 110), mysql.TypeTimestamp, 0))}, 20, 6},
// test datetime
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDatetime, Flen: types.UnspecifiedLength, Decimal: 0}, Value: types.NewTimeDatum(types.NewTime(types.FromDate(2020, 10, 10, 10, 10, 10, 110), mysql.TypeDatetime, 0))}, 14, 0},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDatetime, Flen: types.UnspecifiedLength, Decimal: 3}, Value: types.NewTimeDatum(types.NewTime(types.FromDate(2020, 10, 10, 10, 10, 10, 110), mysql.TypeDatetime, 0))}, 17, 3},
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDatetime, Flen: types.UnspecifiedLength, Decimal: 6}, Value: types.NewTimeDatum(types.NewTime(types.FromDate(2020, 10, 10, 10, 10, 10, 110), mysql.TypeDatetime, 0))}, 20, 6},
// test date
{&Constant{RetType: &types.FieldType{Tp: mysql.TypeDate, Flen: types.UnspecifiedLength, Decimal: 0}, Value: types.NewTimeDatum(types.NewTime(types.FromDate(2020, 10, 10, 10, 10, 10, 110), mysql.TypeDate, 0))}, 8, 0},
}
ctx := createContext(t)
for _, tc := range allTestCase {
expr := WrapWithCastAsDecimal(ctx, tc.input)
require.Equal(t, tc.resultFlen, expr.GetType().Flen)
require.Equal(t, tc.resultDecimal, expr.GetType().Decimal)
}
}
7 changes: 4 additions & 3 deletions expression/builtin_time.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1667,7 +1667,8 @@ func (c *fromUnixTimeFunctionClass) getFunction(ctx sessionctx.Context, args []E
argTps = append(argTps, types.ETString)
}

isArg0Str := args[0].GetType().EvalType() == types.ETString
arg0Tp := args[0].GetType()
isArg0Str := arg0Tp.EvalType() == types.ETString
bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, retTp, argTps...)
if err != nil {
return nil, err
Expand All @@ -1682,8 +1683,8 @@ func (c *fromUnixTimeFunctionClass) getFunction(ctx sessionctx.Context, args []E
// Calculate the time fsp.
fsp := types.MaxFsp
if !isArg0Str {
if args[0].GetType().Decimal != types.UnspecifiedLength {
fsp = mathutil.Min(bf.tp.Decimal, args[0].GetType().Decimal)
if arg0Tp.Decimal != types.UnspecifiedLength {
fsp = mathutil.Min(bf.tp.Decimal, arg0Tp.Decimal)
}
}
bf.setDecimalAndFlenForDatetime(fsp)
Expand Down
9 changes: 4 additions & 5 deletions planner/core/testdata/integration_suite_out.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4771,14 +4771,13 @@
" └─HashJoin 12500.00 cop[tiflash] inner join, equal:[eq(Column#13, Column#14) eq(Column#15, Column#16)]",
" ├─ExchangeReceiver(Build) 10000.00 cop[tiflash] ",
" │ └─ExchangeSender 10000.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: [name: Column#21, collate: binary], [name: Column#15, collate: binary]",
" │ └─Projection 10000.00 cop[tiflash] test.t.c1, test.t.c2, Column#13, Column#15, cast(Column#13, decimal(34,8) BINARY)->Column#21",
" │ └─Projection 10000.00 cop[tiflash] test.t.c1, test.t.c2, Column#13, Column#15, cast(Column#13, decimal(15,8) BINARY)->Column#21",
" │ └─Projection 10000.00 cop[tiflash] test.t.c1, test.t.c2, mul(test.t.c1, 3)->Column#13, plus(test.t.c1, 1)->Column#15",
" │ └─TableFullScan 10000.00 cop[tiflash] table:t1 keep order:false, stats:pseudo",
" └─ExchangeReceiver(Probe) 10000.00 cop[tiflash] ",
" └─ExchangeSender 10000.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: [name: Column#22, collate: binary], [name: Column#16, collate: binary]",
" └─Projection 10000.00 cop[tiflash] test.t.c1, test.t.c2, test.t.c3, Column#14, Column#16, cast(Column#14, decimal(34,8) BINARY)->Column#22",
" └─Projection 10000.00 cop[tiflash] test.t.c1, test.t.c2, test.t.c3, div(test.t.c3, 2)->Column#14, minus(test.t.c2, 10)->Column#16",
" └─TableFullScan 10000.00 cop[tiflash] table:t2 keep order:false, stats:pseudo"
" └─ExchangeSender 10000.00 cop[tiflash] ExchangeType: HashPartition, Hash Cols: [name: Column#14, collate: binary], [name: Column#16, collate: binary]",
" └─Projection 10000.00 cop[tiflash] test.t.c1, test.t.c2, test.t.c3, div(test.t.c3, 2)->Column#14, minus(test.t.c2, 10)->Column#16",
" └─TableFullScan 10000.00 cop[tiflash] table:t2 keep order:false, stats:pseudo"
]
},
{
Expand Down

0 comments on commit fcd4cd4

Please sign in to comment.