diff --git a/.bazelrc b/.bazelrc index 61356f086fda5..8779e55aab209 100644 --- a/.bazelrc +++ b/.bazelrc @@ -24,6 +24,7 @@ test:ci --color=yes test:ci --verbose_failures --test_verbose_timeout_warnings test:ci --test_env=GO_TEST_WRAP_TESTV=1 test:ci --experimental_ui_max_stdouterr_bytes=104857600 +test:ci --test_timeout=120,300,900,3600 test:race --test_timeout=1200,6000,18000,72000 try-import /data/bazel diff --git a/bindinfo/BUILD.bazel b/bindinfo/BUILD.bazel index 64a11f86d5556..2c8f30bbd637e 100644 --- a/bindinfo/BUILD.bazel +++ b/bindinfo/BUILD.bazel @@ -44,7 +44,7 @@ go_library( go_test( name = "bindinfo_test", - timeout = "short", + timeout = "moderate", srcs = [ "bind_cache_test.go", "bind_test.go", diff --git a/br/pkg/backup/BUILD.bazel b/br/pkg/backup/BUILD.bazel index 934a71862ea43..1dccce1fee360 100644 --- a/br/pkg/backup/BUILD.bazel +++ b/br/pkg/backup/BUILD.bazel @@ -67,6 +67,7 @@ go_test( embed = [":backup"], flaky = True, race = "on", + shard_count = 9, deps = [ "//br/pkg/conn", "//br/pkg/gluetidb", diff --git a/br/pkg/backup/main_test.go b/br/pkg/backup/main_test.go index 7c6c43a5743c4..ef2db725841c2 100644 --- a/br/pkg/backup/main_test.go +++ b/br/pkg/backup/main_test.go @@ -29,6 +29,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("github.com/klauspost/compress/zstd.(*blockDec).startDecoder"), goleak.IgnoreTopFunction("github.com/pingcap/goleveldb/leveldb.(*DB).mpoolDrain"), goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } goleak.VerifyTestMain(m, opts...) } diff --git a/br/pkg/utils/BUILD.bazel b/br/pkg/utils/BUILD.bazel index 71b698df8e545..65b647edfea80 100644 --- a/br/pkg/utils/BUILD.bazel +++ b/br/pkg/utils/BUILD.bazel @@ -68,7 +68,7 @@ go_library( go_test( name = "utils_test", - timeout = "short", + timeout = "moderate", srcs = [ "backoff_test.go", "cdc_test.go", @@ -87,7 +87,7 @@ go_test( ], embed = [":utils"], flaky = True, - shard_count = 30, + shard_count = 36, deps = [ "//br/pkg/errors", "//br/pkg/metautil", diff --git a/build/BUILD.bazel b/build/BUILD.bazel index 3c2a569dd80d8..02258edcfdeb5 100644 --- a/build/BUILD.bazel +++ b/build/BUILD.bazel @@ -117,6 +117,7 @@ nogo( "@org_golang_x_tools//go/analysis/passes/structtag:go_default_library", "@org_golang_x_tools//go/analysis/passes/testinggoroutine:go_default_library", "@org_golang_x_tools//go/analysis/passes/tests:go_default_library", + "@org_golang_x_tools//go/analysis/passes/timeformat:go_default_library", "@org_golang_x_tools//go/analysis/passes/unmarshal:go_default_library", "@org_golang_x_tools//go/analysis/passes/unreachable:go_default_library", "@org_golang_x_tools//go/analysis/passes/unsafeptr:go_default_library", diff --git a/ddl/indexmergetest/BUILD.bazel b/ddl/indexmergetest/BUILD.bazel index 11eac6481154f..f2a3c053e9be0 100644 --- a/ddl/indexmergetest/BUILD.bazel +++ b/ddl/indexmergetest/BUILD.bazel @@ -9,7 +9,7 @@ go_test( ], flaky = True, race = "on", - shard_count = 18, + shard_count = 20, deps = [ "//config", "//ddl", diff --git a/ddl/metadatalocktest/BUILD.bazel b/ddl/metadatalocktest/BUILD.bazel index ee9d25ea7ec85..9492758292cc6 100644 --- a/ddl/metadatalocktest/BUILD.bazel +++ b/ddl/metadatalocktest/BUILD.bazel @@ -8,7 +8,7 @@ go_test( "mdl_test.go", ], flaky = True, - shard_count = 30, + shard_count = 31, deps = [ "//config", "//ddl", diff --git a/ddl/resourcegrouptest/BUILD.bazel b/ddl/resourcegrouptest/BUILD.bazel index d24fead34afde..cb638a4201852 100644 --- a/ddl/resourcegrouptest/BUILD.bazel +++ b/ddl/resourcegrouptest/BUILD.bazel @@ -6,6 +6,7 @@ go_test( srcs = ["resource_group_test.go"], flaky = True, race = "on", + shard_count = 4, deps = [ "//ddl/internal/callback", "//ddl/resourcegroup", diff --git a/executor/aggfuncs/BUILD.bazel b/executor/aggfuncs/BUILD.bazel index 11f9267f917c6..e3958fae6a0e1 100644 --- a/executor/aggfuncs/BUILD.bazel +++ b/executor/aggfuncs/BUILD.bazel @@ -58,7 +58,7 @@ go_library( go_test( name = "aggfuncs_test", - timeout = "short", + timeout = "moderate", srcs = [ "aggfunc_test.go", "export_test.go", diff --git a/executor/asyncloaddata/BUILD.bazel b/executor/asyncloaddata/BUILD.bazel index f70abe0bdd49a..b07ad41081fd0 100644 --- a/executor/asyncloaddata/BUILD.bazel +++ b/executor/asyncloaddata/BUILD.bazel @@ -33,6 +33,7 @@ go_test( embed = [":asyncloaddata"], flaky = True, race = "on", + shard_count = 6, deps = [ "//br/pkg/lightning/config", "//executor/importer", diff --git a/executor/autoidtest/BUILD.bazel b/executor/autoidtest/BUILD.bazel index 28aad8e63fa8c..7165d36ad2e67 100644 --- a/executor/autoidtest/BUILD.bazel +++ b/executor/autoidtest/BUILD.bazel @@ -9,7 +9,7 @@ go_test( ], flaky = True, race = "on", - shard_count = 5, + shard_count = 10, deps = [ "//autoid_service", "//config", diff --git a/executor/issuetest/BUILD.bazel b/executor/issuetest/BUILD.bazel index daf9d6d25db53..12282131b98e0 100644 --- a/executor/issuetest/BUILD.bazel +++ b/executor/issuetest/BUILD.bazel @@ -8,7 +8,6 @@ go_test( "main_test.go", ], flaky = True, - race = "on", shard_count = 50, deps = [ "//autoid_service", diff --git a/executor/loaddatatest/BUILD.bazel b/executor/loaddatatest/BUILD.bazel new file mode 100644 index 0000000000000..81d8a39979235 --- /dev/null +++ b/executor/loaddatatest/BUILD.bazel @@ -0,0 +1,27 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_test") + +go_test( + name = "loaddatatest_test", + timeout = "short", + srcs = [ + "load_data_test.go", + "main_test.go", + ], + flaky = True, + race = "on", + shard_count = 11, + deps = [ + "//br/pkg/lightning/mydump", + "//config", + "//executor", + "//meta/autoid", + "//sessionctx", + "//sessiontxn", + "//testkit", + "//util/dbterror/exeerrors", + "@com_github_stretchr_testify//require", + "@com_github_tikv_client_go_v2//tikv", + "@io_opencensus_go//stats/view", + "@org_uber_go_goleak//:goleak", + ], +) diff --git a/executor/loaddatatest/load_data_test.go b/executor/loaddatatest/load_data_test.go new file mode 100644 index 0000000000000..c02e0c419603c --- /dev/null +++ b/executor/loaddatatest/load_data_test.go @@ -0,0 +1,552 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package loaddatatest + +import ( + "context" + "testing" + + "github.com/pingcap/tidb/br/pkg/lightning/mydump" + "github.com/pingcap/tidb/executor" + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/sessiontxn" + "github.com/pingcap/tidb/testkit" + "github.com/pingcap/tidb/util/dbterror/exeerrors" + "github.com/stretchr/testify/require" +) + +type testCase struct { + data []byte + expected []string + expectedMsg string +} + +func checkCases( + tests []testCase, + ld *executor.LoadDataWorker, + t *testing.T, + tk *testkit.TestKit, + ctx sessionctx.Context, + selectSQL, deleteSQL string, +) { + origin := ld.GetController().IgnoreLines + for _, tt := range tests { + ld.GetController().IgnoreLines = origin + require.Nil(t, sessiontxn.NewTxn(context.Background(), ctx)) + ctx.GetSessionVars().StmtCtx.DupKeyAsWarning = true + ctx.GetSessionVars().StmtCtx.BadNullAsWarning = true + ctx.GetSessionVars().StmtCtx.InLoadDataStmt = true + ctx.GetSessionVars().StmtCtx.InDeleteStmt = false + + parser, err := mydump.NewCSVParser( + context.Background(), + ld.GetController().GenerateCSVConfig(), + mydump.NewStringReader(string(tt.data)), + 1, + nil, + false, + nil) + require.NoError(t, err) + + for ld.GetController().IgnoreLines > 0 { + ld.GetController().IgnoreLines-- + //nolint: errcheck + _ = parser.ReadRow() + } + + err1 := ld.ReadOneBatchRows(context.Background(), parser) + require.NoError(t, err1) + err1 = ld.CheckAndInsertOneBatch(context.Background(), ld.GetRows(), ld.GetCurBatchCnt()) + require.NoError(t, err1) + ld.ResetBatch() + ld.SetMessage() + require.Equal(t, tt.expectedMsg, tk.Session().LastMessage(), tt.expected) + ctx.StmtCommit(context.Background()) + txn, err := ctx.Txn(true) + require.NoError(t, err) + err = txn.Commit(context.Background()) + require.NoError(t, err) + tk.MustQuery(selectSQL).Check(testkit.RowsWithSep("|", tt.expected...)) + tk.MustExec(deleteSQL) + } +} + +func TestLoadDataInitParam(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + ctx := tk.Session().(sessionctx.Context) + defer ctx.SetValue(executor.LoadDataVarKey, nil) + + tk.MustExec("use test") + tk.MustExec("drop table if exists load_data_test") + tk.MustExec("create table load_data_test (a varchar(10), b varchar(10))") + + require.ErrorIs(t, tk.ExecToErr("load data infile '' into table load_data_test"), + exeerrors.ErrLoadDataEmptyPath) + require.ErrorIs(t, tk.ExecToErr("load data infile '/a' format '' into table load_data_test"), + exeerrors.ErrLoadDataUnsupportedFormat) + require.ErrorIs(t, tk.ExecToErr("load data infile '/a' format 'aaa' into table load_data_test"), + exeerrors.ErrLoadDataUnsupportedFormat) + require.ErrorContains(t, tk.ExecToErr("load data infile '/a' format 'sql file' into table load_data_test fields terminated by 'a'"), + "cannot specify FIELDS ... or LINES") + require.ErrorContains(t, tk.ExecToErr("load data infile '/a' format 'parquet' into table load_data_test fields terminated by 'a'"), + "cannot specify FIELDS ... or LINES") + require.ErrorContains(t, tk.ExecToErr("load data infile '/a' format 'sql file' into table load_data_test lines terminated by 'a'"), + "cannot specify FIELDS ... or LINES") + require.ErrorContains(t, tk.ExecToErr("load data infile '/a' format 'parquet' into table load_data_test lines terminated by 'a'"), + "cannot specify FIELDS ... or LINES") + require.ErrorContains(t, tk.ExecToErr("load data infile '/a' format 'parquet' into table load_data_test ignore 0 lines"), + "cannot specify FIELDS ... or LINES") + require.ErrorContains(t, tk.ExecToErr("load data infile '/a' format 'parquet' into table load_data_test ignore 3 lines"), + "cannot specify FIELDS ... or LINES") + require.ErrorContains(t, tk.ExecToErr("load data infile '/a' into table load_data_test fields defined null by 'a' optionally enclosed"), + "must specify FIELDS [OPTIONALLY] ENCLOSED BY") + require.ErrorContains(t, tk.ExecToErr("load data infile '/a' into table load_data_test lines terminated by ''"), + "LINES TERMINATED BY is empty") + require.ErrorContains(t, tk.ExecToErr("load data infile '/a' into table load_data_test fields enclosed by 'a' terminated by 'a'"), + "must not be prefix of each other") + + // null def values + testFunc := func(sql string, expectedNullDef []string, expectedNullOptEnclosed bool) { + require.NoError(t, tk.ExecToErr(sql)) + defer ctx.SetValue(executor.LoadDataVarKey, nil) + ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) + require.True(t, ok) + require.NotNil(t, ld) + require.Equal(t, expectedNullDef, ld.GetController().FieldNullDef) + require.Equal(t, expectedNullOptEnclosed, ld.GetController().NullValueOptEnclosed) + } + testFunc("load data local infile '/a' into table load_data_test", + []string{"\\N"}, false) + testFunc("load data local infile '/a' into table load_data_test fields enclosed by ''", + []string{"\\N"}, false) + testFunc("load data local infile '/a' into table load_data_test fields defined null by 'a'", + []string{"a", "\\N"}, false) + testFunc("load data local infile '/a' into table load_data_test fields enclosed by 'b' defined null by 'a' optionally enclosed", + []string{"a", "\\N"}, true) + testFunc("load data local infile '/a' into table load_data_test fields enclosed by 'b'", + []string{"NULL", "\\N"}, false) + testFunc("load data local infile '/a' into table load_data_test fields enclosed by 'b' escaped by ''", + []string{"NULL"}, false) + + // positive case + require.NoError(t, tk.ExecToErr("load data local infile '/a' format 'parquet' into table load_data_test")) + ctx.SetValue(executor.LoadDataVarKey, nil) + require.NoError(t, tk.ExecToErr("load data local infile '/a' into table load_data_test fields terminated by 'a'")) + ctx.SetValue(executor.LoadDataVarKey, nil) + require.NoError(t, tk.ExecToErr("load data local infile '/a' format 'delimited data' into table load_data_test fields terminated by 'a'")) + ctx.SetValue(executor.LoadDataVarKey, nil) + + // According to https://dev.mysql.com/doc/refman/8.0/en/load-data.html , fixed-row format should be used when fields + // terminated by '' and enclosed by ''. However, tidb doesn't support it yet and empty terminator leads to infinite + // loop in `indexOfTerminator` (see https://github.com/pingcap/tidb/issues/33298). + require.ErrorIs(t, tk.ExecToErr("load data local infile '/tmp/nonexistence.csv' into table load_data_test fields terminated by ''"), + exeerrors.ErrLoadDataWrongFormatConfig) + require.ErrorIs(t, tk.ExecToErr("load data local infile '/tmp/nonexistence.csv' into table load_data_test fields terminated by '' enclosed by ''"), + exeerrors.ErrLoadDataWrongFormatConfig) +} + +func TestLoadData(t *testing.T) { + trivialMsg := "Records: 1 Deleted: 0 Skipped: 0 Warnings: 0" + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + createSQL := `drop table if exists load_data_test; + create table load_data_test (id int PRIMARY KEY AUTO_INCREMENT, c1 int, c2 varchar(255) default "def", c3 int);` + err := tk.ExecToErr("load data local infile '/tmp/nonexistence.csv' into table load_data_test") + require.Error(t, err) + tk.MustExec(createSQL) + err = tk.ExecToErr("load data infile '/tmp/nonexistence.csv' into table load_data_test") + require.Error(t, err) + tk.MustExec("load data local infile '/tmp/nonexistence.csv' ignore into table load_data_test") + ctx := tk.Session().(sessionctx.Context) + ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) + require.True(t, ok) + defer ctx.SetValue(executor.LoadDataVarKey, nil) + require.NotNil(t, ld) + + deleteSQL := "delete from load_data_test" + selectSQL := "select * from load_data_test;" + ctx.GetSessionVars().StmtCtx.DupKeyAsWarning = true + ctx.GetSessionVars().StmtCtx.BadNullAsWarning = true + parser, err := mydump.NewCSVParser( + context.Background(), + ld.GetController().GenerateCSVConfig(), + mydump.NewStringReader(""), + 1, + nil, + false, + nil) + require.NoError(t, err) + err = ld.ReadOneBatchRows(context.Background(), parser) + require.NoError(t, err) + err = ld.CheckAndInsertOneBatch(context.Background(), ld.GetRows(), ld.GetCurBatchCnt()) + require.NoError(t, err) + ld.ResetBatch() + r := tk.MustQuery(selectSQL) + r.Check(nil) + + sc := ctx.GetSessionVars().StmtCtx + originIgnoreTruncate := sc.IgnoreTruncate + defer func() { + sc.IgnoreTruncate = originIgnoreTruncate + }() + sc.IgnoreTruncate = false + // fields and lines are default, ReadOneBatchRows returns data is nil + tests := []testCase{ + // In MySQL we have 4 warnings: 1*"Incorrect integer value: '' for column 'id' at row", 3*"Row 1 doesn't contain data for all columns" + {[]byte("\n"), []string{"1|||"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 2"}, + {[]byte("\t\n"), []string{"2|0||"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 3"}, + {[]byte("3\t2\t3\t4\n"), []string{"3|2|3|4"}, trivialMsg}, + {[]byte("3*1\t2\t3\t4\n"), []string{"3|2|3|4"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + {[]byte("4\t2\t\t3\t4\n"), []string{"4|2||3"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + {[]byte("\t1\t2\t3\t4\n"), []string{"5|1|2|3"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 2"}, + {[]byte("6\t2\t3\n"), []string{"6|2|3|"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + {[]byte("\t2\t3\t4\n\t22\t33\t44\n"), []string{"7|2|3|4", "8|22|33|44"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, + {[]byte("7\t2\t3\t4\n7\t22\t33\t44\n"), []string{"7|2|3|4"}, "Records: 2 Deleted: 0 Skipped: 1 Warnings: 1"}, + + // outdated test but still increase AUTO_INCREMENT + {[]byte("\t2\t3\t4"), []string{"9|2|3|4"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + {[]byte("\t2\t3\t4\t5\n"), []string{"10|2|3|4"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 2"}, + {[]byte("\t2\t34\t5\n"), []string{"11|2|34|5"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + } + checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) + + // lines starting symbol is "" and terminated symbol length is 2, ReadOneBatchRows returns data is nil + ld.GetController().LinesTerminatedBy = "||" + tests = []testCase{ + {[]byte("0\t2\t3\t4\t5||"), []string{"12|2|3|4"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + {[]byte("1\t2\t3\t4\t5||"), []string{"1|2|3|4"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + {[]byte("2\t2\t3\t4\t5||3\t22\t33\t44\t55||"), + []string{"2|2|3|4", "3|22|33|44"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, + {[]byte("3\t2\t3\t4\t5||4\t22\t33||"), []string{ + "3|2|3|4", "4|22|33|"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, + {[]byte("4\t2\t3\t4\t5||5\t22\t33||6\t222||"), + []string{"4|2|3|4", "5|22|33|", "6|222||"}, "Records: 3 Deleted: 0 Skipped: 0 Warnings: 3"}, + {[]byte("6\t2\t34\t5||"), []string{"6|2|34|5"}, trivialMsg}, + } + checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) + + // fields and lines aren't default, ReadOneBatchRows returns data is nil + ld.GetController().FieldsTerminatedBy = "\\" + ld.GetController().LinesStartingBy = "xxx" + ld.GetController().LinesTerminatedBy = "|!#^" + tests = []testCase{ + {[]byte("xxx|!#^"), []string{"13|||"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 2"}, + {[]byte("xxx\\|!#^"), []string{"14|0||"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 3"}, + {[]byte("xxx3\\2\\3\\4|!#^"), []string{"3|2|3|4"}, trivialMsg}, + {[]byte("xxx4\\2\\\\3\\4|!#^"), []string{"4|2||3"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + {[]byte("xxx\\1\\2\\3\\4|!#^"), []string{"15|1|2|3"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 2"}, + {[]byte("xxx6\\2\\3|!#^"), []string{"6|2|3|"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + {[]byte("xxx\\2\\3\\4|!#^xxx\\22\\33\\44|!#^"), []string{ + "16|2|3|4", + "17|22|33|44"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, + {[]byte("\\2\\3\\4|!#^\\22\\33\\44|!#^xxx\\222\\333\\444|!#^"), []string{ + "18|222|333|444"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + + {[]byte("xxx\\2\\3\\4"), []string{"19|2|3|4"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + {[]byte("\\2\\3\\4|!#^"), []string{}, "Records: 0 Deleted: 0 Skipped: 0 Warnings: 0"}, + {[]byte("\\2\\3\\4|!#^xxx18\\22\\33\\44|!#^"), + []string{"18|22|33|44"}, trivialMsg}, + + {[]byte("xxx10\\2\\3\\4|!#^"), + []string{"10|2|3|4"}, trivialMsg}, + {[]byte("10\\2\\3xxx11\\4\\5|!#^"), + []string{"11|4|5|"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + {[]byte("xxx21\\2\\3\\4\\5|!#^"), + []string{"21|2|3|4"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + {[]byte("xxx22\\2\\3\\4\\5|!#^xxx23\\22\\33\\44\\55|!#^"), + []string{"22|2|3|4", "23|22|33|44"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, + {[]byte("xxx23\\2\\3\\4\\5|!#^xxx24\\22\\33|!#^"), + []string{"23|2|3|4", "24|22|33|"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, + {[]byte("xxx24\\2\\3\\4\\5|!#^xxx25\\22\\33|!#^xxx26\\222|!#^"), + []string{"24|2|3|4", "25|22|33|", "26|222||"}, "Records: 3 Deleted: 0 Skipped: 0 Warnings: 3"}, + {[]byte("xxx25\\2\\3\\4\\5|!#^26\\22\\33|!#^xxx27\\222|!#^"), + []string{"25|2|3|4", "27|222||"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, + {[]byte("xxx\\2\\34\\5|!#^"), []string{"28|2|34|5"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + } + checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) + + // TODO: not support it now + // lines starting symbol is the same as terminated symbol, ReadOneBatchRows returns data is nil + //ld.LinesInfo.Terminated = "xxx" + //tests = []testCase{ + // // data1 = nil, data2 != nil + // {[]byte("xxxxxx"), []string{"29|||"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + // {[]byte("xxx3\\2\\3\\4xxx"), []string{"3|2|3|4"}, nil, trivialMsg}, + // {[]byte("xxx\\2\\3\\4xxxxxx\\22\\33\\44xxx"), + // []string{"30|2|3|4", "31|22|33|44"}, nil, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, + // + // // data1 != nil, data2 = nil + // {[]byte("xxx\\2\\3\\4"), nil, []string{"32|2|3|4"}, nil, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + // + // // data1 != nil, data2 != nil + // {[]byte("xxx10\\2\\3"), []byte("\\4\\5xxx"), []string{"10|2|3|4"}, nil, trivialMsg}, + // {[]byte("xxxxx10\\2\\3"), []byte("\\4\\5xxx"), []string{"33|2|3|4"}, nil, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + // {[]byte("xxx21\\2\\3\\4\\5xx"), []byte("x"), []string{"21|2|3|4"}, nil, trivialMsg}, + // {[]byte("xxx32\\2\\3\\4\\5x"), []byte("xxxxx33\\22\\33\\44\\55xxx"), + // []string{"32|2|3|4", "33|22|33|44"}, nil, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, + // {[]byte("xxx33\\2\\3\\4\\5xxx"), []byte("xxx34\\22\\33xxx"), + // []string{"33|2|3|4", "34|22|33|"}, nil, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, + // {[]byte("xxx34\\2\\3\\4\\5xx"), []byte("xxxx35\\22\\33xxxxxx36\\222xxx"), + // []string{"34|2|3|4", "35|22|33|", "36|222||"}, nil, "Records: 3 Deleted: 0 Skipped: 0 Warnings: 0"}, + // + // // ReadOneBatchRows returns data isn't nil + // {[]byte("\\2\\3\\4xxxx"), nil, []byte("xxxx"), "Records: 0 Deleted: 0 Skipped: 0 Warnings: 0"}, + // {[]byte("\\2\\3\\4xxx"), nil, []string{"37|||"}, nil, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, + // {[]byte("\\2\\3\\4xxxxxx11\\22\\33\\44xxx"), nil, + // []string{"38|||", "39|||"}, nil, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, + // {[]byte("xx10\\2\\3"), []byte("\\4\\5xxx"), nil, []byte("xxx"), "Records: 0 Deleted: 0 Skipped: 0 Warnings: 0"}, + // {[]byte("xxx10\\2\\3"), []byte("\\4xxxx"), []string{"10|2|3|4"}, []byte("x"), trivialMsg}, + // {[]byte("xxx10\\2\\3\\4\\5x"), []byte("xx11\\22\\33xxxxxx12\\222xxx"), + // []string{"10|2|3|4", "40|||"}, []byte("xxx"), "Records: 2 Deleted: 0 Skipped: 0 Warnings: 1"}, + //} + //checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) + + // test line terminator in field quoter + ld.GetController().LinesTerminatedBy = "\n" + ld.GetController().FieldsEnclosedBy = `"` + tests = []testCase{ + {[]byte("xxx1\\1\\\"2\n\"\\3\nxxx4\\4\\\"5\n5\"\\6"), []string{"1|1|2\n|3", "4|4|5\n5|6"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, + } + checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) + + ld.GetController().LinesTerminatedBy = "#\n" + ld.GetController().FieldsTerminatedBy = "#" + tests = []testCase{ + {[]byte("xxx1#\nxxx2#\n"), []string{"1|||", "2|||"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, + {[]byte("xxx1#2#3#4#\nnxxx2#3#4#5#\n"), []string{"1|2|3|4", "2|3|4|5"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, + {[]byte("xxx1#2#\"3#\"#\"4\n\"#\nxxx2#3#\"#4#\n\"#5#\n"), []string{"1|2|3#|4", "2|3|#4#\n|5"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, + } + checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) + + // TODO: now support it now + //ld.LinesInfo.Terminated = "#" + //ld.FieldsInfo.Terminated = "##" + //ld.LinesInfo.Starting = "" + //tests = []testCase{ + // {[]byte("1#2#"), []string{"1|||", "2|||"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, + // // TODO: WTF? + // {[]byte("1##2##3##4#2##3##4##5#"), []string{"1|2|3|4", "2|3|4|5"}, "Records: 14 Deleted: 0 Skipped: 3 Warnings: 9"}, + // {[]byte("1##2##\"3##\"##\"4\n\"#2##3##\"##4#\"##5#"), []string{"1|2|3##|4", "2|3|##4#|5"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, + //} + //checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) +} + +func TestLoadDataEscape(t *testing.T) { + trivialMsg := "Records: 1 Deleted: 0 Skipped: 0 Warnings: 0" + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test; drop table if exists load_data_test;") + tk.MustExec("CREATE TABLE load_data_test (id INT NOT NULL PRIMARY KEY, value TEXT NOT NULL) CHARACTER SET utf8") + tk.MustExec("load data local infile '/tmp/nonexistence.csv' into table load_data_test") + ctx := tk.Session().(sessionctx.Context) + ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) + require.True(t, ok) + defer ctx.SetValue(executor.LoadDataVarKey, nil) + require.NotNil(t, ld) + // test escape + tests := []testCase{ + // data1 = nil, data2 != nil + {[]byte("1\ta string\n"), []string{"1|a string"}, trivialMsg}, + {[]byte("2\tstr \\t\n"), []string{"2|str \t"}, trivialMsg}, + {[]byte("3\tstr \\n\n"), []string{"3|str \n"}, trivialMsg}, + {[]byte("4\tboth \\t\\n\n"), []string{"4|both \t\n"}, trivialMsg}, + {[]byte("5\tstr \\\\\n"), []string{"5|str \\"}, trivialMsg}, + {[]byte("6\t\\r\\t\\n\\0\\Z\\b\n"), []string{"6|" + string([]byte{'\r', '\t', '\n', 0, 26, '\b'})}, trivialMsg}, + {[]byte("7\trtn0ZbN\n"), []string{"7|" + string([]byte{'r', 't', 'n', '0', 'Z', 'b', 'N'})}, trivialMsg}, + {[]byte("8\trtn0Zb\\N\n"), []string{"8|" + string([]byte{'r', 't', 'n', '0', 'Z', 'b', 'N'})}, trivialMsg}, + {[]byte("9\ttab\\ tab\n"), []string{"9|tab tab"}, trivialMsg}, + } + deleteSQL := "delete from load_data_test" + selectSQL := "select * from load_data_test;" + checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) +} + +// TestLoadDataSpecifiedColumns reuse TestLoadDataEscape's test case :-) +func TestLoadDataSpecifiedColumns(t *testing.T) { + trivialMsg := "Records: 1 Deleted: 0 Skipped: 0 Warnings: 0" + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test; drop table if exists load_data_test;") + tk.MustExec(`create table load_data_test (id int PRIMARY KEY AUTO_INCREMENT, c1 int, c2 varchar(255) default "def", c3 int default 0);`) + tk.MustExec("load data local infile '/tmp/nonexistence.csv' into table load_data_test (c1, c2)") + ctx := tk.Session().(sessionctx.Context) + ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) + require.True(t, ok) + defer ctx.SetValue(executor.LoadDataVarKey, nil) + require.NotNil(t, ld) + // test + tests := []testCase{ + {[]byte("7\ta string\n"), []string{"1|7|a string|0"}, trivialMsg}, + {[]byte("8\tstr \\t\n"), []string{"2|8|str \t|0"}, trivialMsg}, + {[]byte("9\tstr \\n\n"), []string{"3|9|str \n|0"}, trivialMsg}, + {[]byte("10\tboth \\t\\n\n"), []string{"4|10|both \t\n|0"}, trivialMsg}, + {[]byte("11\tstr \\\\\n"), []string{"5|11|str \\|0"}, trivialMsg}, + {[]byte("12\t\\r\\t\\n\\0\\Z\\b\n"), []string{"6|12|" + string([]byte{'\r', '\t', '\n', 0, 26, '\b'}) + "|0"}, trivialMsg}, + {[]byte("\\N\ta string\n"), []string{"7||a string|0"}, trivialMsg}, + } + deleteSQL := "delete from load_data_test" + selectSQL := "select * from load_data_test;" + checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) +} + +func TestLoadDataIgnoreLines(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test; drop table if exists load_data_test;") + tk.MustExec("CREATE TABLE load_data_test (id INT NOT NULL PRIMARY KEY, value TEXT NOT NULL) CHARACTER SET utf8") + tk.MustExec("load data local infile '/tmp/nonexistence.csv' into table load_data_test ignore 1 lines") + ctx := tk.Session().(sessionctx.Context) + ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) + require.True(t, ok) + defer ctx.SetValue(executor.LoadDataVarKey, nil) + require.NotNil(t, ld) + tests := []testCase{ + {[]byte("1\tline1\n2\tline2\n"), []string{"2|line2"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 0"}, + {[]byte("1\tline1\n2\tline2\n3\tline3\n"), []string{"2|line2", "3|line3"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, + } + deleteSQL := "delete from load_data_test" + selectSQL := "select * from load_data_test;" + checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) +} + +func TestLoadDataNULL(t *testing.T) { + // https://dev.mysql.com/doc/refman/8.0/en/load-data.html + // - For the default FIELDS and LINES values, NULL is written as a field value of \N for output, and a field value of \N is read as NULL for input (assuming that the ESCAPED BY character is \). + // - If FIELDS ENCLOSED BY is not empty, a field containing the literal word NULL as its value is read as a NULL value. This differs from the word NULL enclosed within FIELDS ENCLOSED BY characters, which is read as the string 'NULL'. + // - If FIELDS ESCAPED BY is empty, NULL is written as the word NULL. + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test; drop table if exists load_data_test;") + tk.MustExec("CREATE TABLE load_data_test (id VARCHAR(20), value VARCHAR(20)) CHARACTER SET utf8") + tk.MustExec(`load data local infile '/tmp/nonexistence.csv' into table load_data_test +FIELDS TERMINATED BY ',' ENCLOSED BY '"' LINES TERMINATED BY '\n';`) + ctx := tk.Session().(sessionctx.Context) + ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) + require.True(t, ok) + defer ctx.SetValue(executor.LoadDataVarKey, nil) + require.NotNil(t, ld) + tests := []testCase{ + { + []byte(`NULL,"NULL" +\N,"\N" +"\\N"`), + []string{"|NULL", "|", "\\N|"}, + "Records: 3 Deleted: 0 Skipped: 0 Warnings: 1", + }, + } + deleteSQL := "delete from load_data_test" + selectSQL := "select * from load_data_test;" + checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) +} + +func TestLoadDataReplace(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("USE test; DROP TABLE IF EXISTS load_data_replace;") + tk.MustExec("CREATE TABLE load_data_replace (id INT NOT NULL PRIMARY KEY, value TEXT NOT NULL)") + tk.MustExec("INSERT INTO load_data_replace VALUES(1,'val 1'),(2,'val 2')") + tk.MustExec("LOAD DATA LOCAL INFILE '/tmp/nonexistence.csv' REPLACE INTO TABLE load_data_replace") + ctx := tk.Session().(sessionctx.Context) + ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) + require.True(t, ok) + defer ctx.SetValue(executor.LoadDataVarKey, nil) + require.NotNil(t, ld) + tests := []testCase{ + {[]byte("1\tline1\n2\tline2\n"), []string{"1|line1", "2|line2"}, "Records: 2 Deleted: 2 Skipped: 0 Warnings: 0"}, + {[]byte("2\tnew line2\n3\tnew line3\n"), []string{"1|line1", "2|new line2", "3|new line3"}, "Records: 2 Deleted: 1 Skipped: 0 Warnings: 0"}, + } + deleteSQL := "DO 1" + selectSQL := "TABLE load_data_replace;" + checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) +} + +// TestLoadDataOverflowBigintUnsigned related to issue 6360 +func TestLoadDataOverflowBigintUnsigned(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test; drop table if exists load_data_test;") + tk.MustExec("CREATE TABLE load_data_test (a bigint unsigned);") + tk.MustExec("load data local infile '/tmp/nonexistence.csv' into table load_data_test") + ctx := tk.Session().(sessionctx.Context) + ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) + require.True(t, ok) + defer ctx.SetValue(executor.LoadDataVarKey, nil) + require.NotNil(t, ld) + tests := []testCase{ + {[]byte("-1\n-18446744073709551615\n-18446744073709551616\n"), []string{"0", "0", "0"}, "Records: 3 Deleted: 0 Skipped: 0 Warnings: 3"}, + {[]byte("-9223372036854775809\n18446744073709551616\n"), []string{"0", "18446744073709551615"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, + } + deleteSQL := "delete from load_data_test" + selectSQL := "select * from load_data_test;" + checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) +} + +func TestLoadDataWithUppercaseUserVars(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test; drop table if exists load_data_test;") + tk.MustExec("CREATE TABLE load_data_test (a int, b int);") + tk.MustExec("load data local infile '/tmp/nonexistence.csv' into table load_data_test (@V1)" + + " set a = @V1, b = @V1*100") + ctx := tk.Session().(sessionctx.Context) + ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) + require.True(t, ok) + defer ctx.SetValue(executor.LoadDataVarKey, nil) + require.NotNil(t, ld) + tests := []testCase{ + {[]byte("1\n2\n"), []string{"1|100", "2|200"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, + } + deleteSQL := "delete from load_data_test" + selectSQL := "select * from load_data_test;" + checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) +} + +func TestLoadDataIntoPartitionedTable(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table range_t (a int, b int) partition by range (a) ( " + + "partition p0 values less than (4)," + + "partition p1 values less than (7)," + + "partition p2 values less than (11))") + tk.MustExec("load data local infile '/tmp/nonexistence.csv' into table range_t fields terminated by ','") + ctx := tk.Session().(sessionctx.Context) + ld := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) + require.Nil(t, sessiontxn.NewTxn(context.Background(), ctx)) + + parser, err := mydump.NewCSVParser( + context.Background(), + ld.GetController().GenerateCSVConfig(), + mydump.NewStringReader("1,2\n3,4\n5,6\n7,8\n9,10\n"), + 1, + nil, + false, + nil) + require.NoError(t, err) + + err = ld.ReadOneBatchRows(context.Background(), parser) + require.NoError(t, err) + err = ld.CheckAndInsertOneBatch(context.Background(), ld.GetRows(), ld.GetCurBatchCnt()) + require.NoError(t, err) + ld.ResetBatch() + ld.SetMessage() + ctx.StmtCommit(context.Background()) + txn, err := ctx.Txn(true) + require.NoError(t, err) + err = txn.Commit(context.Background()) + require.NoError(t, err) +} diff --git a/executor/loaddatatest/main_test.go b/executor/loaddatatest/main_test.go new file mode 100644 index 0000000000000..e4f80d9c8b031 --- /dev/null +++ b/executor/loaddatatest/main_test.go @@ -0,0 +1,60 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package loaddatatest + +import ( + "fmt" + "testing" + + "github.com/pingcap/tidb/config" + "github.com/pingcap/tidb/meta/autoid" + "github.com/pingcap/tidb/testkit" + "github.com/tikv/client-go/v2/tikv" + "go.opencensus.io/stats/view" + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + autoid.SetStep(5000) + config.UpdateGlobal(func(conf *config.Config) { + conf.Log.SlowThreshold = 30000 // 30s + conf.TiKVClient.AsyncCommit.SafeWindow = 0 + conf.TiKVClient.AsyncCommit.AllowedClockDrift = 0 + conf.Experimental.AllowsExpressionIndex = true + }) + tikv.EnableFailpoints() + + opts := []goleak.Option{ + goleak.Cleanup(func(_ int) { + view.Stop() + }), + goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), + goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"), + goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("gopkg.in/natefinch/lumberjack%2ev2.(*Logger).millRun"), + goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"), + } + + goleak.VerifyTestMain(m, opts...) +} + +func fillData(tk *testkit.TestKit, table string) { + tk.MustExec("use test") + tk.MustExec(fmt.Sprintf("create table %s(id int not null default 1, name varchar(255), PRIMARY KEY(id));", table)) + + // insert data + tk.MustExec(fmt.Sprintf("insert INTO %s VALUES (1, \"hello\");", table)) + tk.MustExec(fmt.Sprintf("insert into %s values (2, \"hello\");", table)) +} diff --git a/executor/passwordtest/BUILD.bazel b/executor/passwordtest/BUILD.bazel new file mode 100644 index 0000000000000..52ba94935a5f6 --- /dev/null +++ b/executor/passwordtest/BUILD.bazel @@ -0,0 +1,25 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_test") + +go_test( + name = "passwordtest_test", + timeout = "short", + srcs = [ + "main_test.go", + "password_management_test.go", + ], + flaky = True, + shard_count = 16, + deps = [ + "//domain", + "//errno", + "//kv", + "//parser/auth", + "//parser/mysql", + "//privilege/privileges", + "//sessionctx/variable", + "//testkit", + "//util/sqlexec", + "@com_github_stretchr_testify//require", + "@org_uber_go_goleak//:goleak", + ], +) diff --git a/executor/passwordtest/main_test.go b/executor/passwordtest/main_test.go new file mode 100644 index 0000000000000..610ba02f3ef6e --- /dev/null +++ b/executor/passwordtest/main_test.go @@ -0,0 +1,31 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package passwordtest + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + opts := []goleak.Option{ + goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), + goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"), + goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + } + goleak.VerifyTestMain(m, opts...) +} diff --git a/executor/simpletest/password_management_test.go b/executor/passwordtest/password_management_test.go similarity index 99% rename from executor/simpletest/password_management_test.go rename to executor/passwordtest/password_management_test.go index 1e79662b5d5a3..b817a8facb965 100644 --- a/executor/simpletest/password_management_test.go +++ b/executor/passwordtest/password_management_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package simpletest +package passwordtest import ( "bytes" diff --git a/executor/showtest/BUILD.bazel b/executor/showtest/BUILD.bazel index 468d704f3fbda..f90b2d6bfb09a 100644 --- a/executor/showtest/BUILD.bazel +++ b/executor/showtest/BUILD.bazel @@ -2,14 +2,14 @@ load("@io_bazel_rules_go//go:def.bzl", "go_test") go_test( name = "showtest_test", - timeout = "short", + timeout = "moderate", srcs = [ "main_test.go", "show_test.go", ], flaky = True, race = "on", - shard_count = 45, + shard_count = 49, deps = [ "//autoid_service", "//config", diff --git a/executor/simpletest/BUILD.bazel b/executor/simpletest/BUILD.bazel index af70683cbe995..831de15602479 100644 --- a/executor/simpletest/BUILD.bazel +++ b/executor/simpletest/BUILD.bazel @@ -6,31 +6,24 @@ go_test( srcs = [ "chunk_reuse_test.go", "main_test.go", - "password_management_test.go", "simple_test.go", ], flaky = True, race = "on", - shard_count = 30, + shard_count = 34, deps = [ "//config", - "//domain", - "//errno", - "//kv", "//parser/auth", "//parser/model", "//parser/mysql", "//parser/terror", "//planner/core", - "//privilege/privileges", "//session", "//sessionctx", - "//sessionctx/variable", "//statistics/handle", "//store/mockstore", "//testkit", "//util/dbterror/exeerrors", - "//util/sqlexec", "@com_github_pingcap_errors//:errors", "@com_github_stretchr_testify//require", "@io_opencensus_go//stats/view", diff --git a/executor/simpletest/main_test.go b/executor/simpletest/main_test.go index 6772de87c1301..45d4cd268a19c 100644 --- a/executor/simpletest/main_test.go +++ b/executor/simpletest/main_test.go @@ -25,6 +25,7 @@ func TestMain(m *testing.M) { goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"), goleak.IgnoreTopFunction("github.com/tikv/client-go/v2/txnkv/transaction.keepAlive"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), } goleak.VerifyTestMain(m, opts...) } diff --git a/executor/tiflashtest/BUILD.bazel b/executor/tiflashtest/BUILD.bazel index 78e275c885fff..8d9308bd0457e 100644 --- a/executor/tiflashtest/BUILD.bazel +++ b/executor/tiflashtest/BUILD.bazel @@ -2,14 +2,14 @@ load("@io_bazel_rules_go//go:def.bzl", "go_test") go_test( name = "tiflashtest_test", - timeout = "short", + timeout = "moderate", srcs = [ "main_test.go", "tiflash_test.go", ], flaky = True, race = "on", - shard_count = 28, + shard_count = 37, deps = [ "//config", "//domain", diff --git a/executor/writetest/BUILD.bazel b/executor/writetest/BUILD.bazel index 2abd7b74218dc..c5b33e0b6871c 100644 --- a/executor/writetest/BUILD.bazel +++ b/executor/writetest/BUILD.bazel @@ -29,7 +29,6 @@ go_test( "//testkit", "//types", "//util", - "//util/dbterror/exeerrors", "//util/mock", "@com_github_pingcap_failpoint//:failpoint", "@com_github_stretchr_testify//require", diff --git a/executor/writetest/write_test.go b/executor/writetest/write_test.go index 0959395bd19d2..5265cba139ab4 100644 --- a/executor/writetest/write_test.go +++ b/executor/writetest/write_test.go @@ -39,7 +39,6 @@ import ( "github.com/pingcap/tidb/testkit" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util" - "github.com/pingcap/tidb/util/dbterror/exeerrors" "github.com/pingcap/tidb/util/mock" "github.com/stretchr/testify/require" ) @@ -2004,81 +2003,6 @@ func TestIssue18681(t *testing.T) { require.Equal(t, uint16(0), sc.WarningCount()) } -func TestLoadDataInitParam(t *testing.T) { - store := testkit.CreateMockStore(t) - tk := testkit.NewTestKit(t, store) - ctx := tk.Session().(sessionctx.Context) - defer ctx.SetValue(executor.LoadDataVarKey, nil) - - tk.MustExec("use test") - tk.MustExec("drop table if exists load_data_test") - tk.MustExec("create table load_data_test (a varchar(10), b varchar(10))") - - require.ErrorIs(t, tk.ExecToErr("load data infile '' into table load_data_test"), - exeerrors.ErrLoadDataEmptyPath) - require.ErrorIs(t, tk.ExecToErr("load data infile '/a' format '' into table load_data_test"), - exeerrors.ErrLoadDataUnsupportedFormat) - require.ErrorIs(t, tk.ExecToErr("load data infile '/a' format 'aaa' into table load_data_test"), - exeerrors.ErrLoadDataUnsupportedFormat) - require.ErrorContains(t, tk.ExecToErr("load data infile '/a' format 'sql file' into table load_data_test fields terminated by 'a'"), - "cannot specify FIELDS ... or LINES") - require.ErrorContains(t, tk.ExecToErr("load data infile '/a' format 'parquet' into table load_data_test fields terminated by 'a'"), - "cannot specify FIELDS ... or LINES") - require.ErrorContains(t, tk.ExecToErr("load data infile '/a' format 'sql file' into table load_data_test lines terminated by 'a'"), - "cannot specify FIELDS ... or LINES") - require.ErrorContains(t, tk.ExecToErr("load data infile '/a' format 'parquet' into table load_data_test lines terminated by 'a'"), - "cannot specify FIELDS ... or LINES") - require.ErrorContains(t, tk.ExecToErr("load data infile '/a' format 'parquet' into table load_data_test ignore 0 lines"), - "cannot specify FIELDS ... or LINES") - require.ErrorContains(t, tk.ExecToErr("load data infile '/a' format 'parquet' into table load_data_test ignore 3 lines"), - "cannot specify FIELDS ... or LINES") - require.ErrorContains(t, tk.ExecToErr("load data infile '/a' into table load_data_test fields defined null by 'a' optionally enclosed"), - "must specify FIELDS [OPTIONALLY] ENCLOSED BY") - require.ErrorContains(t, tk.ExecToErr("load data infile '/a' into table load_data_test lines terminated by ''"), - "LINES TERMINATED BY is empty") - require.ErrorContains(t, tk.ExecToErr("load data infile '/a' into table load_data_test fields enclosed by 'a' terminated by 'a'"), - "must not be prefix of each other") - - // null def values - testFunc := func(sql string, expectedNullDef []string, expectedNullOptEnclosed bool) { - require.NoError(t, tk.ExecToErr(sql)) - defer ctx.SetValue(executor.LoadDataVarKey, nil) - ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) - require.True(t, ok) - require.NotNil(t, ld) - require.Equal(t, expectedNullDef, ld.GetController().FieldNullDef) - require.Equal(t, expectedNullOptEnclosed, ld.GetController().NullValueOptEnclosed) - } - testFunc("load data local infile '/a' into table load_data_test", - []string{"\\N"}, false) - testFunc("load data local infile '/a' into table load_data_test fields enclosed by ''", - []string{"\\N"}, false) - testFunc("load data local infile '/a' into table load_data_test fields defined null by 'a'", - []string{"a", "\\N"}, false) - testFunc("load data local infile '/a' into table load_data_test fields enclosed by 'b' defined null by 'a' optionally enclosed", - []string{"a", "\\N"}, true) - testFunc("load data local infile '/a' into table load_data_test fields enclosed by 'b'", - []string{"NULL", "\\N"}, false) - testFunc("load data local infile '/a' into table load_data_test fields enclosed by 'b' escaped by ''", - []string{"NULL"}, false) - - // positive case - require.NoError(t, tk.ExecToErr("load data local infile '/a' format 'parquet' into table load_data_test")) - ctx.SetValue(executor.LoadDataVarKey, nil) - require.NoError(t, tk.ExecToErr("load data local infile '/a' into table load_data_test fields terminated by 'a'")) - ctx.SetValue(executor.LoadDataVarKey, nil) - require.NoError(t, tk.ExecToErr("load data local infile '/a' format 'delimited data' into table load_data_test fields terminated by 'a'")) - ctx.SetValue(executor.LoadDataVarKey, nil) - - // According to https://dev.mysql.com/doc/refman/8.0/en/load-data.html , fixed-row format should be used when fields - // terminated by '' and enclosed by ''. However, tidb doesn't support it yet and empty terminator leads to infinite - // loop in `indexOfTerminator` (see https://github.com/pingcap/tidb/issues/33298). - require.ErrorIs(t, tk.ExecToErr("load data local infile '/tmp/nonexistence.csv' into table load_data_test fields terminated by ''"), - exeerrors.ErrLoadDataWrongFormatConfig) - require.ErrorIs(t, tk.ExecToErr("load data local infile '/tmp/nonexistence.csv' into table load_data_test fields terminated by '' enclosed by ''"), - exeerrors.ErrLoadDataWrongFormatConfig) -} - func TestIssue34358(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -2098,399 +2022,6 @@ func TestIssue34358(t *testing.T) { }, ld, t, tk, ctx, "select * from load_data_test", "delete from load_data_test") } -func TestLoadData(t *testing.T) { - trivialMsg := "Records: 1 Deleted: 0 Skipped: 0 Warnings: 0" - store := testkit.CreateMockStore(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("use test") - createSQL := `drop table if exists load_data_test; - create table load_data_test (id int PRIMARY KEY AUTO_INCREMENT, c1 int, c2 varchar(255) default "def", c3 int);` - err := tk.ExecToErr("load data local infile '/tmp/nonexistence.csv' into table load_data_test") - require.Error(t, err) - tk.MustExec(createSQL) - err = tk.ExecToErr("load data infile '/tmp/nonexistence.csv' into table load_data_test") - require.Error(t, err) - tk.MustExec("load data local infile '/tmp/nonexistence.csv' ignore into table load_data_test") - ctx := tk.Session().(sessionctx.Context) - ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) - require.True(t, ok) - defer ctx.SetValue(executor.LoadDataVarKey, nil) - require.NotNil(t, ld) - - deleteSQL := "delete from load_data_test" - selectSQL := "select * from load_data_test;" - ctx.GetSessionVars().StmtCtx.DupKeyAsWarning = true - ctx.GetSessionVars().StmtCtx.BadNullAsWarning = true - parser, err := mydump.NewCSVParser( - context.Background(), - ld.GetController().GenerateCSVConfig(), - mydump.NewStringReader(""), - 1, - nil, - false, - nil) - require.NoError(t, err) - err = ld.ReadOneBatchRows(context.Background(), parser) - require.NoError(t, err) - err = ld.CheckAndInsertOneBatch(context.Background(), ld.GetRows(), ld.GetCurBatchCnt()) - require.NoError(t, err) - ld.ResetBatch() - r := tk.MustQuery(selectSQL) - r.Check(nil) - - sc := ctx.GetSessionVars().StmtCtx - originIgnoreTruncate := sc.IgnoreTruncate - defer func() { - sc.IgnoreTruncate = originIgnoreTruncate - }() - sc.IgnoreTruncate = false - // fields and lines are default, ReadOneBatchRows returns data is nil - tests := []testCase{ - // In MySQL we have 4 warnings: 1*"Incorrect integer value: '' for column 'id' at row", 3*"Row 1 doesn't contain data for all columns" - {[]byte("\n"), []string{"1|||"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 2"}, - {[]byte("\t\n"), []string{"2|0||"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 3"}, - {[]byte("3\t2\t3\t4\n"), []string{"3|2|3|4"}, trivialMsg}, - {[]byte("3*1\t2\t3\t4\n"), []string{"3|2|3|4"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - {[]byte("4\t2\t\t3\t4\n"), []string{"4|2||3"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - {[]byte("\t1\t2\t3\t4\n"), []string{"5|1|2|3"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 2"}, - {[]byte("6\t2\t3\n"), []string{"6|2|3|"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - {[]byte("\t2\t3\t4\n\t22\t33\t44\n"), []string{"7|2|3|4", "8|22|33|44"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, - {[]byte("7\t2\t3\t4\n7\t22\t33\t44\n"), []string{"7|2|3|4"}, "Records: 2 Deleted: 0 Skipped: 1 Warnings: 1"}, - - // outdated test but still increase AUTO_INCREMENT - {[]byte("\t2\t3\t4"), []string{"9|2|3|4"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - {[]byte("\t2\t3\t4\t5\n"), []string{"10|2|3|4"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 2"}, - {[]byte("\t2\t34\t5\n"), []string{"11|2|34|5"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - } - checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) - - // lines starting symbol is "" and terminated symbol length is 2, ReadOneBatchRows returns data is nil - ld.GetController().LinesTerminatedBy = "||" - tests = []testCase{ - {[]byte("0\t2\t3\t4\t5||"), []string{"12|2|3|4"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - {[]byte("1\t2\t3\t4\t5||"), []string{"1|2|3|4"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - {[]byte("2\t2\t3\t4\t5||3\t22\t33\t44\t55||"), - []string{"2|2|3|4", "3|22|33|44"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, - {[]byte("3\t2\t3\t4\t5||4\t22\t33||"), []string{ - "3|2|3|4", "4|22|33|"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, - {[]byte("4\t2\t3\t4\t5||5\t22\t33||6\t222||"), - []string{"4|2|3|4", "5|22|33|", "6|222||"}, "Records: 3 Deleted: 0 Skipped: 0 Warnings: 3"}, - {[]byte("6\t2\t34\t5||"), []string{"6|2|34|5"}, trivialMsg}, - } - checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) - - // fields and lines aren't default, ReadOneBatchRows returns data is nil - ld.GetController().FieldsTerminatedBy = "\\" - ld.GetController().LinesStartingBy = "xxx" - ld.GetController().LinesTerminatedBy = "|!#^" - tests = []testCase{ - {[]byte("xxx|!#^"), []string{"13|||"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 2"}, - {[]byte("xxx\\|!#^"), []string{"14|0||"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 3"}, - {[]byte("xxx3\\2\\3\\4|!#^"), []string{"3|2|3|4"}, trivialMsg}, - {[]byte("xxx4\\2\\\\3\\4|!#^"), []string{"4|2||3"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - {[]byte("xxx\\1\\2\\3\\4|!#^"), []string{"15|1|2|3"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 2"}, - {[]byte("xxx6\\2\\3|!#^"), []string{"6|2|3|"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - {[]byte("xxx\\2\\3\\4|!#^xxx\\22\\33\\44|!#^"), []string{ - "16|2|3|4", - "17|22|33|44"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, - {[]byte("\\2\\3\\4|!#^\\22\\33\\44|!#^xxx\\222\\333\\444|!#^"), []string{ - "18|222|333|444"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - - {[]byte("xxx\\2\\3\\4"), []string{"19|2|3|4"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - {[]byte("\\2\\3\\4|!#^"), []string{}, "Records: 0 Deleted: 0 Skipped: 0 Warnings: 0"}, - {[]byte("\\2\\3\\4|!#^xxx18\\22\\33\\44|!#^"), - []string{"18|22|33|44"}, trivialMsg}, - - {[]byte("xxx10\\2\\3\\4|!#^"), - []string{"10|2|3|4"}, trivialMsg}, - {[]byte("10\\2\\3xxx11\\4\\5|!#^"), - []string{"11|4|5|"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - {[]byte("xxx21\\2\\3\\4\\5|!#^"), - []string{"21|2|3|4"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - {[]byte("xxx22\\2\\3\\4\\5|!#^xxx23\\22\\33\\44\\55|!#^"), - []string{"22|2|3|4", "23|22|33|44"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, - {[]byte("xxx23\\2\\3\\4\\5|!#^xxx24\\22\\33|!#^"), - []string{"23|2|3|4", "24|22|33|"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, - {[]byte("xxx24\\2\\3\\4\\5|!#^xxx25\\22\\33|!#^xxx26\\222|!#^"), - []string{"24|2|3|4", "25|22|33|", "26|222||"}, "Records: 3 Deleted: 0 Skipped: 0 Warnings: 3"}, - {[]byte("xxx25\\2\\3\\4\\5|!#^26\\22\\33|!#^xxx27\\222|!#^"), - []string{"25|2|3|4", "27|222||"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, - {[]byte("xxx\\2\\34\\5|!#^"), []string{"28|2|34|5"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - } - checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) - - // TODO: not support it now - // lines starting symbol is the same as terminated symbol, ReadOneBatchRows returns data is nil - //ld.LinesInfo.Terminated = "xxx" - //tests = []testCase{ - // // data1 = nil, data2 != nil - // {[]byte("xxxxxx"), []string{"29|||"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - // {[]byte("xxx3\\2\\3\\4xxx"), []string{"3|2|3|4"}, nil, trivialMsg}, - // {[]byte("xxx\\2\\3\\4xxxxxx\\22\\33\\44xxx"), - // []string{"30|2|3|4", "31|22|33|44"}, nil, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, - // - // // data1 != nil, data2 = nil - // {[]byte("xxx\\2\\3\\4"), nil, []string{"32|2|3|4"}, nil, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - // - // // data1 != nil, data2 != nil - // {[]byte("xxx10\\2\\3"), []byte("\\4\\5xxx"), []string{"10|2|3|4"}, nil, trivialMsg}, - // {[]byte("xxxxx10\\2\\3"), []byte("\\4\\5xxx"), []string{"33|2|3|4"}, nil, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - // {[]byte("xxx21\\2\\3\\4\\5xx"), []byte("x"), []string{"21|2|3|4"}, nil, trivialMsg}, - // {[]byte("xxx32\\2\\3\\4\\5x"), []byte("xxxxx33\\22\\33\\44\\55xxx"), - // []string{"32|2|3|4", "33|22|33|44"}, nil, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, - // {[]byte("xxx33\\2\\3\\4\\5xxx"), []byte("xxx34\\22\\33xxx"), - // []string{"33|2|3|4", "34|22|33|"}, nil, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, - // {[]byte("xxx34\\2\\3\\4\\5xx"), []byte("xxxx35\\22\\33xxxxxx36\\222xxx"), - // []string{"34|2|3|4", "35|22|33|", "36|222||"}, nil, "Records: 3 Deleted: 0 Skipped: 0 Warnings: 0"}, - // - // // ReadOneBatchRows returns data isn't nil - // {[]byte("\\2\\3\\4xxxx"), nil, []byte("xxxx"), "Records: 0 Deleted: 0 Skipped: 0 Warnings: 0"}, - // {[]byte("\\2\\3\\4xxx"), nil, []string{"37|||"}, nil, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 1"}, - // {[]byte("\\2\\3\\4xxxxxx11\\22\\33\\44xxx"), nil, - // []string{"38|||", "39|||"}, nil, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, - // {[]byte("xx10\\2\\3"), []byte("\\4\\5xxx"), nil, []byte("xxx"), "Records: 0 Deleted: 0 Skipped: 0 Warnings: 0"}, - // {[]byte("xxx10\\2\\3"), []byte("\\4xxxx"), []string{"10|2|3|4"}, []byte("x"), trivialMsg}, - // {[]byte("xxx10\\2\\3\\4\\5x"), []byte("xx11\\22\\33xxxxxx12\\222xxx"), - // []string{"10|2|3|4", "40|||"}, []byte("xxx"), "Records: 2 Deleted: 0 Skipped: 0 Warnings: 1"}, - //} - //checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) - - // test line terminator in field quoter - ld.GetController().LinesTerminatedBy = "\n" - ld.GetController().FieldsEnclosedBy = `"` - tests = []testCase{ - {[]byte("xxx1\\1\\\"2\n\"\\3\nxxx4\\4\\\"5\n5\"\\6"), []string{"1|1|2\n|3", "4|4|5\n5|6"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, - } - checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) - - ld.GetController().LinesTerminatedBy = "#\n" - ld.GetController().FieldsTerminatedBy = "#" - tests = []testCase{ - {[]byte("xxx1#\nxxx2#\n"), []string{"1|||", "2|||"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, - {[]byte("xxx1#2#3#4#\nnxxx2#3#4#5#\n"), []string{"1|2|3|4", "2|3|4|5"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, - {[]byte("xxx1#2#\"3#\"#\"4\n\"#\nxxx2#3#\"#4#\n\"#5#\n"), []string{"1|2|3#|4", "2|3|#4#\n|5"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, - } - checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) - - // TODO: now support it now - //ld.LinesInfo.Terminated = "#" - //ld.FieldsInfo.Terminated = "##" - //ld.LinesInfo.Starting = "" - //tests = []testCase{ - // {[]byte("1#2#"), []string{"1|||", "2|||"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, - // // TODO: WTF? - // {[]byte("1##2##3##4#2##3##4##5#"), []string{"1|2|3|4", "2|3|4|5"}, "Records: 14 Deleted: 0 Skipped: 3 Warnings: 9"}, - // {[]byte("1##2##\"3##\"##\"4\n\"#2##3##\"##4#\"##5#"), []string{"1|2|3##|4", "2|3|##4#|5"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, - //} - //checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) -} - -func TestLoadDataEscape(t *testing.T) { - trivialMsg := "Records: 1 Deleted: 0 Skipped: 0 Warnings: 0" - store := testkit.CreateMockStore(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("use test; drop table if exists load_data_test;") - tk.MustExec("CREATE TABLE load_data_test (id INT NOT NULL PRIMARY KEY, value TEXT NOT NULL) CHARACTER SET utf8") - tk.MustExec("load data local infile '/tmp/nonexistence.csv' into table load_data_test") - ctx := tk.Session().(sessionctx.Context) - ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) - require.True(t, ok) - defer ctx.SetValue(executor.LoadDataVarKey, nil) - require.NotNil(t, ld) - // test escape - tests := []testCase{ - // data1 = nil, data2 != nil - {[]byte("1\ta string\n"), []string{"1|a string"}, trivialMsg}, - {[]byte("2\tstr \\t\n"), []string{"2|str \t"}, trivialMsg}, - {[]byte("3\tstr \\n\n"), []string{"3|str \n"}, trivialMsg}, - {[]byte("4\tboth \\t\\n\n"), []string{"4|both \t\n"}, trivialMsg}, - {[]byte("5\tstr \\\\\n"), []string{"5|str \\"}, trivialMsg}, - {[]byte("6\t\\r\\t\\n\\0\\Z\\b\n"), []string{"6|" + string([]byte{'\r', '\t', '\n', 0, 26, '\b'})}, trivialMsg}, - {[]byte("7\trtn0ZbN\n"), []string{"7|" + string([]byte{'r', 't', 'n', '0', 'Z', 'b', 'N'})}, trivialMsg}, - {[]byte("8\trtn0Zb\\N\n"), []string{"8|" + string([]byte{'r', 't', 'n', '0', 'Z', 'b', 'N'})}, trivialMsg}, - {[]byte("9\ttab\\ tab\n"), []string{"9|tab tab"}, trivialMsg}, - } - deleteSQL := "delete from load_data_test" - selectSQL := "select * from load_data_test;" - checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) -} - -// TestLoadDataSpecifiedColumns reuse TestLoadDataEscape's test case :-) -func TestLoadDataSpecifiedColumns(t *testing.T) { - trivialMsg := "Records: 1 Deleted: 0 Skipped: 0 Warnings: 0" - store := testkit.CreateMockStore(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("use test; drop table if exists load_data_test;") - tk.MustExec(`create table load_data_test (id int PRIMARY KEY AUTO_INCREMENT, c1 int, c2 varchar(255) default "def", c3 int default 0);`) - tk.MustExec("load data local infile '/tmp/nonexistence.csv' into table load_data_test (c1, c2)") - ctx := tk.Session().(sessionctx.Context) - ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) - require.True(t, ok) - defer ctx.SetValue(executor.LoadDataVarKey, nil) - require.NotNil(t, ld) - // test - tests := []testCase{ - {[]byte("7\ta string\n"), []string{"1|7|a string|0"}, trivialMsg}, - {[]byte("8\tstr \\t\n"), []string{"2|8|str \t|0"}, trivialMsg}, - {[]byte("9\tstr \\n\n"), []string{"3|9|str \n|0"}, trivialMsg}, - {[]byte("10\tboth \\t\\n\n"), []string{"4|10|both \t\n|0"}, trivialMsg}, - {[]byte("11\tstr \\\\\n"), []string{"5|11|str \\|0"}, trivialMsg}, - {[]byte("12\t\\r\\t\\n\\0\\Z\\b\n"), []string{"6|12|" + string([]byte{'\r', '\t', '\n', 0, 26, '\b'}) + "|0"}, trivialMsg}, - {[]byte("\\N\ta string\n"), []string{"7||a string|0"}, trivialMsg}, - } - deleteSQL := "delete from load_data_test" - selectSQL := "select * from load_data_test;" - checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) -} - -func TestLoadDataIgnoreLines(t *testing.T) { - store := testkit.CreateMockStore(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("use test; drop table if exists load_data_test;") - tk.MustExec("CREATE TABLE load_data_test (id INT NOT NULL PRIMARY KEY, value TEXT NOT NULL) CHARACTER SET utf8") - tk.MustExec("load data local infile '/tmp/nonexistence.csv' into table load_data_test ignore 1 lines") - ctx := tk.Session().(sessionctx.Context) - ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) - require.True(t, ok) - defer ctx.SetValue(executor.LoadDataVarKey, nil) - require.NotNil(t, ld) - tests := []testCase{ - {[]byte("1\tline1\n2\tline2\n"), []string{"2|line2"}, "Records: 1 Deleted: 0 Skipped: 0 Warnings: 0"}, - {[]byte("1\tline1\n2\tline2\n3\tline3\n"), []string{"2|line2", "3|line3"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, - } - deleteSQL := "delete from load_data_test" - selectSQL := "select * from load_data_test;" - checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) -} - -func TestLoadDataNULL(t *testing.T) { - // https://dev.mysql.com/doc/refman/8.0/en/load-data.html - // - For the default FIELDS and LINES values, NULL is written as a field value of \N for output, and a field value of \N is read as NULL for input (assuming that the ESCAPED BY character is \). - // - If FIELDS ENCLOSED BY is not empty, a field containing the literal word NULL as its value is read as a NULL value. This differs from the word NULL enclosed within FIELDS ENCLOSED BY characters, which is read as the string 'NULL'. - // - If FIELDS ESCAPED BY is empty, NULL is written as the word NULL. - store := testkit.CreateMockStore(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("use test; drop table if exists load_data_test;") - tk.MustExec("CREATE TABLE load_data_test (id VARCHAR(20), value VARCHAR(20)) CHARACTER SET utf8") - tk.MustExec(`load data local infile '/tmp/nonexistence.csv' into table load_data_test -FIELDS TERMINATED BY ',' ENCLOSED BY '"' LINES TERMINATED BY '\n';`) - ctx := tk.Session().(sessionctx.Context) - ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) - require.True(t, ok) - defer ctx.SetValue(executor.LoadDataVarKey, nil) - require.NotNil(t, ld) - tests := []testCase{ - { - []byte(`NULL,"NULL" -\N,"\N" -"\\N"`), - []string{"|NULL", "|", "\\N|"}, - "Records: 3 Deleted: 0 Skipped: 0 Warnings: 1", - }, - } - deleteSQL := "delete from load_data_test" - selectSQL := "select * from load_data_test;" - checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) -} - -func TestLoadDataReplace(t *testing.T) { - store := testkit.CreateMockStore(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("USE test; DROP TABLE IF EXISTS load_data_replace;") - tk.MustExec("CREATE TABLE load_data_replace (id INT NOT NULL PRIMARY KEY, value TEXT NOT NULL)") - tk.MustExec("INSERT INTO load_data_replace VALUES(1,'val 1'),(2,'val 2')") - tk.MustExec("LOAD DATA LOCAL INFILE '/tmp/nonexistence.csv' REPLACE INTO TABLE load_data_replace") - ctx := tk.Session().(sessionctx.Context) - ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) - require.True(t, ok) - defer ctx.SetValue(executor.LoadDataVarKey, nil) - require.NotNil(t, ld) - tests := []testCase{ - {[]byte("1\tline1\n2\tline2\n"), []string{"1|line1", "2|line2"}, "Records: 2 Deleted: 2 Skipped: 0 Warnings: 0"}, - {[]byte("2\tnew line2\n3\tnew line3\n"), []string{"1|line1", "2|new line2", "3|new line3"}, "Records: 2 Deleted: 1 Skipped: 0 Warnings: 0"}, - } - deleteSQL := "DO 1" - selectSQL := "TABLE load_data_replace;" - checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) -} - -// TestLoadDataOverflowBigintUnsigned related to issue 6360 -func TestLoadDataOverflowBigintUnsigned(t *testing.T) { - store := testkit.CreateMockStore(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("use test; drop table if exists load_data_test;") - tk.MustExec("CREATE TABLE load_data_test (a bigint unsigned);") - tk.MustExec("load data local infile '/tmp/nonexistence.csv' into table load_data_test") - ctx := tk.Session().(sessionctx.Context) - ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) - require.True(t, ok) - defer ctx.SetValue(executor.LoadDataVarKey, nil) - require.NotNil(t, ld) - tests := []testCase{ - {[]byte("-1\n-18446744073709551615\n-18446744073709551616\n"), []string{"0", "0", "0"}, "Records: 3 Deleted: 0 Skipped: 0 Warnings: 3"}, - {[]byte("-9223372036854775809\n18446744073709551616\n"), []string{"0", "18446744073709551615"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 2"}, - } - deleteSQL := "delete from load_data_test" - selectSQL := "select * from load_data_test;" - checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) -} - -func TestLoadDataWithUppercaseUserVars(t *testing.T) { - store := testkit.CreateMockStore(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("use test; drop table if exists load_data_test;") - tk.MustExec("CREATE TABLE load_data_test (a int, b int);") - tk.MustExec("load data local infile '/tmp/nonexistence.csv' into table load_data_test (@V1)" + - " set a = @V1, b = @V1*100") - ctx := tk.Session().(sessionctx.Context) - ld, ok := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) - require.True(t, ok) - defer ctx.SetValue(executor.LoadDataVarKey, nil) - require.NotNil(t, ld) - tests := []testCase{ - {[]byte("1\n2\n"), []string{"1|100", "2|200"}, "Records: 2 Deleted: 0 Skipped: 0 Warnings: 0"}, - } - deleteSQL := "delete from load_data_test" - selectSQL := "select * from load_data_test;" - checkCases(tests, ld, t, tk, ctx, selectSQL, deleteSQL) -} - -func TestLoadDataIntoPartitionedTable(t *testing.T) { - store := testkit.CreateMockStore(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("use test") - tk.MustExec("create table range_t (a int, b int) partition by range (a) ( " + - "partition p0 values less than (4)," + - "partition p1 values less than (7)," + - "partition p2 values less than (11))") - tk.MustExec("load data local infile '/tmp/nonexistence.csv' into table range_t fields terminated by ','") - ctx := tk.Session().(sessionctx.Context) - ld := ctx.Value(executor.LoadDataVarKey).(*executor.LoadDataWorker) - require.Nil(t, sessiontxn.NewTxn(context.Background(), ctx)) - - parser, err := mydump.NewCSVParser( - context.Background(), - ld.GetController().GenerateCSVConfig(), - mydump.NewStringReader("1,2\n3,4\n5,6\n7,8\n9,10\n"), - 1, - nil, - false, - nil) - require.NoError(t, err) - - err = ld.ReadOneBatchRows(context.Background(), parser) - require.NoError(t, err) - err = ld.CheckAndInsertOneBatch(context.Background(), ld.GetRows(), ld.GetCurBatchCnt()) - require.NoError(t, err) - ld.ResetBatch() - ld.SetMessage() - ctx.StmtCommit(context.Background()) - txn, err := ctx.Txn(true) - require.NoError(t, err) - err = txn.Commit(context.Background()) - require.NoError(t, err) -} - func TestNullDefault(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) diff --git a/session/BUILD.bazel b/session/BUILD.bazel index e52dec372b686..ef4ab3c919764 100644 --- a/session/BUILD.bazel +++ b/session/BUILD.bazel @@ -110,7 +110,7 @@ go_library( go_test( name = "session_test", - timeout = "short", + timeout = "moderate", srcs = [ "bench_test.go", "bootstrap_test.go", diff --git a/session/nontransactionaltest/BUILD.bazel b/session/nontransactionaltest/BUILD.bazel index 99d96fd0f79e2..c5e8ff9ad501c 100644 --- a/session/nontransactionaltest/BUILD.bazel +++ b/session/nontransactionaltest/BUILD.bazel @@ -8,8 +8,7 @@ go_test( "nontransactional_test.go", ], flaky = True, - race = "on", - shard_count = 23, + shard_count = 22, deps = [ "//config", "//testkit", diff --git a/statistics/handle/handletest/BUILD.bazel b/statistics/handle/handletest/BUILD.bazel index 0ab6fff072cb4..e8f00b0655a86 100644 --- a/statistics/handle/handletest/BUILD.bazel +++ b/statistics/handle/handletest/BUILD.bazel @@ -8,7 +8,6 @@ go_test( "main_test.go", ], flaky = True, - race = "on", shard_count = 50, deps = [ "//config", diff --git a/store/copr/BUILD.bazel b/store/copr/BUILD.bazel index 096f789f336b3..9384916aabfc3 100644 --- a/store/copr/BUILD.bazel +++ b/store/copr/BUILD.bazel @@ -28,6 +28,7 @@ go_library( "//store/driver/options", "//util", "//util/execdetails", + "//util/intest", "//util/logutil", "//util/mathutil", "//util/memory", diff --git a/store/copr/batch_coprocessor.go b/store/copr/batch_coprocessor.go index 659edda135dc4..10e60058d700e 100644 --- a/store/copr/batch_coprocessor.go +++ b/store/copr/batch_coprocessor.go @@ -36,6 +36,7 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/store/driver/backoff" derr "github.com/pingcap/tidb/store/driver/error" + "github.com/pingcap/tidb/util/intest" "github.com/pingcap/tidb/util/logutil" "github.com/pingcap/tidb/util/tiflashcompute" "github.com/stathat/consistent" @@ -681,6 +682,9 @@ func buildBatchCopTasksConsistentHash( if len(storesStr) == 0 { retErr := errors.New("Cannot find proper topo from AutoScaler") logutil.BgLogger().Info("buildBatchCopTasksConsistentHash retry because FetchAndGetTopo return empty topo", zap.Int("retryNum", retryNum)) + if intest.InTest && retryNum > 3 { + return nil, retErr + } err := fetchTopoBo.Backoff(tikv.BoTiFlashRPC(), retErr) if err != nil { return nil, retErr diff --git a/util/gpool/spmc/BUILD.bazel b/util/gpool/spmc/BUILD.bazel index a3cc3dad188d9..2cf4fd4c63d04 100644 --- a/util/gpool/spmc/BUILD.bazel +++ b/util/gpool/spmc/BUILD.bazel @@ -37,6 +37,7 @@ go_test( embed = [":spmc"], flaky = True, race = "on", + shard_count = 2, deps = [ "//resourcemanager/pooltask", "//resourcemanager/util", diff --git a/util/gpool/spmc/spmcpool_test.go b/util/gpool/spmc/spmcpool_test.go index 164f92b909228..70346d1df5e9a 100644 --- a/util/gpool/spmc/spmcpool_test.go +++ b/util/gpool/spmc/spmcpool_test.go @@ -436,7 +436,7 @@ func TestBenchPool(t *testing.T) { return struct{}{} }) - for i := 0; i < 1000; i++ { + for i := 0; i < 500; i++ { sema := make(chan struct{}, 10) var wg sync.WaitGroup exitCh := make(chan struct{})