diff --git a/cmd/explaintest/r/vitess_hash.result b/cmd/explaintest/r/vitess_hash.result new file mode 100644 index 0000000000000..1e20ff988c8f4 --- /dev/null +++ b/cmd/explaintest/r/vitess_hash.result @@ -0,0 +1,30 @@ +use test; +drop table if exists t; +create table t( +customer_id bigint, +id bigint, +expected_shard bigint unsigned, +computed_shard bigint unsigned null, +primary key (customer_id, id) +); +create index t_vitess_shard on t((vitess_hash(customer_id) >> 56)); +explain format = 'brief' select customer_id from t where (vitess_hash(customer_id) >> 56) = x'd6' ORDER BY id; +id estRows task access object operator info +Projection 10.00 root test.t.customer_id +└─Sort 10.00 root test.t.id + └─IndexLookUp 10.00 root + ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:t_vitess_shard(vitess_hash(`customer_id`) >> 56) range:[214,214], keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo +explain format = 'brief' select id from t where (vitess_hash(customer_id) >> 56) IN (x'e0', x'e1') AND id BETWEEN 2 AND 5 ORDER BY id; +id estRows task access object operator info +Projection 0.50 root test.t.id +└─Sort 0.50 root test.t.id + └─IndexLookUp 0.50 root + ├─IndexRangeScan(Build) 20.00 cop[tikv] table:t, index:t_vitess_shard(vitess_hash(`customer_id`) >> 56) range:[224,224], [225,225], keep order:false, stats:pseudo + └─Selection(Probe) 0.50 cop[tikv] ge(test.t.id, 2), le(test.t.id, 5) + └─TableRowIDScan 20.00 cop[tikv] table:t keep order:false, stats:pseudo +explain format = 'brief' select hex(vitess_hash(1123)) from t; +id estRows task access object operator info +Projection 10000.00 root 31B565D41BDF8CA->Column#7 +└─IndexReader 10000.00 root index:IndexFullScan + └─IndexFullScan 10000.00 cop[tikv] table:t, index:t_vitess_shard(vitess_hash(`customer_id`) >> 56) keep order:false, stats:pseudo diff --git a/cmd/explaintest/t/vitess_hash.test b/cmd/explaintest/t/vitess_hash.test new file mode 100644 index 0000000000000..6fd221a44019d --- /dev/null +++ b/cmd/explaintest/t/vitess_hash.test @@ -0,0 +1,13 @@ +use test; +drop table if exists t; +create table t( + customer_id bigint, + id bigint, + expected_shard bigint unsigned, + computed_shard bigint unsigned null, + primary key (customer_id, id) +); +create index t_vitess_shard on t((vitess_hash(customer_id) >> 56)); +explain format = 'brief' select customer_id from t where (vitess_hash(customer_id) >> 56) = x'd6' ORDER BY id; +explain format = 'brief' select id from t where (vitess_hash(customer_id) >> 56) IN (x'e0', x'e1') AND id BETWEEN 2 AND 5 ORDER BY id; +explain format = 'brief' select hex(vitess_hash(1123)) from t; diff --git a/config/config.toml.example b/config/config.toml.example index e4ebe7a5defc6..7b859063ff0d7 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -294,11 +294,6 @@ max-txn-ttl = 3600000 # the interval duration between two memory profile into global tracker mem-profile-interval = "1m" -# Index usage sync lease duration, which influences the time of dump index usage information to KV. -# Here we set to 0 to not dump index usage information to KV, -# because we have not completed GC and other functions. -index-usage-sync-lease = "0s" - # The Go GC trigger factor, you can get more information about it at https://golang.org/pkg/runtime. # If you encounter OOM when executing large query, you can decrease this value to trigger GC earlier. # If you find the CPU used by GC is too high or GC is too frequent and impact your business you can increase this value. diff --git a/ddl/column_test.go b/ddl/column_test.go index 2e287ac82d4d0..862fb4aa04c59 100644 --- a/ddl/column_test.go +++ b/ddl/column_test.go @@ -1171,8 +1171,6 @@ func (s *testColumnSuite) TestModifyColumn(c *C) { }{ {"int", "bigint", nil}, {"int", "int unsigned", errUnsupportedModifyColumn.GenWithStackByArgs("can't change unsigned integer to signed or vice versa, and tidb_enable_change_column_type is false")}, - {"varchar(10)", "text", nil}, - {"varbinary(10)", "blob", nil}, {"text", "blob", errUnsupportedModifyCharset.GenWithStackByArgs("charset from utf8mb4 to binary")}, {"varchar(10)", "varchar(8)", errUnsupportedModifyColumn.GenWithStackByArgs("length 8 is less than origin 10, and tidb_enable_change_column_type is false")}, {"varchar(10)", "varchar(11)", nil}, diff --git a/ddl/column_type_change_test.go b/ddl/column_type_change_test.go index fb82096dd3755..402704401eff6 100644 --- a/ddl/column_type_change_test.go +++ b/ddl/column_type_change_test.go @@ -317,14 +317,10 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromIntegerToOthers(c *C // integer to string prepare(tk) - tk.MustExec("alter table t modify a varchar(10)") - modifiedColumn := getModifyColumn(c, tk.Se, "test", "t", "a", false) - c.Assert(modifiedColumn, NotNil) - c.Assert(modifiedColumn.Tp, Equals, parser_mysql.TypeVarchar) - tk.MustQuery("select a from t").Check(testkit.Rows("1")) + tk.MustGetErrCode("alter table t modify a varchar(10)", mysql.ErrUnsupportedDDLOperation) tk.MustExec("alter table t modify b char(10)") - modifiedColumn = getModifyColumn(c, tk.Se, "test", "t", "b", false) + modifiedColumn := getModifyColumn(c, tk.Se, "test", "t", "b", false) c.Assert(modifiedColumn, NotNil) c.Assert(modifiedColumn.Tp, Equals, parser_mysql.TypeString) tk.MustQuery("select b from t").Check(testkit.Rows("11")) @@ -335,11 +331,7 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromIntegerToOthers(c *C c.Assert(modifiedColumn.Tp, Equals, parser_mysql.TypeString) tk.MustQuery("select c from t").Check(testkit.Rows("111\x00\x00\x00\x00\x00\x00\x00")) - tk.MustExec("alter table t modify d varbinary(10)") - modifiedColumn = getModifyColumn(c, tk.Se, "test", "t", "d", false) - c.Assert(modifiedColumn, NotNil) - c.Assert(modifiedColumn.Tp, Equals, parser_mysql.TypeVarchar) - tk.MustQuery("select d from t").Check(testkit.Rows("1111")) + tk.MustGetErrCode("alter table t modify d varbinary(10)", mysql.ErrUnsupportedDDLOperation) tk.MustExec("alter table t modify e blob(10)") modifiedColumn = getModifyColumn(c, tk.Se, "test", "t", "e", false) @@ -450,6 +442,27 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromIntegerToOthers(c *C tk.MustGetErrCode("alter table t modify e set(\"11111\", \"22222\")", mysql.WarnDataTruncated) } +func (s *testColumnTypeChangeSuite) TestColumnTypeChangeBetweenVarcharAndNonVarchar(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + tk.MustExec("drop database if exists col_type_change_char;") + tk.MustExec("create database col_type_change_char;") + tk.MustExec("use col_type_change_char;") + tk.MustExec("create table t(a char(10), b varchar(10));") + tk.MustExec("insert into t values ('aaa ', 'bbb ');") + tk.MustExec("alter table t change column a a char(10);") + tk.MustExec("alter table t change column b b varchar(10);") + tk.MustGetErrCode("alter table t change column a a varchar(10);", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t change column b b char(10);", mysql.ErrUnsupportedDDLOperation) + + tk.MustExec("alter table t add index idx_a(a);") + tk.MustExec("alter table t add index idx_b(b);") + tk.MustGetErrCode("alter table t change column a a varchar(10);", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t change column b b char(10);", mysql.ErrUnsupportedDDLOperation) + tk.MustExec("admin check table t;") +} + func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromStringToOthers(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") @@ -467,12 +480,11 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromStringToOthers(c *C) // Init string date type table. reset := func(tk *testkit.TestKit) { tk.MustExec("drop table if exists t") + // FIXME(tangenta): not support changing from varchar/varbinary to other types. tk.MustExec(` create table t ( c char(8), - vc varchar(8), bny binary(8), - vbny varbinary(8), bb blob, txt text, e enum('123', '2020-07-15 18:32:17.888', 'str', '{"k1": "value"}'), @@ -484,181 +496,147 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromStringToOthers(c *C) // To numeric data types. // tinyint reset(tk) - tk.MustExec("insert into t values ('123', '123', '123', '123', '123', '123', '123', '123')") + tk.MustExec("insert into t values ('123', '123', '123', '123', '123', '123')") tk.MustExec("alter table t modify c tinyint") - tk.MustExec("alter table t modify vc tinyint") tk.MustExec("alter table t modify bny tinyint") - tk.MustExec("alter table t modify vbny tinyint") tk.MustExec("alter table t modify bb tinyint") tk.MustExec("alter table t modify txt tinyint") tk.MustExec("alter table t modify e tinyint") tk.MustExec("alter table t modify s tinyint") - tk.MustQuery("select * from t").Check(testkit.Rows("123 123 123 123 123 123 1 1")) + tk.MustQuery("select * from t").Check(testkit.Rows("123 123 123 123 1 1")) // int reset(tk) - tk.MustExec("insert into t values ('17305', '17305', '17305', '17305', '17305', '17305', '123', '123')") + tk.MustExec("insert into t values ('17305', '17305', '17305', '17305', '123', '123')") tk.MustExec("alter table t modify c int") - tk.MustExec("alter table t modify vc int") tk.MustExec("alter table t modify bny int") - tk.MustExec("alter table t modify vbny int") tk.MustExec("alter table t modify bb int") tk.MustExec("alter table t modify txt int") tk.MustExec("alter table t modify e int") tk.MustExec("alter table t modify s int") - tk.MustQuery("select * from t").Check(testkit.Rows("17305 17305 17305 17305 17305 17305 1 1")) + tk.MustQuery("select * from t").Check(testkit.Rows("17305 17305 17305 17305 1 1")) // bigint reset(tk) - tk.MustExec("insert into t values ('17305867', '17305867', '17305867', '17305867', '17305867', '17305867', '123', '123')") + tk.MustExec("insert into t values ('17305867', '17305867', '17305867', '17305867', '123', '123')") tk.MustExec("alter table t modify c bigint") - tk.MustExec("alter table t modify vc bigint") tk.MustExec("alter table t modify bny bigint") - tk.MustExec("alter table t modify vbny bigint") tk.MustExec("alter table t modify bb bigint") tk.MustExec("alter table t modify txt bigint") tk.MustExec("alter table t modify e bigint") tk.MustExec("alter table t modify s bigint") - tk.MustQuery("select * from t").Check(testkit.Rows("17305867 17305867 17305867 17305867 17305867 17305867 1 1")) + tk.MustQuery("select * from t").Check(testkit.Rows("17305867 17305867 17305867 17305867 1 1")) // bit reset(tk) - tk.MustExec("insert into t values ('1', '1', '1', '1', '1', '1', '123', '123')") + tk.MustExec("insert into t values ('1', '1', '1', '1', '123', '123')") tk.MustGetErrCode("alter table t modify c bit", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify vc bit", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify bny bit", mysql.ErrUnsupportedDDLOperation) - tk.MustGetErrCode("alter table t modify vbny bit", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify bb bit", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify txt bit", mysql.ErrUnsupportedDDLOperation) tk.MustExec("alter table t modify e bit") tk.MustExec("alter table t modify s bit") - tk.MustQuery("select * from t").Check(testkit.Rows("1 1 1\x00\x00\x00\x00\x00\x00\x00 1 1 1 \x01 \x01")) + tk.MustQuery("select * from t").Check(testkit.Rows("1 1\x00\x00\x00\x00\x00\x00\x00 1 1 \x01 \x01")) // decimal reset(tk) - tk.MustExec("insert into t values ('123.45', '123.45', '123.45', '123.45', '123.45', '123.45', '123', '123')") + tk.MustExec("insert into t values ('123.45', '123.45', '123.45', '123.45', '123', '123')") tk.MustExec("alter table t modify c decimal(7, 4)") - tk.MustExec("alter table t modify vc decimal(7, 4)") tk.MustExec("alter table t modify bny decimal(7, 4)") - tk.MustExec("alter table t modify vbny decimal(7, 4)") tk.MustExec("alter table t modify bb decimal(7, 4)") tk.MustExec("alter table t modify txt decimal(7, 4)") tk.MustExec("alter table t modify e decimal(7, 4)") tk.MustExec("alter table t modify s decimal(7, 4)") - tk.MustQuery("select * from t").Check(testkit.Rows("123.4500 123.4500 123.4500 123.4500 123.4500 123.4500 1.0000 1.0000")) + tk.MustQuery("select * from t").Check(testkit.Rows("123.4500 123.4500 123.4500 123.4500 1.0000 1.0000")) // double reset(tk) - tk.MustExec("insert into t values ('123.45', '123.45', '123.45', '123.45', '123.45', '123.45', '123', '123')") + tk.MustExec("insert into t values ('123.45', '123.45', '123.45', '123.45', '123', '123')") tk.MustExec("alter table t modify c double(7, 4)") - tk.MustExec("alter table t modify vc double(7, 4)") tk.MustExec("alter table t modify bny double(7, 4)") - tk.MustExec("alter table t modify vbny double(7, 4)") tk.MustExec("alter table t modify bb double(7, 4)") tk.MustExec("alter table t modify txt double(7, 4)") tk.MustExec("alter table t modify e double(7, 4)") tk.MustExec("alter table t modify s double(7, 4)") - tk.MustQuery("select * from t").Check(testkit.Rows("123.45 123.45 123.45 123.45 123.45 123.45 1 1")) + tk.MustQuery("select * from t").Check(testkit.Rows("123.45 123.45 123.45 123.45 1 1")) // To date and time data types. // date reset(tk) - tk.MustExec("insert into t values ('20200826', '2008261', '20200826', '200826', '2020-08-26', '08-26 19:35:41', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") + tk.MustExec("insert into t values ('20200826', '20200826', '2020-08-26', '08-26 19:35:41', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") tk.MustExec("alter table t modify c date") - tk.MustExec("alter table t modify vc date") tk.MustExec("alter table t modify bny date") - tk.MustExec("alter table t modify vbny date") tk.MustExec("alter table t modify bb date") // Alter text '08-26 19:35:41' to date will error. (same as mysql does) tk.MustGetErrCode("alter table t modify txt date", mysql.ErrTruncatedWrongValue) tk.MustGetErrCode("alter table t modify e date", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify s date", mysql.ErrUnsupportedDDLOperation) - tk.MustQuery("select * from t").Check(testkit.Rows("2020-08-26 2020-08-26 2020-08-26 2020-08-26 2020-08-26 08-26 19:35:41 2020-07-15 18:32:17.888 2020-07-15 18:32:17.888")) + tk.MustQuery("select * from t").Check(testkit.Rows("2020-08-26 2020-08-26 2020-08-26 08-26 19:35:41 2020-07-15 18:32:17.888 2020-07-15 18:32:17.888")) // time reset(tk) - tk.MustExec("insert into t values ('19:35:41', '19:35:41', '19:35:41', '19:35:41', '19:35:41.45678', '19:35:41.45678', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") + tk.MustExec("insert into t values ('19:35:41', '19:35:41', '19:35:41.45678', '19:35:41.45678', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") tk.MustExec("alter table t modify c time") - tk.MustExec("alter table t modify vc time") tk.MustExec("alter table t modify bny time") - tk.MustExec("alter table t modify vbny time") tk.MustExec("alter table t modify bb time") tk.MustExec("alter table t modify txt time") tk.MustGetErrCode("alter table t modify e time", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify s time", mysql.ErrUnsupportedDDLOperation) - tk.MustQuery("select * from t").Check(testkit.Rows("19:35:41 19:35:41 19:35:41 19:35:41 19:35:41 19:35:41 2020-07-15 18:32:17.888 2020-07-15 18:32:17.888")) + tk.MustQuery("select * from t").Check(testkit.Rows("19:35:41 19:35:41 19:35:41 19:35:41 2020-07-15 18:32:17.888 2020-07-15 18:32:17.888")) // datetime reset(tk) tk.MustExec("alter table t modify c char(23)") - tk.MustExec("alter table t modify vc varchar(23)") tk.MustExec("alter table t modify bny binary(23)") - tk.MustExec("alter table t modify vbny varbinary(23)") - tk.MustExec("insert into t values ('2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") + tk.MustExec("insert into t values ('2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") tk.MustExec("alter table t modify c datetime") - tk.MustExec("alter table t modify vc datetime") tk.MustExec("alter table t modify bny datetime") - tk.MustExec("alter table t modify vbny datetime") tk.MustExec("alter table t modify bb datetime") tk.MustExec("alter table t modify txt datetime") tk.MustGetErrCode("alter table t modify e datetime", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify s datetime", mysql.ErrUnsupportedDDLOperation) - tk.MustQuery("select * from t").Check(testkit.Rows("2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:17.888 2020-07-15 18:32:17.888")) + tk.MustQuery("select * from t").Check(testkit.Rows("2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:17.888 2020-07-15 18:32:17.888")) // timestamp reset(tk) tk.MustExec("alter table t modify c char(23)") - tk.MustExec("alter table t modify vc varchar(23)") tk.MustExec("alter table t modify bny binary(23)") - tk.MustExec("alter table t modify vbny varbinary(23)") - tk.MustExec("insert into t values ('2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") + tk.MustExec("insert into t values ('2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") tk.MustExec("alter table t modify c timestamp") - tk.MustExec("alter table t modify vc timestamp") tk.MustExec("alter table t modify bny timestamp") - tk.MustExec("alter table t modify vbny timestamp") tk.MustExec("alter table t modify bb timestamp") tk.MustExec("alter table t modify txt timestamp") tk.MustGetErrCode("alter table t modify e timestamp", mysql.ErrUnsupportedDDLOperation) tk.MustGetErrCode("alter table t modify s timestamp", mysql.ErrUnsupportedDDLOperation) - tk.MustQuery("select * from t").Check(testkit.Rows("2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:17.888 2020-07-15 18:32:17.888")) + tk.MustQuery("select * from t").Check(testkit.Rows("2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:18 2020-07-15 18:32:17.888 2020-07-15 18:32:17.888")) // year reset(tk) - tk.MustExec("insert into t values ('2020', '91', '2', '2020', '20', '99', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") + tk.MustExec("insert into t values ('2020', '2', '20', '99', '2020-07-15 18:32:17.888', '2020-07-15 18:32:17.888')") tk.MustExec("alter table t modify c year") - tk.MustExec("alter table t modify vc year") tk.MustExec("alter table t modify bny year") - tk.MustExec("alter table t modify vbny year") tk.MustExec("alter table t modify bb year") tk.MustExec("alter table t modify txt year") tk.MustExec("alter table t modify e year") tk.MustExec("alter table t modify s year") - tk.MustQuery("select * from t").Check(testkit.Rows("2020 1991 2002 2020 2020 1999 2002 2002")) + tk.MustQuery("select * from t").Check(testkit.Rows("2020 2002 2020 1999 2002 2002")) // To json data type. reset(tk) tk.MustExec("alter table t modify c char(15)") - tk.MustExec("alter table t modify vc varchar(15)") tk.MustExec("alter table t modify bny binary(15)") - tk.MustExec("alter table t modify vbny varbinary(15)") - tk.MustExec("insert into t values ('{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}')") + tk.MustExec("insert into t values ('{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}', '{\"k1\": \"value\"}')") tk.MustExec("alter table t modify c json") - tk.MustExec("alter table t modify vc json") tk.MustExec("alter table t modify bny json") - tk.MustExec("alter table t modify vbny json") tk.MustExec("alter table t modify bb json") tk.MustExec("alter table t modify txt json") tk.MustExec("alter table t modify e json") tk.MustExec("alter table t modify s json") - tk.MustQuery("select * from t").Check(testkit.Rows("{\"k1\": \"value\"} {\"k1\": \"value\"} {\"k1\": \"value\"} {\"k1\": \"value\"} {\"k1\": \"value\"} {\"k1\": \"value\"} \"{\\\"k1\\\": \\\"value\\\"}\" \"{\\\"k1\\\": \\\"value\\\"}\"")) + tk.MustQuery("select * from t").Check(testkit.Rows("{\"k1\": \"value\"} {\"k1\": \"value\"} {\"k1\": \"value\"} {\"k1\": \"value\"} \"{\\\"k1\\\": \\\"value\\\"}\" \"{\\\"k1\\\": \\\"value\\\"}\"")) reset(tk) - tk.MustExec("insert into t values ('123x', 'x123', 'abc', 'datetime', 'timestamp', 'date', '123', '123')") + tk.MustExec("insert into t values ('123x', 'abc', 'timestamp', 'date', '123', '123')") tk.MustGetErrCode("alter table t modify c int", mysql.ErrTruncatedWrongValue) - tk.MustGetErrCode("alter table t modify vc smallint", mysql.ErrTruncatedWrongValue) - tk.MustGetErrCode("alter table t modify bny bigint", mysql.ErrTruncatedWrongValue) - tk.MustGetErrCode("alter table t modify vbny datetime", mysql.ErrTruncatedWrongValue) - tk.MustGetErrCode("alter table t modify bb timestamp", mysql.ErrTruncatedWrongValue) tk.MustGetErrCode("alter table t modify txt date", mysql.ErrTruncatedWrongValue) reset(tk) - tk.MustExec("alter table t modify vc varchar(20)") + tk.MustExec("alter table t add vc char(20)") tk.MustExec("insert into t(c, vc) values ('1x', '20200915110836')") tk.MustGetErrCode("alter table t modify c year", mysql.ErrTruncatedWrongValue) @@ -671,6 +649,7 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromStringToOthers(c *C) // Both error but different error message. // MySQL will get "ERROR 3140 (22032): Invalid JSON text: "The document root must not be followed by other values." at position 1 in value for column '#sql-5b_42.c'." error. reset(tk) + tk.MustExec("alter table t add vc char(20)") tk.MustExec("alter table t modify c char(15)") tk.MustExec("insert into t(c) values ('{\"k1\": \"value\"')") tk.MustGetErrCode("alter table t modify c json", mysql.ErrInvalidJSONText) @@ -778,15 +757,15 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromNumericToOthers(c *C // varchar reset(tk) tk.MustExec("insert into t values (-258.12345, 333.33, 2000000.20000002, 323232323.3232323232, -111.11111111, -222222222222.222222222222222, b'10101')") - tk.MustExec("alter table t modify d varchar(30)") - tk.MustExec("alter table t modify n varchar(30)") - tk.MustExec("alter table t modify r varchar(30)") - tk.MustExec("alter table t modify db varchar(30)") + tk.MustGetErrCode("alter table t modify d varchar(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify n varchar(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify r varchar(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify db varchar(30)", mysql.ErrUnsupportedDDLOperation) // MySQL will get "-111.111" rather than "-111.111115" at TiDB. - tk.MustExec("alter table t modify f32 varchar(30)") + tk.MustGetErrCode("alter table t modify f32 varchar(30)", mysql.ErrUnsupportedDDLOperation) // MySQL will get "ERROR 1406 (22001): Data truncation: Data too long for column 'f64' at row 1". - tk.MustExec("alter table t modify f64 varchar(30)") - tk.MustExec("alter table t modify b varchar(30)") + tk.MustGetErrCode("alter table t modify f64 varchar(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify b varchar(30)", mysql.ErrUnsupportedDDLOperation) tk.MustQuery("select * from t").Check(testkit.Rows("-258.1234500 333.33 2000000.20000002 323232323.32323235 -111.111115 -222222222222.22223 \x15")) // binary @@ -805,15 +784,15 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromNumericToOthers(c *C // varbinary reset(tk) tk.MustExec("insert into t values (-258.12345, 333.33, 2000000.20000002, 323232323.3232323232, -111.11111111, -222222222222.222222222222222, b'10101')") - tk.MustExec("alter table t modify d varbinary(30)") - tk.MustExec("alter table t modify n varbinary(30)") - tk.MustExec("alter table t modify r varbinary(30)") - tk.MustExec("alter table t modify db varbinary(30)") + tk.MustGetErrCode("alter table t modify d varbinary(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify n varbinary(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify r varbinary(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify db varbinary(30)", mysql.ErrUnsupportedDDLOperation) // MySQL will get "-111.111" rather than "-111.111115" at TiDB. - tk.MustExec("alter table t modify f32 varbinary(30)") + tk.MustGetErrCode("alter table t modify f32 varbinary(30)", mysql.ErrUnsupportedDDLOperation) // MySQL will get "ERROR 1406 (22001): Data truncation: Data too long for column 'f64' at row 1". - tk.MustExec("alter table t modify f64 varbinary(30)") - tk.MustExec("alter table t modify b varbinary(30)") + tk.MustGetErrCode("alter table t modify f64 varbinary(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify b varbinary(30)", mysql.ErrUnsupportedDDLOperation) tk.MustQuery("select * from t").Check(testkit.Rows("-258.1234500 333.33 2000000.20000002 323232323.32323235 -111.111115 -222222222222.22223 \x15")) // blob @@ -1091,11 +1070,11 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromDateTimeTypeToOthers // varchar reset(tk) tk.MustExec("insert into t values ('2020-10-30', '19:38:25.001', 20201030082133.455555, 20201030082133.455555, 2020)") - tk.MustExec("alter table t modify d varchar(30)") - tk.MustExec("alter table t modify t varchar(30)") - tk.MustExec("alter table t modify dt varchar(30)") - tk.MustExec("alter table t modify tmp varchar(30)") - tk.MustExec("alter table t modify y varchar(30)") + tk.MustGetErrCode("alter table t modify d varchar(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify t varchar(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify dt varchar(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify tmp varchar(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify y varchar(30)", mysql.ErrUnsupportedDDLOperation) tk.MustQuery("select * from t").Check(testkit.Rows("2020-10-30 19:38:25.001 2020-10-30 08:21:33.455555 2020-10-30 08:21:33.455555 2020")) // binary @@ -1115,11 +1094,11 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromDateTimeTypeToOthers // varbinary reset(tk) tk.MustExec("insert into t values ('2020-10-30', '19:38:25.001', 20201030082133.455555, 20201030082133.455555, 2020)") - tk.MustExec("alter table t modify d varbinary(30)") - tk.MustExec("alter table t modify t varbinary(30)") - tk.MustExec("alter table t modify dt varbinary(30)") - tk.MustExec("alter table t modify tmp varbinary(30)") - tk.MustExec("alter table t modify y varbinary(30)") + tk.MustGetErrCode("alter table t modify d varbinary(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify t varbinary(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify dt varbinary(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify tmp varbinary(30)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify y varbinary(30)", mysql.ErrUnsupportedDDLOperation) tk.MustQuery("select * from t").Check(testkit.Rows("2020-10-30 19:38:25.001 2020-10-30 08:21:33.455555 2020-10-30 08:21:33.455555 2020")) // text @@ -1357,15 +1336,15 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromJsonToOthers(c *C) { // varchar reset(tk) tk.MustExec("insert into t values ('{\"obj\": 100}', '[-1, 0, 1]', 'null', 'true', 'false', '-22', '22', '323232323.3232323232', '\"json string\"')") - tk.MustExec("alter table t modify obj varchar(20)") - tk.MustExec("alter table t modify arr varchar(20)") - tk.MustExec("alter table t modify nil varchar(20)") - tk.MustExec("alter table t modify t varchar(20)") - tk.MustExec("alter table t modify f varchar(20)") - tk.MustExec("alter table t modify i varchar(20)") - tk.MustExec("alter table t modify ui varchar(20)") - tk.MustExec("alter table t modify f64 varchar(20)") - tk.MustExec("alter table t modify str varchar(20)") + tk.MustGetErrCode("alter table t modify obj varchar(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify arr varchar(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify nil varchar(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify t varchar(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify f varchar(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify i varchar(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify ui varchar(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify f64 varchar(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify str varchar(20)", mysql.ErrUnsupportedDDLOperation) tk.MustQuery("select * from t").Check(testkit.Rows("{\"obj\": 100} [-1, 0, 1] null true false -22 22 323232323.32323235 \"json string\"")) // binary @@ -1393,15 +1372,15 @@ func (s *testColumnTypeChangeSuite) TestColumnTypeChangeFromJsonToOthers(c *C) { // varbinary reset(tk) tk.MustExec("insert into t values ('{\"obj\": 100}', '[-1, 0, 1]', 'null', 'true', 'false', '-22', '22', '323232323.3232323232', '\"json string\"')") - tk.MustExec("alter table t modify obj varbinary(20)") - tk.MustExec("alter table t modify arr varbinary(20)") - tk.MustExec("alter table t modify nil varbinary(20)") - tk.MustExec("alter table t modify t varbinary(20)") - tk.MustExec("alter table t modify f varbinary(20)") - tk.MustExec("alter table t modify i varbinary(20)") - tk.MustExec("alter table t modify ui varbinary(20)") - tk.MustExec("alter table t modify f64 varbinary(20)") - tk.MustExec("alter table t modify str varbinary(20)") + tk.MustGetErrCode("alter table t modify obj varbinary(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify arr varbinary(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify nil varbinary(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify t varbinary(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify f varbinary(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify i varbinary(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify ui varbinary(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify f64 varbinary(20)", mysql.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table t modify str varbinary(20)", mysql.ErrUnsupportedDDLOperation) tk.MustQuery("select * from t").Check(testkit.Rows("{\"obj\": 100} [-1, 0, 1] null true false -22 22 323232323.32323235 \"json string\"")) // blob diff --git a/ddl/db_integration_test.go b/ddl/db_integration_test.go index c73685358a9cf..d73ef6c0e6850 100644 --- a/ddl/db_integration_test.go +++ b/ddl/db_integration_test.go @@ -1510,6 +1510,86 @@ func (s *testIntegrationSuite8) TestCreateTooManyIndexes(c *C) { tk.MustGetErrCode(alterSQL, errno.ErrTooManyKeys) } +func (s *testIntegrationSuite8) TestCreateSecondaryIndexInCluster(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + + // test create table with non-unique key + tk.MustGetErrCode(` +CREATE TABLE t ( + c01 varchar(255) NOT NULL, + c02 varchar(255) NOT NULL, + c03 varchar(255) NOT NULL, + c04 varchar(255) DEFAULT NULL, + c05 varchar(255) DEFAULT NULL, + c06 varchar(255) DEFAULT NULL, + PRIMARY KEY (c01,c02,c03) clustered, + KEY c04 (c04) +)`, errno.ErrTooLongKey) + + // test create long clustered primary key. + tk.MustGetErrCode(` +CREATE TABLE t ( + c01 varchar(255) NOT NULL, + c02 varchar(255) NOT NULL, + c03 varchar(255) NOT NULL, + c04 varchar(255) NOT NULL, + c05 varchar(255) DEFAULT NULL, + c06 varchar(255) DEFAULT NULL, + PRIMARY KEY (c01,c02,c03,c04) clustered +)`, errno.ErrTooLongKey) + + // test create table with unique key + tk.MustExec(` +CREATE TABLE t ( + c01 varchar(255) NOT NULL, + c02 varchar(255) NOT NULL, + c03 varchar(255) NOT NULL, + c04 varchar(255) DEFAULT NULL, + c05 varchar(255) DEFAULT NULL, + c06 varchar(255) DEFAULT NULL, + PRIMARY KEY (c01,c02,c03) clustered, + unique key c04 (c04) +)`) + tk.MustExec("drop table t") + + // test create index + tk.MustExec(` +CREATE TABLE t ( + c01 varchar(255) NOT NULL, + c02 varchar(255) NOT NULL, + c03 varchar(255) NOT NULL, + c04 varchar(255) DEFAULT NULL, + c05 varchar(255) DEFAULT NULL, + c06 varchar(255) DEFAULT NULL, + PRIMARY KEY (c01,c02) clustered +)`) + tk.MustExec("create index idx1 on t(c03)") + tk.MustGetErrCode("create index idx2 on t(c03, c04)", errno.ErrTooLongKey) + tk.MustExec("create unique index uk2 on t(c03, c04)") + tk.MustExec("drop table t") + + // test change/modify column + tk.MustExec(` +CREATE TABLE t ( + c01 varchar(255) NOT NULL, + c02 varchar(255) NOT NULL, + c03 varchar(255) NOT NULL, + c04 varchar(255) DEFAULT NULL, + c05 varchar(255) DEFAULT NULL, + c06 varchar(255) DEFAULT NULL, + Index idx1(c03), + PRIMARY KEY (c01,c02) clustered, + unique index uk1(c06) +)`) + tk.MustExec("alter table t change c03 c10 varchar(256) default null") + tk.MustGetErrCode("alter table t change c10 c100 varchar(1024) default null", errno.ErrTooLongKey) + tk.MustGetErrCode("alter table t modify c10 varchar(600) default null", errno.ErrTooLongKey) + tk.MustExec("alter table t modify c06 varchar(600) default null") + tk.MustGetErrCode("alter table t modify c01 varchar(510)", errno.ErrTooLongKey) + tk.MustExec("create table t2 like t") +} + func (s *testIntegrationSuite3) TestAlterColumn(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test_db") diff --git a/ddl/db_test.go b/ddl/db_test.go index d3b1927ef1a18..d4d116cf54450 100644 --- a/ddl/db_test.go +++ b/ddl/db_test.go @@ -4078,11 +4078,9 @@ func (s *testSerialDBSuite) TestModifyColumnBetweenStringTypes(c *C) { tk.MustGetErrMsg("alter table tt change a a varchar(4);", "[types:1406]Data Too Long, field len 4, data len 5") tk.MustExec("alter table tt change a a varchar(100);") - // varchar to char - tk.MustExec("alter table tt change a a char(10);") - c2 = getModifyColumn(c, s.s.(sessionctx.Context), "test", "tt", "a", false) - c.Assert(c2.FieldType.Tp, Equals, mysql.TypeString) - c.Assert(c2.FieldType.Flen, Equals, 10) + tk.MustExec("drop table if exists tt;") + tk.MustExec("create table tt (a char(10));") + tk.MustExec("insert into tt values ('111'),('10000');") tk.MustQuery("select * from tt").Check(testkit.Rows("111", "10000")) tk.MustGetErrMsg("alter table tt change a a char(4);", "[types:1406]Data Too Long, field len 4, data len 5") diff --git a/ddl/ddl_api.go b/ddl/ddl_api.go index e4f8b42fe03fb..9c4eac7a422f3 100644 --- a/ddl/ddl_api.go +++ b/ddl/ddl_api.go @@ -1505,6 +1505,44 @@ func buildTableInfo( idxInfo.ID = allocateIndexID(tbInfo) tbInfo.Indices = append(tbInfo.Indices, idxInfo) } + if tbInfo.IsCommonHandle { + // Ensure tblInfo's each non-unique secondary-index's len + primary-key's len <= MaxIndexLength for clustered index table. + var pkLen, idxLen int + pkLen, err = indexColumnsLen(tbInfo.Columns, tables.FindPrimaryIndex(tbInfo).Columns) + if err != nil { + return + } + for _, idx := range tbInfo.Indices { + if idx.Unique { + // Only need check for non-unique secondary-index. + continue + } + idxLen, err = indexColumnsLen(tbInfo.Columns, idx.Columns) + if err != nil { + return + } + if pkLen+idxLen > config.GetGlobalConfig().MaxIndexLength { + return nil, errTooLongKey.GenWithStackByArgs(config.GetGlobalConfig().MaxIndexLength) + } + } + } + return +} + +func indexColumnsLen(cols []*model.ColumnInfo, idxCols []*model.IndexColumn) (len int, err error) { + for _, idxCol := range idxCols { + col := model.FindColumnInfo(cols, idxCol.Name.L) + if col == nil { + err = errKeyColumnDoesNotExits.GenWithStack("column does not exist: %s", idxCol.Name.L) + return + } + var colLen int + colLen, err = getIndexColumnLength(col, idxCol.Length) + if err != nil { + return + } + len += colLen + } return } @@ -3555,11 +3593,10 @@ func checkTypeChangeSupported(origin *types.FieldType, to *types.FieldType) bool func checkModifyTypes(ctx sessionctx.Context, origin *types.FieldType, to *types.FieldType, needRewriteCollationData bool) error { canReorg, changeColumnErrMsg, err := CheckModifyTypeCompatible(origin, to) if err != nil { - enableChangeColumnType := ctx.GetSessionVars().EnableChangeColumnType if !canReorg { return errors.Trace(err) } - + enableChangeColumnType := ctx.GetSessionVars().EnableChangeColumnType if !enableChangeColumnType { msg := fmt.Sprintf("%s, and tidb_enable_change_column_type is false", changeColumnErrMsg) return errUnsupportedModifyColumn.GenWithStackByArgs(msg) @@ -3568,6 +3605,10 @@ func checkModifyTypes(ctx sessionctx.Context, origin *types.FieldType, to *types return errUnsupportedModifyColumn.GenWithStackByArgs(msg) } } + if types.IsTypeVarchar(origin.Tp) != types.IsTypeVarchar(to.Tp) { + unsupportedMsg := "column type conversion between 'varchar' and 'non-varchar' is currently unsupported yet" + return errUnsupportedModifyColumn.GenWithStackByArgs(unsupportedMsg) + } err = checkModifyCharsetAndCollation(to.Charset, to.Collate, origin.Charset, origin.Collate, needRewriteCollationData) // column type change can handle the charset change between these two types in the process of the reorg. @@ -3878,33 +3919,70 @@ func (d *ddl) getModifiableColumnJob(ctx sessionctx.Context, ident ast.Ident, or // checkColumnWithIndexConstraint is used to check the related index constraint of the modified column. // Index has a max-prefix-length constraint. eg: a varchar(100), index idx(a), modifying column a to a varchar(4000) // will cause index idx to break the max-prefix-length constraint. +// +// For clustered index: +// Change column in pk need recheck all non-unique index, new pk len + index len < maxIndexLength. +// Change column in secondary only need related index, pk len + new index len < maxIndexLength. func checkColumnWithIndexConstraint(tbInfo *model.TableInfo, originalCol, newCol *model.ColumnInfo) error { - var columns []*model.ColumnInfo - for _, indexInfo := range tbInfo.Indices { - containColumn := false + columns := make([]*model.ColumnInfo, 0, len(tbInfo.Columns)) + columns = append(columns, tbInfo.Columns...) + // Replace old column with new column. + for i, col := range columns { + if col.Name.L != originalCol.Name.L { + continue + } + columns[i] = newCol.Clone() + columns[i].Name = originalCol.Name + break + } + + pkIndex := tables.FindPrimaryIndex(tbInfo) + var clusteredPkLen int + if tbInfo.IsCommonHandle { + var err error + clusteredPkLen, err = indexColumnsLen(columns, pkIndex.Columns) + if err != nil { + return err + } + } + + checkOneIndex := func(indexInfo *model.IndexInfo, pkLenAppendToKey int, skipCheckIfNotModify bool) (modified bool, err error) { for _, col := range indexInfo.Columns { if col.Name.L == originalCol.Name.L { - containColumn = true + modified = true break } } - if !containColumn { + if skipCheckIfNotModify && !modified { + return + } + err = checkIndexPrefixLength(columns, indexInfo.Columns, pkLenAppendToKey) + return + } + + // Check primary key first and get "does primary key's column has be modified?" info. + var ( + pkModified bool + err error + ) + if pkIndex != nil { + pkModified, err = checkOneIndex(pkIndex, 0, true) + if err != nil { + return err + } + } + + // Check secondary indexes. + for _, indexInfo := range tbInfo.Indices { + if indexInfo.Primary { continue } - if columns == nil { - columns = make([]*model.ColumnInfo, 0, len(tbInfo.Columns)) - columns = append(columns, tbInfo.Columns...) - // replace old column with new column. - for i, col := range columns { - if col.Name.L != originalCol.Name.L { - continue - } - columns[i] = newCol.Clone() - columns[i].Name = originalCol.Name - break - } + var pkLenAppendToKey int + if !indexInfo.Unique { + pkLenAppendToKey = clusteredPkLen } - err := checkIndexPrefixLength(columns, indexInfo.Columns) + + _, err = checkOneIndex(indexInfo, pkLenAppendToKey, !tbInfo.IsCommonHandle || !pkModified) if err != nil { return err } @@ -4978,6 +5056,22 @@ func (d *ddl) CreateIndex(ctx sessionctx.Context, ti ast.Ident, keyType ast.Inde return errors.Trace(err) } + if !unique && tblInfo.IsCommonHandle { + // Ensure new created non-unique secondary-index's len + primary-key's len <= MaxIndexLength in clustered index table. + var pkLen, idxLen int + pkLen, err = indexColumnsLen(tblInfo.Columns, tables.FindPrimaryIndex(tblInfo).Columns) + if err != nil { + return err + } + idxLen, err = indexColumnsLen(tblInfo.Columns, indexColumns) + if err != nil { + return err + } + if pkLen+idxLen > config.GetGlobalConfig().MaxIndexLength { + return errTooLongKey.GenWithStackByArgs(config.GetGlobalConfig().MaxIndexLength) + } + } + global := false if unique && tblInfo.GetPartitionInfo() != nil { ck, err := checkPartitionKeysConstraint(tblInfo.GetPartitionInfo(), indexColumns, tblInfo) diff --git a/ddl/index.go b/ddl/index.go index 1e2423f82ef19..a056462e81bfd 100644 --- a/ddl/index.go +++ b/ddl/index.go @@ -108,24 +108,13 @@ func checkPKOnGeneratedColumn(tblInfo *model.TableInfo, indexPartSpecifications return lastCol, nil } -func checkIndexPrefixLength(columns []*model.ColumnInfo, idxColumns []*model.IndexColumn) error { - // The sum of length of all index columns. - sumLength := 0 - for _, ic := range idxColumns { - col := model.FindColumnInfo(columns, ic.Name.L) - if col == nil { - return errKeyColumnDoesNotExits.GenWithStack("column does not exist: %s", ic.Name) - } - - indexColumnLength, err := getIndexColumnLength(col, ic.Length) - if err != nil { - return err - } - sumLength += indexColumnLength - // The sum of all lengths must be shorter than the max length for prefix. - if sumLength > config.GetGlobalConfig().MaxIndexLength { - return errTooLongKey.GenWithStackByArgs(config.GetGlobalConfig().MaxIndexLength) - } +func checkIndexPrefixLength(columns []*model.ColumnInfo, idxColumns []*model.IndexColumn, pkLenAppendToKey int) error { + idxLen, err := indexColumnsLen(columns, idxColumns) + if err != nil { + return err + } + if idxLen+pkLenAppendToKey > config.GetGlobalConfig().MaxIndexLength { + return errTooLongKey.GenWithStackByArgs(config.GetGlobalConfig().MaxIndexLength) } return nil } @@ -488,7 +477,6 @@ func (w *worker) onCreateIndex(d *ddlCtx, t *meta.Meta, job *model.Job, isPK boo indexInfo.Global = global indexInfo.ID = allocateIndexID(tblInfo) tblInfo.Indices = append(tblInfo.Indices, indexInfo) - if err = checkTooManyIndexes(tblInfo.Indices); err != nil { job.State = model.JobStateCancelled return ver, errors.Trace(err) @@ -927,7 +915,7 @@ func (w *baseIndexWorker) getIndexRecord(idxInfo *model.IndexInfo, handle kv.Han idxVal[j] = idxColumnVal } - rsData := tables.TryGetHandleRestoredDataWrapper(w.table, nil, w.rowMap) + rsData := tables.TryGetHandleRestoredDataWrapper(w.table, nil, w.rowMap, idxInfo) idxRecord := &indexRecord{handle: handle, key: recordKey, vals: idxVal, rsData: rsData} return idxRecord, nil } diff --git a/executor/admin.go b/executor/admin.go index 1b81dcf342c43..85b8158832064 100644 --- a/executor/admin.go +++ b/executor/admin.go @@ -378,7 +378,7 @@ func (e *RecoverIndexExec) fetchRecoverRows(ctx context.Context, srcResult dists } idxVals := extractIdxVals(row, e.idxValsBufs[result.scanRowCount], e.colFieldTypes, idxValLen) e.idxValsBufs[result.scanRowCount] = idxVals - rsData := tables.TryGetHandleRestoredDataWrapper(e.table, plannercore.GetCommonHandleDatum(e.handleCols, row), nil) + rsData := tables.TryGetHandleRestoredDataWrapper(e.table, plannercore.GetCommonHandleDatum(e.handleCols, row), nil, e.index.Meta()) e.recoverRows = append(e.recoverRows, recoverRows{handle: handle, idxVals: idxVals, rsData: rsData, skip: false}) result.scanRowCount++ result.currentHandle = handle diff --git a/executor/batch_point_get.go b/executor/batch_point_get.go index ca4416658306b..a72b0eefd77c9 100644 --- a/executor/batch_point_get.go +++ b/executor/batch_point_get.go @@ -213,6 +213,9 @@ func (e *BatchPointGetExec) initialize(ctx context.Context) error { if err1 != nil && !kv.ErrNotExist.Equal(err1) { return err1 } + if idxKey == nil { + continue + } s := hack.String(idxKey) if _, found := dedup[s]; found { continue diff --git a/executor/builder.go b/executor/builder.go index 1d20ee55a1a7a..11bd81094e83a 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -2360,6 +2360,12 @@ func (b *executorBuilder) buildIndexLookUpJoin(v *plannercore.PhysicalIndexJoin) } } + // Use the probe table's collation. + for i, col := range v.OuterHashKeys { + outerTypes[col.Index] = outerTypes[col.Index].Clone() + outerTypes[col.Index].Collate = innerTypes[v.InnerHashKeys[i].Index].Collate + } + var ( outerFilter []expression.Expression leftTypes, rightTypes []*types.FieldType diff --git a/executor/ddl_test.go b/executor/ddl_test.go index 5ea9171649c71..c55908066de62 100644 --- a/executor/ddl_test.go +++ b/executor/ddl_test.go @@ -546,12 +546,12 @@ func (s *testSuite6) TestAlterTableModifyColumn(c *C) { _, err = tk.Exec("alter table mc modify column c2 varchar(8)") c.Assert(err, NotNil) tk.MustExec("alter table mc modify column c2 varchar(11)") - tk.MustExec("alter table mc modify column c2 text(13)") - tk.MustExec("alter table mc modify column c2 text") + tk.MustGetErrCode("alter table mc modify column c2 text(13)", errno.ErrUnsupportedDDLOperation) + tk.MustGetErrCode("alter table mc modify column c2 text", errno.ErrUnsupportedDDLOperation) tk.MustExec("alter table mc modify column c3 bit") result := tk.MustQuery("show create table mc") createSQL := result.Rows()[0][1] - expected := "CREATE TABLE `mc` (\n `c1` bigint(20) DEFAULT NULL,\n `c2` text DEFAULT NULL,\n `c3` bit(1) DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" + expected := "CREATE TABLE `mc` (\n `c1` bigint(20) DEFAULT NULL,\n `c2` varchar(11) DEFAULT NULL,\n `c3` bit(1) DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin" c.Assert(createSQL, Equals, expected) tk.MustExec("create or replace view alter_view as select c1,c2 from mc") _, err = tk.Exec("alter table alter_view modify column c2 text") diff --git a/executor/distsql.go b/executor/distsql.go index 9f95de0b8775a..316697230f2db 100644 --- a/executor/distsql.go +++ b/executor/distsql.go @@ -925,9 +925,7 @@ func (e *IndexLookUpExecutor) getHandle(row chunk.Row, handleIdx []int, } datums = append(datums, row.GetDatum(idx, e.handleCols[i].RetType)) } - if tp == getHandleFromTable { - tablecodec.TruncateIndexValues(e.table.Meta(), e.primaryKeyIndex, datums) - } + tablecodec.TruncateIndexValues(e.table.Meta(), e.primaryKeyIndex, datums) handleEncoded, err = codec.EncodeKey(e.ctx.GetSessionVars().StmtCtx, nil, datums...) if err != nil { return nil, err diff --git a/executor/index_lookup_join.go b/executor/index_lookup_join.go index 207859dc3a891..af32b43668f48 100644 --- a/executor/index_lookup_join.go +++ b/executor/index_lookup_join.go @@ -530,10 +530,11 @@ func (iw *innerWorker) constructLookupContent(task *lookUpJoinTask) ([]*indexJoi // Store the encoded lookup key in chunk, so we can use it to lookup the matched inners directly. task.encodedLookUpKeys[chkIdx].AppendBytes(0, keyBuf) if iw.hasPrefixCol { - for i := range iw.outerCtx.keyCols { + for i, outerOffset := range iw.outerCtx.keyCols { // If it's a prefix column. Try to fix it. - if iw.colLens[iw.keyCols[i]] != types.UnspecifiedLength { - ranger.CutDatumByPrefixLen(&dLookUpKey[i], iw.colLens[iw.keyCols[i]], iw.rowTypes[iw.keyCols[i]]) + joinKeyColPrefixLen := iw.colLens[outerOffset] + if joinKeyColPrefixLen != types.UnspecifiedLength { + ranger.CutDatumByPrefixLen(&dLookUpKey[i], joinKeyColPrefixLen, iw.rowTypes[iw.keyCols[i]]) } } // dLookUpKey is sorted and deduplicated at sortAndDedupLookUpContents. diff --git a/executor/index_lookup_join_test.go b/executor/index_lookup_join_test.go index 906a961f8cc8f..2d7743c5db976 100644 --- a/executor/index_lookup_join_test.go +++ b/executor/index_lookup_join_test.go @@ -298,3 +298,36 @@ func (s *testSuite5) TestIssue23656(c *C) { "1 clever jang 1 clever jang", "2 blissful aryabhata 2 blissful aryabhata")) } + +func (s *testSuite5) TestIssue23722(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int, b char(10), c blob, primary key (c(5)) clustered);") + tk.MustExec("insert into t values (20301,'Charlie',x'7a');") + tk.MustQuery("select * from t;").Check(testkit.Rows("20301 Charlie z")) + tk.MustQuery("select * from t where c in (select c from t where t.c >= 'a');").Check(testkit.Rows("20301 Charlie z")) + + // Test lookup content exceeds primary key prefix. + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t (a int, b char(10), c varchar(255), primary key (c(5)) clustered);") + tk.MustExec("insert into t values (20301,'Charlie','aaaaaaa');") + tk.MustQuery("select * from t;").Check(testkit.Rows("20301 Charlie aaaaaaa")) + tk.MustQuery("select * from t where c in (select c from t where t.c >= 'a');").Check(testkit.Rows("20301 Charlie aaaaaaa")) + + // Test the original case. + tk.MustExec("drop table if exists t;") + tk.MustExec(`CREATE TABLE t ( + col_15 decimal(49,3), + col_16 smallint(5), + col_17 char(118) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT 'tLOOjbIXuuLKPFjkLo', + col_18 set('Alice','Bob','Charlie','David') NOT NULL, + col_19 tinyblob, + PRIMARY KEY (col_19(5),col_16) /*T![clustered_index] NONCLUSTERED */, + UNIQUE KEY idx_10 (col_19(5),col_16) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;`) + tk.MustExec("INSERT INTO `t` VALUES (38799.400,20301,'KETeFZhkoxnwMAhA','Charlie',x'7a7968584570705a647179714e56');") + tk.MustQuery("select t.* from t where col_19 in " + + "( select col_19 from t where t.col_18 <> 'David' and t.col_19 >= 'jDzNn' ) " + + "order by col_15 , col_16 , col_17 , col_18 , col_19;").Check(testkit.Rows("38799.400 20301 KETeFZhkoxnwMAhA Charlie zyhXEppZdqyqNV")) +} diff --git a/executor/show_test.go b/executor/show_test.go index 032c4d08839c8..a4a0b763337c7 100644 --- a/executor/show_test.go +++ b/executor/show_test.go @@ -1078,9 +1078,9 @@ func (s *testSuite5) TestShowBuiltin(c *C) { res := tk.MustQuery("show builtins;") c.Assert(res, NotNil) rows := res.Rows() - c.Assert(267, Equals, len(rows)) + c.Assert(268, Equals, len(rows)) c.Assert("abs", Equals, rows[0][0].(string)) - c.Assert("yearweek", Equals, rows[266][0].(string)) + c.Assert("yearweek", Equals, rows[267][0].(string)) } func (s *testSuite5) TestShowClusterConfig(c *C) { diff --git a/expression/builtin.go b/expression/builtin.go index 46c61220f8ce5..9c530f92949e1 100644 --- a/expression/builtin.go +++ b/expression/builtin.go @@ -787,6 +787,7 @@ var funcs = map[string]functionClass{ ast.ReleaseAllLocks: &releaseAllLocksFunctionClass{baseFunctionClass{ast.ReleaseAllLocks, 0, 0}}, ast.UUID: &uuidFunctionClass{baseFunctionClass{ast.UUID, 0, 0}}, ast.UUIDShort: &uuidShortFunctionClass{baseFunctionClass{ast.UUIDShort, 0, 0}}, + ast.VitessHash: &vitessHashFunctionClass{baseFunctionClass{ast.VitessHash, 1, 1}}, // get_lock() and release_lock() are parsed but do nothing. // It is used for preventing error in Ruby's activerecord migrations. diff --git a/expression/builtin_miscellaneous.go b/expression/builtin_miscellaneous.go index 25b4ddb3c4fbb..72f1b8e6ac2ed 100644 --- a/expression/builtin_miscellaneous.go +++ b/expression/builtin_miscellaneous.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/types/json" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/vitess" "github.com/pingcap/tipb/go-tipb" ) @@ -51,6 +52,7 @@ var ( _ functionClass = &releaseAllLocksFunctionClass{} _ functionClass = &uuidFunctionClass{} _ functionClass = &uuidShortFunctionClass{} + _ functionClass = &vitessHashFunctionClass{} ) var ( @@ -73,6 +75,7 @@ var ( _ builtinFunc = &builtinIsIPv4MappedSig{} _ builtinFunc = &builtinIsIPv6Sig{} _ builtinFunc = &builtinUUIDSig{} + _ builtinFunc = &builtinVitessHashSig{} _ builtinFunc = &builtinNameConstIntSig{} _ builtinFunc = &builtinNameConstRealSig{} @@ -1046,3 +1049,48 @@ type uuidShortFunctionClass struct { func (c *uuidShortFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (builtinFunc, error) { return nil, errFunctionNotExists.GenWithStackByArgs("FUNCTION", "UUID_SHORT") } + +type vitessHashFunctionClass struct { + baseFunctionClass +} + +func (c *vitessHashFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (builtinFunc, error) { + if err := c.verifyArgs(args); err != nil { + return nil, err + } + bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETInt, types.ETInt) + if err != nil { + return nil, err + } + + bf.tp.Flen = 20 //64 bit unsigned + bf.tp.Flag |= mysql.UnsignedFlag + types.SetBinChsClnFlag(bf.tp) + + sig := &builtinVitessHashSig{bf} + sig.setPbCode(tipb.ScalarFuncSig_VitessHash) + return sig, nil +} + +type builtinVitessHashSig struct { + baseBuiltinFunc +} + +func (b *builtinVitessHashSig) Clone() builtinFunc { + newSig := &builtinVitessHashSig{} + newSig.cloneFrom(&b.baseBuiltinFunc) + return newSig +} + +// evalInt evals VITESS_HASH(int64). +func (b *builtinVitessHashSig) evalInt(row chunk.Row) (int64, bool, error) { + shardKeyInt, isNull, err := b.args[0].EvalInt(b.ctx, row) + if isNull || err != nil { + return 0, true, err + } + var hashed uint64 + if hashed, err = vitess.HashUint64(uint64(shardKeyInt)); err != nil { + return 0, true, err + } + return int64(hashed), false, nil +} diff --git a/expression/builtin_miscellaneous_vec.go b/expression/builtin_miscellaneous_vec.go index 8780920c13a04..8e0689be3aab8 100644 --- a/expression/builtin_miscellaneous_vec.go +++ b/expression/builtin_miscellaneous_vec.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/vitess" ) func (b *builtinInetNtoaSig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { @@ -617,3 +618,38 @@ func (b *builtinReleaseLockSig) vecEvalInt(input *chunk.Chunk, result *chunk.Col } return nil } + +func (b *builtinVitessHashSig) vectorized() bool { + return true +} + +func (b *builtinVitessHashSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) error { + n := input.NumRows() + column, err := b.bufAllocator.get(types.ETInt, n) + if err != nil { + return err + } + defer b.bufAllocator.put(column) + + if err := b.args[0].VecEvalInt(b.ctx, input, column); err != nil { + return err + } + + result.ResizeInt64(n, false) + r64s := result.Uint64s() + result.MergeNulls(column) + + for i := 0; i < n; i++ { + if column.IsNull(i) { + continue + } + var uintKey = column.GetUint64(i) + var hash uint64 + if hash, err = vitess.HashUint64(uintKey); err != nil { + return err + } + r64s[i] = hash + } + + return nil +} diff --git a/expression/integration_test.go b/expression/integration_test.go index 5c461b8dff08c..c554524ab8b99 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -8280,6 +8280,32 @@ func (s *testIntegrationSerialSuite) TestCollationIndexJoin(c *C) { tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1815 Optimizer Hint /*+ INL_MERGE_JOIN(t2) */ is inapplicable")) } +func (s *testIntegrationSerialSuite) TestCollationMergeJoin(c *C) { + collate.SetNewCollationEnabledForTest(true) + defer collate.SetNewCollationEnabledForTest(false) + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("CREATE TABLE `t` (" + + " `col_10` blob DEFAULT NULL," + + " `col_11` decimal(17,5) NOT NULL," + + " `col_13` varchar(381) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'Yr'," + + " PRIMARY KEY (`col_13`,`col_11`) CLUSTERED," + + " KEY `idx_4` (`col_10`(3))" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin") + tk.MustExec("insert into t values ('a', 12523, 'A');") + tk.MustExec("insert into t values ('A', 2, 'a');") + tk.MustExec("insert into t values ('a', 23, 'A');") + tk.MustExec("insert into t values ('a', 23, 'h2');") + tk.MustExec("insert into t values ('a', 23, 'h3');") + tk.MustExec("insert into t values ('a', 23, 'h4');") + tk.MustExec("insert into t values ('a', 23, 'h5');") + tk.MustExec("insert into t values ('a', 23, 'h6');") + tk.MustExec("insert into t values ('a', 23, 'h7');") + tk.MustQuery("select /*+ MERGE_JOIN(t) */ t.* from t where col_13 in ( select col_10 from t where t.col_13 in ( 'a', 'b' ) ) order by col_10 ;").Check( + testkit.Rows("\x41 2.00000 a", "\x61 23.00000 A", "\x61 12523.00000 A")) +} + func (s *testIntegrationSuite) TestIssue19892(c *C) { defer s.cleanEnv(c) tk := testkit.NewTestKit(c, s.store) @@ -8937,3 +8963,110 @@ func (s *testIntegrationSuite) TestIssue23623(c *C) { tk.MustExec("insert into t1 values(-2147483648), (-2147483648), (null);") tk.MustQuery("select count(*) from t1 where c1 > (select sum(c1) from t1);").Check(testkit.Rows("2")) } + +func (s *testIntegrationSerialSuite) TestCollationPrefixClusteredIndex(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + collate.SetNewCollationEnabledForTest(true) + defer collate.SetNewCollationEnabledForTest(false) + + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (k char(20), v int, primary key (k(4)) clustered, key (k)) collate utf8mb4_general_ci;") + tk.MustExec("insert into t values('01233', 1);") + tk.MustExec("create index idx on t(k(2))") + tk.MustQuery("select * from t use index(k_2);").Check(testkit.Rows("01233 1")) + tk.MustQuery("select * from t use index(idx);").Check(testkit.Rows("01233 1")) + tk.MustExec("admin check table t;") +} + +func (s *testIntegrationSerialSuite) TestIssue23805(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + collate.SetNewCollationEnabledForTest(true) + defer collate.SetNewCollationEnabledForTest(false) + + tk.MustExec("CREATE TABLE `tbl_5` (" + + " `col_25` time NOT NULL DEFAULT '05:35:58'," + + " `col_26` blob NOT NULL," + + " `col_27` double NOT NULL," + + " `col_28` char(83) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL," + + " `col_29` timestamp NOT NULL," + + " `col_30` varchar(36) COLLATE utf8mb4_general_ci NOT NULL DEFAULT 'ywzIn'," + + " `col_31` binary(85) DEFAULT 'OIstcXsGmAyc'," + + " `col_32` datetime NOT NULL DEFAULT '2024-08-02 00:00:00'," + + " PRIMARY KEY (`col_26`(3),`col_27`) /*T![clustered_index] CLUSTERED */," + + " UNIQUE KEY `idx_10` (`col_26`(5)));") + tk.MustExec("insert ignore into tbl_5 set col_28 = 'ZmZIdSnq' , col_25 = '18:50:52.00' on duplicate key update col_26 = 'y';\n") +} + +func (s *testIntegrationSuite) TestVitessHash(c *C) { + defer s.cleanEnv(c) + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t_int, t_blob, t_varchar;") + tk.MustExec("create table t_int(id int, a bigint unsigned null);") + tk.MustExec("insert into t_int values " + + "(1, 30375298039), " + + "(2, 1123), " + + "(3, 30573721600), " + + "(4, " + fmt.Sprintf("%d", uint64(math.MaxUint64)) + ")," + + "(5, 116)," + + "(6, null);") + + // Integers + tk.MustQuery("select hex(vitess_hash(a)) from t_int"). + Check(testkit.Rows( + "31265661E5F1133", + "31B565D41BDF8CA", + "1EFD6439F2050FFD", + "355550B2150E2451", + "1E1788FF0FDE093C", + "")) + + // Nested function sanity test + tk.MustQuery("select hex(vitess_hash(convert(a, decimal(8,4)))) from t_int where id = 5"). + Check(testkit.Rows("1E1788FF0FDE093C")) +} + +func (s *testIntegrationSuite) TestVitessHashMatchesVitessShards(c *C) { + defer s.cleanEnv(c) + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t;") + tk.MustExec("create table t(customer_id bigint, id bigint, expected_shard bigint unsigned, computed_shard bigint unsigned null, primary key (customer_id, id));") + + tk.MustExec("insert into t (customer_id, id, expected_shard) values " + + "(30370720100, 1, x'd6'), " + + "(30370670010, 2, x'd6'), " + + "(30370689320, 3, x'e1'), " + + "(30370693008, 4, x'e0'), " + + "(30370656005, 5, x'89'), " + + "(30370702638, 6, x'89'), " + + "(30370658809, 7, x'ce'), " + + "(30370665369, 8, x'cf'), " + + "(30370706138, 9, x'85'), " + + "(30370708769, 10, x'85'), " + + "(30370711915, 11, x'a3'), " + + "(30370712595, 12, x'a3'), " + + "(30370656340, 13, x'7d'), " + + "(30370660143, 14, x'7c'), " + + "(30371738450, 15, x'fc'), " + + "(30371683979, 16, x'fd'), " + + "(30370664597, 17, x'92'), " + + "(30370667361, 18, x'93'), " + + "(30370656406, 19, x'd2'), " + + "(30370716959, 20, x'd3'), " + + "(30375207698, 21, x'9a'), " + + "(30375168766, 22, x'9a'), " + + "(30370711813, 23, x'ca'), " + + "(30370721803, 24, x'ca'), " + + "(30370717957, 25, x'97'), " + + "(30370734969, 26, x'96'), " + + "(30375203572, 27, x'98'), " + + "(30375292643, 28, x'99'); ") + + // Sanity check the shards being computed correctly + tk.MustExec("update t set computed_shard = (vitess_hash(customer_id) >> 56);") + tk.MustQuery("select customer_id, id, hex(expected_shard), hex(computed_shard) from t where expected_shard <> computed_shard"). + Check(testkit.Rows()) +} diff --git a/go.mod b/go.mod index c9595e91666e6..1f9b7c41f6f27 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( github.com/pingcap/parser v0.0.0-20210325072920-0d17053a8a69 github.com/pingcap/sysutil v0.0.0-20210221112134-a07bda3bde99 github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible - github.com/pingcap/tipb v0.0.0-20210309080453-72c4feaa6da7 + github.com/pingcap/tipb v0.0.0-20210326161441-1164ca065d1b github.com/prometheus/client_golang v1.5.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.9.1 @@ -79,7 +79,7 @@ require ( golang.org/x/mod v0.4.2 // indirect golang.org/x/net v0.0.0-20210323141857-08027d57d8cf golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 - golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 + golang.org/x/sys v0.0.0-20210324051608-47abb6519492 golang.org/x/text v0.3.5 golang.org/x/tools v0.1.0 google.golang.org/grpc v1.27.1 diff --git a/go.sum b/go.sum index f0a4bb4cadbce..6e36beeed5cf9 100644 --- a/go.sum +++ b/go.sum @@ -474,8 +474,8 @@ github.com/pingcap/sysutil v0.0.0-20210221112134-a07bda3bde99/go.mod h1:EB/852NM github.com/pingcap/tidb-dashboard v0.0.0-20210312062513-eef5d6404638/go.mod h1:OzFN8H0EDMMqeulPhPMw2i2JaiZWOKFQ7zdRPhENNgo= github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible h1:ceznmu/lLseGHP/jKyOa/3u/5H3wtLLLqkH2V3ssSjg= github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tipb v0.0.0-20210309080453-72c4feaa6da7 h1:j8MkWmy5tduhHVsdsgZJugN1U9OWTMSBQoZIpn8kqPc= -github.com/pingcap/tipb v0.0.0-20210309080453-72c4feaa6da7/go.mod h1:nsEhnMokcn7MRqd2J60yxpn/ac3ZH8A6GOJ9NslabUo= +github.com/pingcap/tipb v0.0.0-20210326161441-1164ca065d1b h1:sZHSH0mh8PcRbmZlsIqP7CEwnfFuBpmkGt5i9JStLWA= +github.com/pingcap/tipb v0.0.0-20210326161441-1164ca065d1b/go.mod h1:nsEhnMokcn7MRqd2J60yxpn/ac3ZH8A6GOJ9NslabUo= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -523,6 +523,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil v2.19.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.20.12+incompatible h1:6VEGkOXP/eP4o2Ilk8cSsX0PhOEfX6leqAnD+urrp9M= @@ -797,8 +798,8 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492 h1:Paq34FxTluEPvVyayQqMPgHm+vTOrIifmcYxFBx9TLg= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/metrics/grafana/overview.json b/metrics/grafana/overview.json index 13fd49e024895..fcff9d21021c9 100644 --- a/metrics/grafana/overview.json +++ b/metrics/grafana/overview.json @@ -497,9 +497,9 @@ "tableColumn": "", "targets": [ { - "expr": "delta(pd_tso_events{tidb_cluster=\"$tidb_cluster\", type=\"save\",instance=\"$instance\"}[1m]) > bool 0", + "expr": "pd_tso_role{tidb_cluster=\"$tidb_cluster\", instance=\"$instance\", dc=\"global\"}", "format": "time_series", - "interval": "", + "instant": true, "intervalFactor": 2, "legendFormat": "", "metric": "pd_server_tso", @@ -514,13 +514,18 @@ "valueMaps": [ { "op": "=", - "text": "Leader", - "value": "1" + "text": "Follower", + "value": "null" }, { "op": "=", "text": "Follower", - "value": "null" + "value": "0" + }, + { + "op": "=", + "text": "Leader", + "value": "1" } ], "valueName": "current" @@ -5165,25 +5170,20 @@ "list": [ { "allValue": null, - "current": { - }, + "current": {}, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "tidb_cluster", "multi": false, "name": "tidb_cluster", - "options": [ - - ], + "options": [], "query": "label_values(pd_cluster_status, tidb_cluster)", "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", - "tags": [ - - ], + "tags": [], "tagsQuery": "", "type": "query", "useTags": false diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 4d7b444e2ce70..cb006275187f0 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -480,16 +480,7 @@ func (p *LogicalJoin) constructIndexJoin( IsNullEQ: newIsNullEQ, DefaultValues: p.DefaultValues, } - // Correct the collation used by hash. - for i := range outerHashKeys { - // Make compiler happy. - if len(innerHashKeys) == 0 { - return nil - } - chs, coll := expression.DeriveCollationFromExprs(nil, outerHashKeys[i], innerHashKeys[i]) - outerHashKeys[i].GetType().Charset, outerHashKeys[i].GetType().Collate = chs, coll - innerHashKeys[i].GetType().Charset, innerHashKeys[i].GetType().Collate = chs, coll - } + join := PhysicalIndexJoin{ basePhysicalJoin: baseJoin, innerTask: innerTask, diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index 6425f9d89ad9b..612d70399b907 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -1751,10 +1751,11 @@ func (ds *DataSource) getOriginalPhysicalTableScan(prop *property.PhysicalProper physicalTableID: ds.physicalTableID, Ranges: path.Ranges, AccessCondition: path.AccessConds, - filterCondition: path.TableFilters, StoreType: path.StoreType, IsGlobalRead: path.IsTiFlashGlobalRead, }.Init(ds.ctx, ds.blockOffset) + ts.filterCondition = make([]expression.Expression, len(path.TableFilters)) + copy(ts.filterCondition, path.TableFilters) ts.SetSchema(ds.schema.Clone()) if ts.Table.PKIsHandle { if pkColInfo := ts.Table.GetPkColInfo(); pkColInfo != nil { diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 2575fc1749a41..fe09e7a46126e 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -3033,3 +3033,21 @@ func (s *testIntegrationSuite) TestIndexMergeTableFilter(c *C) { "10 1 1 10", )) } + +func (s *testIntegrationSuite) TestIssue23736(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t0, t1") + tk.MustExec("create table t0(a int, b int, c int as (a + b) virtual, unique index (c) invisible);") + tk.MustExec("create table t1(a int, b int, c int as (a + b) virtual);") + tk.MustExec("insert into t0(a, b) values (12, -1), (8, 7);") + tk.MustExec("insert into t1(a, b) values (12, -1), (8, 7);") + tk.MustQuery("select /*+ stream_agg() */ count(1) from t0 where c > 10 and b < 2;").Check(testkit.Rows("1")) + tk.MustQuery("select /*+ stream_agg() */ count(1) from t1 where c > 10 and b < 2;").Check(testkit.Rows("1")) + tk.MustExec("delete from t0") + tk.MustExec("insert into t0(a, b) values (5, 1);") + tk.MustQuery("select /*+ nth_plan(3) */ count(1) from t0 where c > 10 and b < 2;").Check(testkit.Rows("0")) + + // Should not use invisible index + c.Assert(tk.MustUseIndex("select /*+ stream_agg() */ count(1) from t0 where c > 10 and b < 2", "c"), IsFalse) +} diff --git a/planner/core/prepare_test.go b/planner/core/prepare_test.go index 6cdc6af508c56..e3a0390ae26ba 100644 --- a/planner/core/prepare_test.go +++ b/planner/core/prepare_test.go @@ -1024,3 +1024,32 @@ func (s *testPlanSerialSuite) TestPlanCacheSnapshot(c *C) { tk.MustQuery("execute stmt using @p").Check(testkit.Rows("1")) tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("1")) } + +func (s *testPlanSerialSuite) TestIssue23671(c *C) { + store, _, err := newStoreWithBootstrap() + c.Assert(err, IsNil) + tk := testkit.NewTestKit(c, store) + orgEnable := core.PreparedPlanCacheEnabled() + defer func() { + store.Close() + core.SetPreparedPlanCache(orgEnable) + }() + core.SetPreparedPlanCache(true) + + tk.Se, err = session.CreateSession4TestWithOpt(store, &session.Opt{ + PreparedPlanCache: kvcache.NewSimpleLRUCache(100, 0.1, math.MaxUint64), + }) + c.Assert(err, IsNil) + + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + + tk.MustExec("create table t (a int, b int, index ab(a, b))") + tk.MustExec("insert into t values (1, 1), (2, 2)") + tk.MustExec("prepare s1 from 'select * from t use index(ab) where a>=? and b>=? and b<=?'") + tk.MustExec("set @a=1, @b=1, @c=1") + tk.MustQuery("execute s1 using @a, @b, @c").Check(testkit.Rows("1 1")) + tk.MustExec("set @a=1, @b=1, @c=10") + tk.MustQuery("execute s1 using @a, @b, @c").Check(testkit.Rows("1 1", "2 2")) + tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0")) +} diff --git a/planner/core/task.go b/planner/core/task.go index 4944cd7ce6fac..fbfc5d3ce368d 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -957,7 +957,20 @@ func (t *copTask) convertToRootTaskImpl(ctx sessionctx.Context) *rootTask { }.Init(ctx, t.tablePlan.SelectBlockOffset()) p.PartitionInfo = t.partitionInfo p.stats = t.tablePlan.statsInfo() - if needExtraProj { + + // If agg was pushed down in attach2Task(), the partial agg was placed on the top of tablePlan, the final agg was + // placed above the PhysicalTableReader, and the schema should have been set correctly for them, the schema of + // partial agg contains the columns needed by the final agg. + // If we add the projection here, the projection will be between the final agg and the partial agg, then the + // schema will be broken, the final agg will fail to find needed columns in ResolveIndices(). + // Besides, the agg would only be pushed down if it doesn't contain virtual columns, so virtual column should not be affected. + aggPushedDown := false + switch p.tablePlan.(type) { + case *PhysicalHashAgg, *PhysicalStreamAgg: + aggPushedDown = true + } + + if needExtraProj && !aggPushedDown { proj := PhysicalProjection{Exprs: expression.Column2Exprs(prevSchema.Columns)}.Init(ts.ctx, ts.stats, ts.SelectBlockOffset(), nil) proj.SetSchema(prevSchema) proj.SetChildren(p) diff --git a/session/pessimistic_test.go b/session/pessimistic_test.go index 3341484d1cccb..200cba88e7c68 100644 --- a/session/pessimistic_test.go +++ b/session/pessimistic_test.go @@ -293,6 +293,7 @@ func (s *testPessimisticSuite) TestPointGetOverflow(c *C) { tk.MustExec("create table t(k tinyint, v int, unique key(k))") tk.MustExec("begin pessimistic") tk.MustExec("update t set v = 100 where k = -200;") + tk.MustExec("update t set v = 100 where k in (-200, -400);") } func (s *testPessimisticSuite) TestPointGetKeyLock(c *C) { diff --git a/session/schema_amender.go b/session/schema_amender.go index 5088bfb81e6e5..19c7ac5a9667f 100644 --- a/session/schema_amender.go +++ b/session/schema_amender.go @@ -442,7 +442,7 @@ func (a *amendOperationAddIndexInfo) genIndexKeyValue(ctx context.Context, sctx idxVals = append(idxVals, chk.GetRow(0).GetDatum(oldCol.Offset, &oldCol.FieldType)) } - rsData := tables.TryGetHandleRestoredDataWrapper(a.tblInfoAtCommit, getCommonHandleDatum(a.tblInfoAtCommit, chk.GetRow(0)), nil) + rsData := tables.TryGetHandleRestoredDataWrapper(a.tblInfoAtCommit, getCommonHandleDatum(a.tblInfoAtCommit, chk.GetRow(0)), nil, a.indexInfoAtCommit.Meta()) // Generate index key buf. newIdxKey, distinct, err := tablecodec.GenIndexKey(sctx.GetSessionVars().StmtCtx, diff --git a/table/tables/tables.go b/table/tables/tables.go index ee1a526a183ea..979aa13a2fde7 100644 --- a/table/tables/tables.go +++ b/table/tables/tables.go @@ -846,7 +846,7 @@ func (t *TableCommon) addIndices(sctx sessionctx.Context, recordID kv.Handle, r idxMeta := v.Meta() dupErr = kv.ErrKeyExists.FastGenByArgs(entryKey, idxMeta.Name.String()) } - rsData := TryGetHandleRestoredDataWrapper(t, r, nil) + rsData := TryGetHandleRestoredDataWrapper(t, r, nil, v.Meta()) if dupHandle, err := v.Create(sctx, txn, indexVals, recordID, rsData, opts...); err != nil { if kv.ErrKeyExists.Equal(err) { return dupHandle, dupErr @@ -1155,7 +1155,7 @@ func (t *TableCommon) buildIndexForRow(ctx sessionctx.Context, h kv.Handle, vals if untouched { opts = append(opts, table.IndexIsUntouched) } - rsData := TryGetHandleRestoredDataWrapper(t, newData, nil) + rsData := TryGetHandleRestoredDataWrapper(t, newData, nil, idx.Meta()) if _, err := idx.Create(ctx, txn, vals, h, rsData, opts...); err != nil { if kv.ErrKeyExists.Equal(err) { // Make error message consistent with MySQL. @@ -1695,47 +1695,56 @@ func (t *TableCommon) GetSequenceCommon() *sequenceCommon { } // TryGetHandleRestoredDataWrapper tries to get the restored data for handle if needed. The argument can be a slice or a map. -func TryGetHandleRestoredDataWrapper(t table.Table, row []types.Datum, rowMap map[int64]types.Datum) []types.Datum { +func TryGetHandleRestoredDataWrapper(t table.Table, row []types.Datum, rowMap map[int64]types.Datum, idx *model.IndexInfo) []types.Datum { if !collate.NewCollationEnabled() || !t.Meta().IsCommonHandle || t.Meta().CommonHandleVersion == 0 { return nil } - - useIDMap := false - if len(rowMap) > 0 { - useIDMap = true - } - - var datum types.Datum rsData := make([]types.Datum, 0, 4) - pkCols := TryGetCommonPkColumns(t) - for _, col := range pkCols { - if !types.NeedRestoredData(&col.FieldType) { + pkIdx := FindPrimaryIndex(t.Meta()) + for _, pkIdxCol := range pkIdx.Columns { + pkCol := t.Meta().Columns[pkIdxCol.Offset] + if !types.NeedRestoredData(&pkCol.FieldType) { continue } - if collate.IsBinCollation(col.Collate) { - if useIDMap { - datum = rowMap[col.ID] - } else { - datum = row[col.Offset] + var datum types.Datum + if len(rowMap) > 0 { + datum = rowMap[pkCol.ID] + } else { + datum = row[pkCol.Offset] + } + // Try to truncate index values. + // Says that primary key(a (8)), + // For index t(a), don't truncate the value. + // For index t(a(9)), truncate to a(9). + // For index t(a(7)), truncate to a(8). + truncateTargetCol := pkIdxCol + for _, idxCol := range idx.Columns { + if idxCol.Offset == pkCol.Offset { + truncateTargetCol = maxIndexLen(pkIdxCol, idxCol) + break } + } + tablecodec.TruncateIndexValue(&datum, truncateTargetCol, pkCol) + if collate.IsBinCollation(pkCol.Collate) { rsData = append(rsData, types.NewIntDatum(stringutil.GetTailSpaceCount(datum.GetString()))) } else { - if useIDMap { - rsData = append(rsData, rowMap[col.ID]) - } else { - rsData = append(rsData, row[col.Offset]) - } + rsData = append(rsData, datum) } } + return rsData +} - for _, idx := range t.Meta().Indices { - if idx.Primary { - tablecodec.TruncateIndexValues(t.Meta(), idx, rsData) - break - } +func maxIndexLen(idxA, idxB *model.IndexColumn) *model.IndexColumn { + if idxA.Length == types.UnspecifiedLength { + return idxA } - - return rsData + if idxB.Length == types.UnspecifiedLength { + return idxB + } + if idxA.Length > idxB.Length { + return idxA + } + return idxB } func getSequenceAllocator(allocs autoid.Allocators) (autoid.Allocator, error) { diff --git a/tablecodec/tablecodec.go b/tablecodec/tablecodec.go index b6331240934c1..215f8d05c27fa 100644 --- a/tablecodec/tablecodec.go +++ b/tablecodec/tablecodec.go @@ -576,11 +576,11 @@ func Unflatten(datum types.Datum, ft *types.FieldType, loc *time.Location) (type case mysql.TypeFloat: datum.SetFloat32(float32(datum.GetFloat64())) return datum, nil - case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString: + case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString, mysql.TypeTinyBlob, + mysql.TypeMediumBlob, mysql.TypeBlob, mysql.TypeLongBlob: datum.SetString(datum.GetString(), ft.Collate) case mysql.TypeTiny, mysql.TypeShort, mysql.TypeYear, mysql.TypeInt24, - mysql.TypeLong, mysql.TypeLonglong, mysql.TypeDouble, mysql.TypeTinyBlob, - mysql.TypeMediumBlob, mysql.TypeBlob, mysql.TypeLongBlob: + mysql.TypeLong, mysql.TypeLonglong, mysql.TypeDouble: return datum, nil case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp: t := types.NewTime(types.ZeroCoreTime, ft.Tp, int8(ft.Decimal)) diff --git a/tablecodec/tablecodec_test.go b/tablecodec/tablecodec_test.go index 82ad65c32b5d8..4b416279cb015 100644 --- a/tablecodec/tablecodec_test.go +++ b/tablecodec/tablecodec_test.go @@ -222,6 +222,16 @@ func (s *testTableCodecSuite) TestUnflattenDatums(c *C) { cmp, err := input[0].CompareDatum(sc, &output[0]) c.Assert(err, IsNil) c.Assert(cmp, Equals, 0) + + input = []types.Datum{types.NewCollationStringDatum("aaa", "utf8mb4_unicode_ci", 0)} + tps = []*types.FieldType{types.NewFieldType(mysql.TypeBlob)} + tps[0].Collate = "utf8mb4_unicode_ci" + output, err = UnflattenDatums(input, tps, sc.TimeZone) + c.Assert(err, IsNil) + cmp, err = input[0].CompareDatum(sc, &output[0]) + c.Assert(err, IsNil) + c.Assert(cmp, Equals, 0) + c.Assert(output[0].Collation(), Equals, "utf8mb4_unicode_ci") } func (s *testTableCodecSuite) TestTimeCodec(c *C) { diff --git a/util/ranger/detacher.go b/util/ranger/detacher.go index 817ab2b8c2004..fdd0c21fa45ee 100644 --- a/util/ranger/detacher.go +++ b/util/ranger/detacher.go @@ -505,6 +505,7 @@ func ExtractEqAndInCondition(sctx sessionctx.Context, conditions []expression.Ex // All Intervals are single points accesses[i] = points2EqOrInCond(sctx, points[i], cols[i]) newConditions = append(newConditions, accesses[i]) + sctx.GetSessionVars().StmtCtx.OptimDependOnMutableConst = true } } for i, offset := range offsets { diff --git a/util/vitess/vitess_hash.go b/util/vitess/vitess_hash.go new file mode 100644 index 0000000000000..08e2d57f2c527 --- /dev/null +++ b/util/vitess/vitess_hash.go @@ -0,0 +1,42 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package vitess + +import ( + "crypto/cipher" + "crypto/des" + "encoding/binary" + + "github.com/pingcap/errors" +) + +var nullKeyBlock cipher.Block + +func init() { + var err error + nullKeyBlock, err = des.NewCipher(make([]byte, 8)) + if err != nil { + panic(errors.Trace(err)) + } +} + +// HashUint64 implements vitess' method of calculating a hash used for determining a shard key range. +// Uses a DES encryption with 64 bit key, 64 bit block, null-key +func HashUint64(shardKey uint64) (uint64, error) { + var keybytes [8]byte + binary.BigEndian.PutUint64(keybytes[:], shardKey) + var hashed [8]byte + nullKeyBlock.Encrypt(hashed[:], keybytes[:]) + return binary.BigEndian.Uint64(hashed[:]), nil +} diff --git a/util/vitess/vitess_hash_test.go b/util/vitess/vitess_hash_test.go new file mode 100644 index 0000000000000..70a4b5aaf0b83 --- /dev/null +++ b/util/vitess/vitess_hash_test.go @@ -0,0 +1,66 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package vitess + +import ( + "encoding/binary" + "encoding/hex" + "math" + "strings" + "testing" + + . "github.com/pingcap/check" + "github.com/pingcap/tidb/util/testleak" +) + +var _ = Suite(&testVitessSuite{}) + +func TestT(t *testing.T) { + TestingT(t) +} + +type testVitessSuite struct { +} + +func toHex(value uint64) string { + var keybytes [8]byte + binary.BigEndian.PutUint64(keybytes[:], value) + return strings.ToUpper(hex.EncodeToString(keybytes[:])) +} + +var _ = Suite(&testVitessSuite{}) + +func (s *testVitessSuite) TestVitessHash(c *C) { + defer testleak.AfterTest(c)() + + hashed, err := HashUint64(30375298039) + c.Assert(err, IsNil) + c.Assert(toHex(hashed), Equals, "031265661E5F1133") + + hashed, err = HashUint64(1123) + c.Assert(err, IsNil) + c.Assert(toHex(hashed), Equals, "031B565D41BDF8CA") + + hashed, err = HashUint64(30573721600) + c.Assert(err, IsNil) + c.Assert(toHex(hashed), Equals, "1EFD6439F2050FFD") + + hashed, err = HashUint64(116) + c.Assert(err, IsNil) + c.Assert(toHex(hashed), Equals, "1E1788FF0FDE093C") + + hashed, err = HashUint64(math.MaxUint64) + c.Assert(err, IsNil) + c.Assert(toHex(hashed), Equals, "355550B2150E2451") +}