util/ranger: Handle boundary value correctly in ranger to avoid incor…

…rect tableDual plan (#52225) (#53320)

close #50051

util/ranger/detacher.go

@@ -491,7 +491,9 @@ func (d *rangeDetacher) detachCNFCondAndBuildRangeForIndex(conditions []expressi
 // excludeToIncludeForIntPoint converts `(i` to `[i+1` and `i)` to `i-1]` if `i` is integer.
 // For example, if p is `(3`, i.e., point { value: int(3), excl: true, start: true }, it is equal to `[4`, i.e., point { value: int(4), excl: false, start: true }.
 // Similarly, if p is `8)`, i.e., point { value: int(8), excl: true, start: false}, it is equal to `7]`, i.e., point { value: int(7), excl: false, start: false }.
-// If return value is nil, it means p is unsatisfiable. For example, `(MaxInt64` is unsatisfiable.
+// If return value is nil, it means p is unsatisfiable. For example, `(MaxUint64` is unsatisfiable.
+// The boundary value will be treated as the bigger type: For example, `(MaxInt64` of type KindInt64 will become `[MaxInt64+1` of type KindUint64,
+// and vice versa for `0)` of type KindUint64 will become `-1]` of type KindInt64.
 func excludeToIncludeForIntPoint(p *point) *point {
 	if !p.excl {
 		return p
@@ -500,9 +502,10 @@ func excludeToIncludeForIntPoint(p *point) *point {
 		val := p.value.GetInt64()
 		if p.start {
 			if val == math.MaxInt64 {
-				return nil
+				p.value.SetUint64(uint64(val + 1))
+			} else {
+				p.value.SetInt64(val + 1)
 			}
-			p.value.SetInt64(val + 1)
 			p.excl = false
 		} else {
 			if val == math.MinInt64 {
@@ -521,9 +524,10 @@ func excludeToIncludeForIntPoint(p *point) *point {
 			p.excl = false
 		} else {
 			if val == 0 {
-				return nil
+				p.value.SetInt64(int64(val - 1))
+			} else {
+				p.value.SetUint64(val - 1)
 			}
-			p.value.SetUint64(val - 1)
 			p.excl = false
 		}
 	}

util/ranger/ranger_test.go

@@ -2620,3 +2620,22 @@ func TestIssue44389(t *testing.T) {
 		testKit.MustQuery(tt).Sort().Check(testkit.Rows(output[i].Result...))
 	}
 }
+
+func TestIssue50051(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+
+	tk.MustExec("drop table if exists tt")
+	tk.MustExec("CREATE TABLE tt (c bigint UNSIGNED not null, d int not null, PRIMARY KEY (c,d));")
+	tk.MustExec("insert into tt values (9223372036854775810, 3);")
+	tk.MustQuery("SELECT c FROM tt WHERE c>9223372036854775807 AND c>1;").Check(testkit.Rows("9223372036854775810"))
+
+	tk.MustExec("drop table if exists t5")
+	tk.MustExec("drop table if exists t6")
+	tk.MustExec("CREATE TABLE `t5` (`d` int not null, `c` int not null, PRIMARY KEY (`d`, `c`));")
+	tk.MustExec("CREATE TABLE `t6` (`d` bigint UNSIGNED not null);")
+	tk.MustExec("insert into t5 values (-3, 6);")
+	tk.MustExec("insert into t6 values (0), (1), (2), (3);")
+	tk.MustQuery("select d from t5 where d < (select min(d) from t6) and d < 3;").Check(testkit.Rows("-3"))
+}