planner, util/ranger: apply PushDownNot to condition before pruning p…

…artition (pingcap#23628) (pingcap#23641)

planner/core/integration_test.go

@@ -2716,10 +2716,17 @@ func (s *testIntegrationSuite) TestIssue22199(c *C) {
 func (s *testIntegrationSuite) TestIssue22892(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
+	tk.MustExec("set @@tidb_partition_prune_mode='static'")
 	tk.MustExec("drop table if exists t1")
 	tk.MustExec("create table t1(a int) partition by hash (a) partitions 5;")
 	tk.MustExec("insert into t1 values (0);")
 	tk.MustQuery("select * from t1 where a not between 1 and 2;").Check(testkit.Rows("0"))
+
+	tk.MustExec("set @@tidb_partition_prune_mode='dynamic'")
+	tk.MustExec("drop table if exists t2")
+	tk.MustExec("create table t2(a int) partition by hash (a) partitions 5;")
+	tk.MustExec("insert into t2 values (0);")
+	tk.MustQuery("select * from t2 where a not between 1 and 2;").Check(testkit.Rows("0"))
 }
 
 func (s *testIntegrationSerialSuite) TestPushDownProjectionForTiFlash(c *C) {

planner/core/partition_prune.go

@@ -27,6 +27,12 @@ func PartitionPruning(ctx sessionctx.Context, tbl table.PartitionedTable, conds
 	columns []*expression.Column, names types.NameSlice) ([]int, error) {
 	s := partitionProcessor{}
 	pi := tbl.Meta().Partition
+	// PushDownNot here can convert condition 'not (a != 1)' to 'a = 1'. When we build range from conds, the condition like
+	// 'not (a != 1)' would not be handled so we need to convert it to 'a = 1', which can be handled when building range.
+	// TODO: there may be a better way to push down Not once for all.
+	for i, cond := range conds {
+		conds[i] = expression.PushDownNot(ctx, cond)
+	}
 	switch pi.Type {
 	case model.PartitionTypeHash:
 		return s.pruneHashPartition(ctx, tbl, partitionNames, conds, columns, names)

planner/core/partition_pruner_test.go

@@ -498,3 +498,48 @@ func (s *testPartitionPruneSuit) Test22396(c *C) {
 	tk.MustQuery("SELECT * FROM test WHERE a = 1 AND b = 1;")
 	tk.MustQuery("SELECT * FROM test WHERE a + b = 2;")
 }
+
+func (s *testPartitionPruneSuit) TestIssue23608(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec("set @@tidb_partition_prune_mode='static'")
+	tk.MustExec("drop table if exists t1")
+	tk.MustExec("create table t1(a int) partition by hash (a) partitions 10")
+	tk.MustExec("insert into t1 values (1), (2), (12), (3), (11), (13)")
+	tk.MustQuery("select * from t1 where a not between 2 and 2").Sort().Check(testkit.Rows("1", "11", "12", "13", "3"))
+	tk.MustQuery("select * from t1 where not (a < -20 or a > 20)").Sort().Check(testkit.Rows("1", "11", "12", "13", "2", "3"))
+	tk.MustQuery("select * from t1 where not (a > 0 and a < 10)").Sort().Check(testkit.Rows("11", "12", "13"))
+	tk.MustQuery("select * from t1 where not (a < -20)").Sort().Check(testkit.Rows("1", "11", "12", "13", "2", "3"))
+	tk.MustQuery("select * from t1 where not (a > 20)").Sort().Check(testkit.Rows("1", "11", "12", "13", "2", "3"))
+	tk.MustQuery("select * from t1 where not (a = 1)").Sort().Check(testkit.Rows("11", "12", "13", "2", "3"))
+	tk.MustQuery("select * from t1 where not (a != 1)").Check(testkit.Rows("1"))
+
+	tk.MustExec("drop table if exists t2")
+	tk.MustExec(`
+create table t2(a int)
+partition by range (a) (
+    partition p0 values less than (0),
+    partition p1 values less than (10),
+    partition p2 values less than (20)
+)`)
+	tk.MustQuery("explain format = 'brief' select * from t2 where not (a < 5)").Check(testkit.Rows(
+		"PartitionUnion 6666.67 root  ",
+		"├─TableReader 3333.33 root  data:Selection",
+		"│ └─Selection 3333.33 cop[tikv]  ge(test.t2.a, 5)",
+		"│   └─TableFullScan 10000.00 cop[tikv] table:t2, partition:p1 keep order:false, stats:pseudo",
+		"└─TableReader 3333.33 root  data:Selection",
+		"  └─Selection 3333.33 cop[tikv]  ge(test.t2.a, 5)",
+		"    └─TableFullScan 10000.00 cop[tikv] table:t2, partition:p2 keep order:false, stats:pseudo"))
+
+	tk.MustExec("set @@tidb_partition_prune_mode='dynamic'")
+	tk.MustExec("drop table if exists t3")
+	tk.MustExec("create table t3(a int) partition by hash (a) partitions 10")
+	tk.MustExec("insert into t3 values (1), (2), (12), (3), (11), (13)")
+	tk.MustQuery("select * from t3 where a not between 2 and 2").Sort().Check(testkit.Rows("1", "11", "12", "13", "3"))
+	tk.MustQuery("select * from t3 where not (a < -20 or a > 20)").Sort().Check(testkit.Rows("1", "11", "12", "13", "2", "3"))
+	tk.MustQuery("select * from t3 where not (a > 0 and a < 10)").Sort().Check(testkit.Rows("11", "12", "13"))
+	tk.MustQuery("select * from t3 where not (a < -20)").Sort().Check(testkit.Rows("1", "11", "12", "13", "2", "3"))
+	tk.MustQuery("select * from t3 where not (a > 20)").Sort().Check(testkit.Rows("1", "11", "12", "13", "2", "3"))
+	tk.MustQuery("select * from t3 where not (a = 1)").Sort().Check(testkit.Rows("11", "12", "13", "2", "3"))
+	tk.MustQuery("select * from t3 where not (a != 1)").Check(testkit.Rows("1"))
+}

planner/core/rule_partition_processor.go

@@ -534,6 +534,12 @@ func (s *partitionProcessor) prune(ds *DataSource) (LogicalPlan, error) {
 	if pi == nil {
 		return ds, nil
 	}
+	// PushDownNot here can convert condition 'not (a != 1)' to 'a = 1'. When we build range from ds.allConds, the condition
+	// like 'not (a != 1)' would not be handled so we need to convert it to 'a = 1', which can be handled when building range.
+	// TODO: there may be a better way to push down Not once for all.
+	for i, cond := range ds.allConds {
+		ds.allConds[i] = expression.PushDownNot(ds.ctx, cond)
+	}
 	// Try to locate partition directly for hash partition.
 	switch pi.Type {
 	case model.PartitionTypeRange:

util/ranger/points.go

@@ -638,10 +638,11 @@ func (r *builder) buildFromNot(expr *expression.ScalarFunction) []*point {
 		startPoint := &point{value: types.MinNotNullDatum(), start: true}
 		endPoint := &point{value: types.MaxValueDatum()}
 		return []*point{startPoint, endPoint}
-	case ast.LogicAnd:
-		return r.intersection(r.build(expr.GetArgs()[0]), r.build(expr.GetArgs()[1]))
 	}
-	return nil
+	// TODO: currently we don't handle ast.LogicAnd, ast.LogicOr, ast.GT, ast.LT and so on. Most of those cases are eliminated
+	// by PushDownNot but they may happen. For now, we return full range for those unhandled cases in order to keep correctness.
+	// Later we need to cover those cases and set r.err when meeting some unexpected case.
+	return getFullRange()
 }
 
 func (r *builder) buildFromScalarFunc(expr *expression.ScalarFunction) []*point {