Fix incorrect NULL IN () optimization

Author: findepi

datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs

@@ -1663,20 +1663,20 @@ impl<S: SimplifyInfo> TreeNodeRewriter for Simplifier<'_, S> {
             // expr IN () --> false
             // expr NOT IN () --> true
             Expr::InList(InList {
-                expr,
+                expr: _,
                 list,
                 negated,
-            }) if list.is_empty() && *expr != Expr::Literal(ScalarValue::Null, None) => {
-                Transformed::yes(lit(negated))
-            }
+            }) if list.is_empty() => Transformed::yes(lit(negated)),
 
             // null in (x, y, z) --> null
             // null not in (x, y, z) --> null
             Expr::InList(InList {
                 expr,
-                list: _,
+                list,
                 negated: _,
-            }) if is_null(expr.as_ref()) => Transformed::yes(lit_bool_null()),
+            }) if is_null(expr.as_ref()) && !list.is_empty() => {
+                Transformed::yes(lit_bool_null())
+            }
 
             // expr IN ((subquery)) -> expr IN (subquery), see ##5529
             Expr::InList(InList {

datafusion/sqllogictest/test_files/predicates.slt

@@ -811,5 +811,26 @@ explain select x from t where x NOT IN (1,2,3,4,5) AND x IN (1,2,3);
 logical_plan EmptyRelation
 physical_plan EmptyExec
 
+query error DataFusion error: This feature is not implemented: Physical plan does not support logical expression InSubquery\(InSubquery \{ expr: Literal\(Int64\(NULL\), None\), subquery: <subquery>, negated: false \}\)
+WITH empty AS (SELECT 10 WHERE false)
+SELECT
+    NULL IN (SELECT * FROM empty), -- should be false, as the right side is empty relation
+    NULL NOT IN (SELECT * FROM empty) -- should be true, as the right side is empty relation
+FROM (SELECT 1) t;
+
+query I
+WITH empty AS (SELECT 10 WHERE false)
+SELECT * FROM (SELECT 1) t
+WHERE NOT (NULL IN (SELECT * FROM empty)); -- all rows should be returned
+----
+1
+
+query I
+WITH empty AS (SELECT 10 WHERE false)
+SELECT * FROM (SELECT 1) t
+WHERE NULL NOT IN (SELECT * FROM empty); -- all rows should be returned
+----
+1
+
 statement ok
 drop table t;