Allow WHERE terms to be pushed down into sub-queries that contain window functions, provided that the WHERE term is made up of entirely of constants and copies of expressions found in the PARTITION BY clauses of all window functions in the sub-query.

Author: danielk-1977

src/select.c

@@ -4489,6 +4489,35 @@ static int propagateConstants(
   return nChng;
 }
 
+#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
+# if !defined(SQLITE_OMIT_WINDOWFUNC)
+/*
+** This function is called to determine whether or not it is safe to
+** push WHERE clause expression pExpr down to FROM clause sub-query
+** pSubq, which contains at least one window function. Return 1
+** if it is safe and the expression should be pushed down, or 0 
+** otherwise.
+**
+** It is only safe to push the expression down if it consists only 
+** of constants and copies of expressions that appear in the PARTITION
+** BY clause of all window function used by the sub-query. It is safe
+** to filter out entire partitions, but not rows within partitions, as
+** this may change the results of the window functions.
+**
+** At the time this function is called it is guaranteed that 
+**
+**   * the sub-query uses only one distinct window frame, and 
+**   * that the window frame has a PARTITION BY clase.
+*/
+static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){
+  assert( pSubq->pWin->pPartition );
+  assert( (pSubq->selFlags & SF_MultiPart)==0 );
+  assert( pSubq->pPrior==0 );
+  return sqlite3ExprIsConstantOrGroupBy(pParse, pExpr, pSubq->pWin->pPartition);
+}
+# endif /* SQLITE_OMIT_WINDOWFUNC */
+#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
+
 #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
 /*
 ** Make copies of relevant WHERE clause terms of the outer query into
@@ -4536,9 +4565,20 @@ static int propagateConstants(
 **       But if the (b2=2) term were to be pushed down into the bb subquery,
 **       then the (1,1,NULL) row would be suppressed.
 **
-**   (6) The inner query features one or more window-functions (since 
-**       changes to the WHERE clause of the inner query could change the 
-**       window over which window functions are calculated).
+**   (6) Window functions make things tricky as changes to the WHERE clause 
+**       of the inner query could change the window over which window 
+**       functions are calculated. Therefore, do not attempt the optimization
+**       if:
+**
+**     (6a) The inner query uses multiple incompatible window partitions.
+**
+**     (6b) The inner query is a compound and uses window-functions. 
+**
+**     (6c) The WHERE clause does not consist entirely of constants and
+**          copies of expressions found in the PARTITION BY clause of
+**          all window-functions used by the sub-query. It is safe to
+**          filter out entire partitions, as this does not change the 
+**          window over which any window-function is calculated.
 **
 **   (7) The inner query is a Common Table Expression (CTE) that should
 **       be materialized.  (This restriction is implemented in the calling
@@ -4556,13 +4596,17 @@ static int pushDownWhereTerms(
 ){
   Expr *pNew;
   int nChng = 0;
-  Select *pSel;
   if( pWhere==0 ) return 0;
-  if( pSubq->selFlags & SF_Recursive ) return 0;  /* restriction (2) */
+  if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ) return 0;
 
 #ifndef SQLITE_OMIT_WINDOWFUNC
-  for(pSel=pSubq; pSel; pSel=pSel->pPrior){
-    if( pSel->pWin ) return 0;    /* restriction (6) */
+  if( pSubq->pPrior ){
+    Select *pSel;
+    for(pSel=pSubq; pSel; pSel=pSel->pPrior){
+      if( pSel->pWin ) return 0;    /* restriction (6b) */
+    }
+  }else{
+    if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0;
   }
 #endif
 
@@ -4609,6 +4653,14 @@ static int pushDownWhereTerms(
       x.isLeftJoin = 0;
       x.pEList = pSubq->pEList;
       pNew = substExpr(&x, pNew);
+#ifndef SQLITE_OMIT_WINDOWFUNC
+      if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){
+        /* Restriction 6c has prevented push-down in this case */
+        sqlite3ExprDelete(pParse->db, pNew);
+        nChng--;
+        break;
+      }
+#endif
       if( pSubq->selFlags & SF_Aggregate ){
         pSubq->pHaving = sqlite3ExprAnd(pParse, pSubq->pHaving, pNew);
       }else{

src/sqliteInt.h

@@ -3204,6 +3204,7 @@ struct Select {
 #define SF_NoopOrderBy   0x0400000 /* ORDER BY is ignored for this query */
 #define SF_UpdateFrom    0x0800000 /* Statement is an UPDATE...FROM */
 #define SF_PushDown      0x1000000 /* SELECT has be modified by push-down opt */
+#define SF_MultiPart     0x2000000 /* Has multiple incompatible PARTITIONs */
 
 /*
 ** The results of a SELECT can be distributed in several ways, as defined

src/window.c

@@ -1304,15 +1304,19 @@ void sqlite3WindowAttach(Parse *pParse, Expr *p, Window *pWin){
 ** SELECT, or (b) the windows already linked use a compatible window frame.
 */
 void sqlite3WindowLink(Select *pSel, Window *pWin){
-  if( pSel!=0
-   && (0==pSel->pWin || 0==sqlite3WindowCompare(0, pSel->pWin, pWin, 0))
-  ){
-    pWin->pNextWin = pSel->pWin;
-    if( pSel->pWin ){
-      pSel->pWin->ppThis = &pWin->pNextWin;
+  if( pSel ){
+    if( 0==pSel->pWin || 0==sqlite3WindowCompare(0, pSel->pWin, pWin, 0) ){
+      pWin->pNextWin = pSel->pWin;
+      if( pSel->pWin ){
+        pSel->pWin->ppThis = &pWin->pNextWin;
+      }
+      pSel->pWin = pWin;
+      pWin->ppThis = &pSel->pWin;
+    }else{
+      if( sqlite3ExprListCompare(pWin->pPartition, pSel->pWin->pPartition,-1) ){
+        pSel->selFlags |= SF_MultiPart;
+      }
     }
-    pSel->pWin = pWin;
-    pWin->ppThis = &pSel->pWin;
   }
 }
 

test/windowpushd.test

@@ -0,0 +1,160 @@
+# 2021 February 23
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the push-down optimization when
+# WHERE constraints are pushed down into a sub-query that uses
+# window functions.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix windowpushd
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(id INTEGER PRIMARY KEY, grp_id);
+  CREATE INDEX i1 ON t1(grp_id);
+  CREATE VIEW lll AS SELECT
+    row_number() OVER (PARTITION BY grp_id), 
+    grp_id, id 
+  FROM t1
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO t1 VALUES
+    (1, 2), (2, 3), (3, 3), (4, 1), (5, 1),
+    (6, 1), (7, 1), (8, 1), (9, 3), (10, 3), 
+    (11, 2), (12, 3), (13, 3), (14, 2), (15, 1),
+    (16, 2), (17, 1), (18, 2), (19, 3), (20, 2)
+}
+
+do_execsql_test 1.2 {
+  SELECT * FROM lll
+} {
+  1 1 4 2 1 5 3 1 6 4 1 7 5 1 8 6 1 15 7 1 17 
+  1 2 1 2 2 11 3 2 14 4 2 16 5 2 18 6 2 20 
+  1 3 2 2 3 3 3 3 9 4 3 10 5 3 12 6 3 13 7 3 19
+}
+
+do_execsql_test 1.3 {
+  SELECT * FROM lll WHERE grp_id=2
+} {
+  1 2 1 2 2 11 3 2 14 4 2 16 5 2 18 6 2 20 
+}
+
+do_eqp_test 1.4 {
+  SELECT * FROM lll WHERE grp_id=2
+} {SEARCH TABLE t1 USING COVERING INDEX i1 (grp_id=?)}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE TABLE t1(a, b, c, d);
+  INSERT INTO t1 VALUES('A', 'C', 1,  0.1);
+  INSERT INTO t1 VALUES('A', 'D', 2,  0.2);
+  INSERT INTO t1 VALUES('A', 'E', 3,  0.3);
+  INSERT INTO t1 VALUES('A', 'C', 4,  0.4);
+  INSERT INTO t1 VALUES('B', 'D', 5,  0.5);
+  INSERT INTO t1 VALUES('B', 'E', 6,  0.6);
+  INSERT INTO t1 VALUES('B', 'C', 7,  0.7);
+  INSERT INTO t1 VALUES('B', 'D', 8,  0.8);
+  INSERT INTO t1 VALUES('C', 'E', 9,  0.9);
+  INSERT INTO t1 VALUES('C', 'C', 10, 1.0);
+  INSERT INTO t1 VALUES('C', 'D', 11, 1.1);
+  INSERT INTO t1 VALUES('C', 'E', 12, 1.2);
+
+  CREATE INDEX i1 ON t1(a);
+  CREATE INDEX i2 ON t1(b);
+
+  CREATE VIEW v1 AS SELECT a, c, max(c) OVER (PARTITION BY a) FROM t1;
+
+  CREATE VIEW v2 AS SELECT a, c, 
+      max(c) OVER (PARTITION BY a),
+      row_number() OVER ()
+  FROM t1;
+
+  CREATE VIEW v3 AS SELECT b, d, 
+      max(d) OVER (PARTITION BY b),
+      row_number() OVER (PARTITION BY b)
+  FROM t1;
+}
+
+foreach tn {0 1} {
+  optimization_control db push-down $tn
+
+  do_execsql_test 2.$tn.1.1 {
+    SELECT * FROM v1;
+  } {
+    A 1 4   A 2 4   A 3 4   A 4 4
+    B 5 8   B 6 8   B 7 8   B 8 8
+    C 9 12  C 10 12 C 11 12 C 12 12
+  }
+
+  do_execsql_test 2.$tn.1.2 {
+    SELECT * FROM v1 WHERE a IN ('A', 'B');
+  } {
+    A 1 4   A 2 4   A 3 4   A 4 4
+    B 5 8   B 6 8   B 7 8   B 8 8
+  }
+
+  do_execsql_test 2.$tn.1.3 {
+    SELECT * FROM v1 WHERE a IS 'C'
+  } {
+    C 9 12  C 10 12 C 11 12 C 12 12
+  }
+
+  if {$tn==1} {
+    do_eqp_test 2.$tn.1.4 {
+      SELECT * FROM v1 WHERE a IN ('A', 'B');
+    } {USING INDEX i1 (a=?)}
+
+    do_eqp_test 2.$tn.1.5 {
+      SELECT * FROM v1 WHERE a = 'c' COLLATE nocase
+    } {USING INDEX i1}
+  }
+
+  do_execsql_test 2.$tn.2.1 {
+    SELECT * FROM v2;
+  } {
+    A 1 4 1    A 2 4 2     A 3 4 3      A 4 4 4
+    B 5 8 5    B 6 8 6     B 7 8 7      B 8 8 8
+    C 9 12 9   C 10 12 10  C 11 12 11   C 12 12 12
+  }
+
+  do_execsql_test 2.$tn.2.2 {
+    SELECT * FROM v2 WHERE a = 'C';
+  } {
+    C 9 12 9   C 10 12 10  C 11 12 11   C 12 12 12
+  }
+
+  do_execsql_test 2.$tn.3.1 { SELECT * FROM v3; } { 
+    C 0.1 1.0 1 C 0.4 1.0 2 C 0.7 1.0 3 C 1.0 1.0 4 
+    D 0.2 1.1 1 D 0.5 1.1 2 D 0.8 1.1 3 D 1.1 1.1 4 
+    E 0.3 1.2 1 E 0.6 1.2 2 E 0.9 1.2 3 E 1.2 1.2 4
+  }
+
+  do_execsql_test 2.$tn.3.2 { SELECT * FROM v3 WHERE b<'E' } { 
+    C 0.1 1.0 1 C 0.4 1.0 2 C 0.7 1.0 3 C 1.0 1.0 4 
+    D 0.2 1.1 1 D 0.5 1.1 2 D 0.8 1.1 3 D 1.1 1.1 4 
+  }
+
+  if {$tn==1} {
+    do_eqp_test 2.$tn.3.3 {
+      SELECT * FROM v3 WHERE b='E'
+    } {USING INDEX i2 (b=?)}
+  }
+
+}
+
+
+
+
+finish_test
+